diff --git a/.circleci/config.yml b/.circleci/config.yml index a9615f7..9e8dbce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,7 @@ jobs: description: gotestsum format. https://github.com/gotestyourself/gotestsum#format coverage: type: string - default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/venus-miner/... + default: -coverprofile=coverage.txt -coverpkg=./... description: Coverage flag. Set to the empty string to disable. codecov-upload: type: boolean @@ -131,7 +131,7 @@ jobs: - install-deps - prepare - run: - command: make deps miner + command: make deps no_output_timeout: 30m - run: name: go test diff --git a/api/api_common.go b/api/api_common.go index 8499512..7afce8c 100644 --- a/api/api_common.go +++ b/api/api_common.go @@ -42,11 +42,6 @@ type APIVersion struct { // // See APIVersion in build/version.go APIVersion api.Version - - // TODO: git commit / os / genesis cid? - - // Seconds - BlockDelay uint64 } func (v APIVersion) String() string { diff --git a/api/api_miner.go b/api/api_miner.go index 2d2531f..35ca91e 100644 --- a/api/api_miner.go +++ b/api/api_miner.go @@ -2,21 +2,19 @@ package api import ( "context" + "github.com/filecoin-project/venus-miner/types" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - - "github.com/filecoin-project/venus-miner/node/modules/dtypes" ) type MinerAPI interface { Common - UpdateAddress(context.Context, int64, int64) ([]dtypes.MinerInfo, error) //perm:write - ListAddress(context.Context) ([]dtypes.MinerInfo, error) //perm:read - StatesForMining(context.Context, []address.Address) ([]dtypes.MinerState, error) //perm:read - CountWinners(context.Context, []address.Address, abi.ChainEpoch, abi.ChainEpoch) ([]dtypes.CountWinners, error) //perm:read - Start(context.Context, []address.Address) error //perm:admin - Stop(context.Context, []address.Address) error //perm:admin - AddAddress(context.Context, dtypes.MinerInfo) error //perm:admin + UpdateAddress(context.Context, int64, int64) ([]types.MinerInfo, error) //perm:write + ListAddress(context.Context) ([]types.MinerInfo, error) //perm:read + StatesForMining(context.Context, []address.Address) ([]types.MinerState, error) //perm:read + CountWinners(context.Context, []address.Address, abi.ChainEpoch, abi.ChainEpoch) ([]types.CountWinners, error) //perm:read + Start(context.Context, []address.Address) error //perm:admin + Stop(context.Context, []address.Address) error //perm:admin } diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 196df4c..41491a5 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -5,11 +5,11 @@ package api import ( "context" "errors" + "github.com/filecoin-project/venus-miner/types" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" "github.com/google/uuid" ) @@ -42,19 +42,17 @@ type MinerAPIStruct struct { CommonStruct Internal struct { - AddAddress func(p0 context.Context, p1 dtypes.MinerInfo) error `perm:"admin"` + CountWinners func(p0 context.Context, p1 []address.Address, p2 abi.ChainEpoch, p3 abi.ChainEpoch) ([]types.CountWinners, error) `perm:"read"` - CountWinners func(p0 context.Context, p1 []address.Address, p2 abi.ChainEpoch, p3 abi.ChainEpoch) ([]dtypes.CountWinners, error) `perm:"read"` - - ListAddress func(p0 context.Context) ([]dtypes.MinerInfo, error) `perm:"read"` + ListAddress func(p0 context.Context) ([]types.MinerInfo, error) `perm:"read"` Start func(p0 context.Context, p1 []address.Address) error `perm:"admin"` - StatesForMining func(p0 context.Context, p1 []address.Address) ([]dtypes.MinerState, error) `perm:"read"` + StatesForMining func(p0 context.Context, p1 []address.Address) ([]types.MinerState, error) `perm:"read"` Stop func(p0 context.Context, p1 []address.Address) error `perm:"admin"` - UpdateAddress func(p0 context.Context, p1 int64, p2 int64) ([]dtypes.MinerInfo, error) `perm:"write"` + UpdateAddress func(p0 context.Context, p1 int64, p2 int64) ([]types.MinerInfo, error) `perm:"write"` } } @@ -150,37 +148,26 @@ func (s *CommonStub) Version(p0 context.Context) (APIVersion, error) { return *new(APIVersion), ErrNotSupported } -func (s *MinerAPIStruct) AddAddress(p0 context.Context, p1 dtypes.MinerInfo) error { - if s.Internal.AddAddress == nil { - return ErrNotSupported - } - return s.Internal.AddAddress(p0, p1) -} - -func (s *MinerAPIStub) AddAddress(p0 context.Context, p1 dtypes.MinerInfo) error { - return ErrNotSupported -} - -func (s *MinerAPIStruct) CountWinners(p0 context.Context, p1 []address.Address, p2 abi.ChainEpoch, p3 abi.ChainEpoch) ([]dtypes.CountWinners, error) { +func (s *MinerAPIStruct) CountWinners(p0 context.Context, p1 []address.Address, p2 abi.ChainEpoch, p3 abi.ChainEpoch) ([]types.CountWinners, error) { if s.Internal.CountWinners == nil { - return *new([]dtypes.CountWinners), ErrNotSupported + return *new([]types.CountWinners), ErrNotSupported } return s.Internal.CountWinners(p0, p1, p2, p3) } -func (s *MinerAPIStub) CountWinners(p0 context.Context, p1 []address.Address, p2 abi.ChainEpoch, p3 abi.ChainEpoch) ([]dtypes.CountWinners, error) { - return *new([]dtypes.CountWinners), ErrNotSupported +func (s *MinerAPIStub) CountWinners(p0 context.Context, p1 []address.Address, p2 abi.ChainEpoch, p3 abi.ChainEpoch) ([]types.CountWinners, error) { + return *new([]types.CountWinners), ErrNotSupported } -func (s *MinerAPIStruct) ListAddress(p0 context.Context) ([]dtypes.MinerInfo, error) { +func (s *MinerAPIStruct) ListAddress(p0 context.Context) ([]types.MinerInfo, error) { if s.Internal.ListAddress == nil { - return *new([]dtypes.MinerInfo), ErrNotSupported + return *new([]types.MinerInfo), ErrNotSupported } return s.Internal.ListAddress(p0) } -func (s *MinerAPIStub) ListAddress(p0 context.Context) ([]dtypes.MinerInfo, error) { - return *new([]dtypes.MinerInfo), ErrNotSupported +func (s *MinerAPIStub) ListAddress(p0 context.Context) ([]types.MinerInfo, error) { + return *new([]types.MinerInfo), ErrNotSupported } func (s *MinerAPIStruct) Start(p0 context.Context, p1 []address.Address) error { @@ -194,15 +181,15 @@ func (s *MinerAPIStub) Start(p0 context.Context, p1 []address.Address) error { return ErrNotSupported } -func (s *MinerAPIStruct) StatesForMining(p0 context.Context, p1 []address.Address) ([]dtypes.MinerState, error) { +func (s *MinerAPIStruct) StatesForMining(p0 context.Context, p1 []address.Address) ([]types.MinerState, error) { if s.Internal.StatesForMining == nil { - return *new([]dtypes.MinerState), ErrNotSupported + return *new([]types.MinerState), ErrNotSupported } return s.Internal.StatesForMining(p0, p1) } -func (s *MinerAPIStub) StatesForMining(p0 context.Context, p1 []address.Address) ([]dtypes.MinerState, error) { - return *new([]dtypes.MinerState), ErrNotSupported +func (s *MinerAPIStub) StatesForMining(p0 context.Context, p1 []address.Address) ([]types.MinerState, error) { + return *new([]types.MinerState), ErrNotSupported } func (s *MinerAPIStruct) Stop(p0 context.Context, p1 []address.Address) error { @@ -216,15 +203,15 @@ func (s *MinerAPIStub) Stop(p0 context.Context, p1 []address.Address) error { return ErrNotSupported } -func (s *MinerAPIStruct) UpdateAddress(p0 context.Context, p1 int64, p2 int64) ([]dtypes.MinerInfo, error) { +func (s *MinerAPIStruct) UpdateAddress(p0 context.Context, p1 int64, p2 int64) ([]types.MinerInfo, error) { if s.Internal.UpdateAddress == nil { - return *new([]dtypes.MinerInfo), ErrNotSupported + return *new([]types.MinerInfo), ErrNotSupported } return s.Internal.UpdateAddress(p0, p1, p2) } -func (s *MinerAPIStub) UpdateAddress(p0 context.Context, p1 int64, p2 int64) ([]dtypes.MinerInfo, error) { - return *new([]dtypes.MinerInfo), ErrNotSupported +func (s *MinerAPIStub) UpdateAddress(p0 context.Context, p1 int64, p2 int64) ([]types.MinerInfo, error) { + return *new([]types.MinerInfo), ErrNotSupported } var _ Common = new(CommonStruct) diff --git a/build/params.go b/build/params.go index 33fef97..6c1d7f8 100644 --- a/build/params.go +++ b/build/params.go @@ -1,6 +1,3 @@ -//go:build !testground -// +build !testground - package build var ( diff --git a/cli/cmd.go b/cli/cmd.go index eb53f22..501d20b 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -6,7 +6,6 @@ import ( "net/http" "os" "os/signal" - "strings" "syscall" logging "github.com/ipfs/go-log/v2" @@ -17,11 +16,10 @@ import ( "github.com/filecoin-project/venus-miner/api" "github.com/filecoin-project/venus-miner/api/client" - cliutil "github.com/filecoin-project/venus-miner/cli/util" "github.com/filecoin-project/venus-miner/node/config" "github.com/filecoin-project/venus-miner/node/repo" - "github.com/filecoin-project/venus/venus-shared/api/chain/v1" + v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" ) var log = logging.Logger("cli") @@ -30,62 +28,20 @@ const ( metadataTraceContext = "traceContext" ) -// custom CLI error - -type ErrCmdFailed struct { - msg string -} - -func (e *ErrCmdFailed) Error() string { - return e.msg -} - -// The flag passed on the command line with the listen address of the API -// server (only used by the tests) -func flagForAPI(t repo.RepoType) string { - switch t { - case repo.Miner: - return "miner-api-url" - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - -func flagForRepo(t repo.RepoType) string { - switch t { - case repo.Miner: - return "miner-repo" - default: - panic(fmt.Sprintf("Unknown repo type: %v", t)) - } -} - -func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) { - // Check if there was a flag passed with the listen address of the API - // server (only used by the tests) - apiFlag := flagForAPI(t) - if ctx.IsSet(apiFlag) { - strma := ctx.String(apiFlag) - strma = strings.TrimSpace(strma) - - return cliutil.APIInfo{Addr: strma}, nil - } - - repoFlag := flagForRepo(t) - - p, err := homedir.Expand(ctx.String(repoFlag)) +func GetAPIInfo(ctx *cli.Context) (config.APIInfo, error) { + p, err := homedir.Expand(ctx.String("miner-repo")) if err != nil { - return cliutil.APIInfo{}, fmt.Errorf("could not expand home dir (%s): %w", repoFlag, err) + return config.APIInfo{}, fmt.Errorf("could not expand home dir: %w", err) } r, err := repo.NewFS(p) if err != nil { - return cliutil.APIInfo{}, fmt.Errorf("could not open repo at path: %s; %w", p, err) + return config.APIInfo{}, fmt.Errorf("could not open repo at path: %s; %w", p, err) } ma, err := r.APIEndpoint() if err != nil { - return cliutil.APIInfo{}, fmt.Errorf("could not get api endpoint: %w", err) + return config.APIInfo{}, fmt.Errorf("could not get api endpoint: %w", err) } token, err := r.APIToken() @@ -93,14 +49,14 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (cliutil.APIInfo, error) { log.Warnf("Couldn't load CLI token, capabilities may be limited: %v", err) } - return cliutil.APIInfo{ + return config.APIInfo{ Addr: ma.String(), - Token: token, + Token:string(token), }, nil } -func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http.Header, error) { - ainfo, err := GetAPIInfo(ctx, t) +func GetRawAPI(ctx *cli.Context, version string) (string, http.Header, error) { + ainfo, err := GetAPIInfo(ctx) if err != nil { return "", nil, fmt.Errorf("could not get API info: %w", err) } @@ -113,22 +69,22 @@ func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http. return addr, ainfo.AuthHeader(), nil } -func GetFullNodeAPI(ctx *cli.Context, fn config.FullNode, version string) (v1.FullNode, jsonrpc.ClientCloser, error) { - addr, err := fn.DialArgs(version) +func GetMinerAPI(ctx *cli.Context) (api.MinerAPI, jsonrpc.ClientCloser, error) { + addr, headers, err := GetRawAPI(ctx, "v0") if err != nil { - return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) + return nil, nil, err } - return v1.NewFullNodeRPC(ctx.Context, addr, fn.AuthHeader()) + return client.NewMinerRPC(ctx.Context, addr, headers) } -func GetMinerAPI(ctx *cli.Context) (api.MinerAPI, jsonrpc.ClientCloser, error) { - addr, headers, err := GetRawAPI(ctx, repo.Miner, "v0") +func GetFullNodeAPI(ctx *cli.Context, fn *config.APIInfo, version string) (v1.FullNode, jsonrpc.ClientCloser, error) { + addr, err := fn.DialArgs(version) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) } - return client.NewMinerRPC(ctx.Context, addr, headers) + return v1.NewFullNodeRPC(ctx.Context, addr, fn.AuthHeader()) } func DaemonContext(cctx *cli.Context) context.Context { diff --git a/cmd/address.go b/cmd/address.go index 0ffb98a..2956a03 100644 --- a/cmd/address.go +++ b/cmd/address.go @@ -8,7 +8,6 @@ import ( "github.com/filecoin-project/go-address" lcli "github.com/filecoin-project/venus-miner/cli" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" ) var addressCmd = &cli.Command{ @@ -20,7 +19,6 @@ var addressCmd = &cli.Command{ stateCmd, startMiningCmd, stopMiningCmd, - addCmd, }, } @@ -62,49 +60,6 @@ var updateCmd = &cli.Command{ }, } -var addCmd = &cli.Command{ - Name: "add", - Usage: "add a miner", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "miner", - Required: true, - }, - &cli.StringFlag{ - Name: "id", - Required: false, - }, - &cli.StringFlag{ - Name: "name", - Required: false, - }, - }, - Action: func(cctx *cli.Context) error { - mi := dtypes.MinerInfo{Id: cctx.String("id"), Name: cctx.String("name")} - - addr, err := address.NewFromString(cctx.String("miner")) - if err != nil { - return nil - } - mi.Addr = addr - - postApi, closer, err := lcli.GetMinerAPI(cctx) - if err != nil { - return err - } - defer closer() - - err = postApi.AddAddress(cctx.Context, mi) - if err != nil { - return err - } - - fmt.Println("add miner success.") - return nil - - }, -} - var listCmd = &cli.Command{ Name: "list", Usage: "print miners", diff --git a/cmd/init.go b/cmd/init.go index b2bcc16..ccd498c 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/venus/venus-shared/api" v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" - types2 "github.com/filecoin-project/venus/venus-shared/types" + "github.com/filecoin-project/venus/venus-shared/types" lcli "github.com/filecoin-project/venus-miner/cli" "github.com/filecoin-project/venus-miner/node/config" @@ -61,13 +61,13 @@ var initCmd = &cli.Command{ log.Info("Trying to connect to full node RPC") - fullnode := config.FullNode{} + fullnode := config.APIInfo{} if cctx.String("api") != "" && cctx.String("token") != "" { - fullnode.ListenAPI = cctx.String("api") + fullnode.Addr = cctx.String("api") fullnode.Token = cctx.String("token") } - fullNodeAPI, closer, err := lcli.GetFullNodeAPI(cctx, fullnode, "v1") + fullNodeAPI, closer, err := lcli.GetFullNodeAPI(cctx, &fullnode, "v1") if err != nil { return err } @@ -102,11 +102,11 @@ var initCmd = &cli.Command{ log.Info("Initializing repo") - if err := r.Init(repo.Miner); err != nil { + if err := r.Init(); err != nil { return err } - if err := storageMinerInit(cctx, r, fullnode); err != nil { + if err := storageMinerInit(cctx, r, &fullnode); err != nil { log.Errorf("Failed to initialize venus-miner: %+v", err) path, err := homedir.Expand(repoPath) if err != nil { @@ -125,8 +125,8 @@ var initCmd = &cli.Command{ }, } -func storageMinerInit(cctx *cli.Context, r repo.Repo, fn config.FullNode) error { - lr, err := r.Lock(repo.Miner) +func storageMinerInit(cctx *cli.Context, r repo.Repo, fn *config.APIInfo) error { + lr, err := r.Lock() if err != nil { return err } @@ -152,17 +152,14 @@ func storageMinerInit(cctx *cli.Context, r repo.Repo, fn config.FullNode) error } if cctx.String("auth-api") != "" { - cfg.Db.Type = "auth" - cfg.Db.Auth = &config.AuthConfig{ - ListenAPI: cctx.String("auth-api"), - Token: gt, + cfg.Auth = &config.APIInfo{ + Addr: cctx.String("auth-api"), + Token: gt, } } - cfg.Db.SFType = sfType - if cfg.Db.SFType == "mysql" { - cfg.Db.MySQL.Conn = cctx.String("mysql-conn") - } + cfg.SlashFilter.Type = sfType + cfg.SlashFilter.MySQL.Conn = cctx.String("mysql-conn") }); err != nil { return fmt.Errorf("modify config failed: %w", err) } @@ -210,10 +207,10 @@ func SyncWait(ctx context.Context, fullNode v1.FullNode, watch bool) error { working := -1 for i, ss := range state.ActiveSyncs { switch ss.Stage { - case types2.StageSyncComplete: + case types.StageSyncComplete: default: working = i - case types2.StageIdle: + case types.StageIdle: // not complete, not actively working } } diff --git a/cmd/main.go b/cmd/main.go index 3109994..293f0ce 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -9,16 +9,12 @@ import ( "github.com/filecoin-project/venus-miner/build" lcli "github.com/filecoin-project/venus-miner/cli" "github.com/filecoin-project/venus-miner/lib/logger" - "github.com/filecoin-project/venus-miner/node/repo" ) var log = logging.Logger("main") const FlagMinerRepo = "miner-repo" -// TODO remove after deprecation period -const FlagMinerRepoDeprecation = "storagerepo" - func main() { logger.SetupLogLevels() @@ -42,17 +38,15 @@ func main() { }, &cli.StringFlag{ Name: FlagMinerRepo, - Aliases: []string{FlagMinerRepoDeprecation}, EnvVars: []string{"VENUS_MINER_PATH"}, Value: "~/.venusminer", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(VENUS_MINER_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), + Usage: fmt.Sprintf("Specify miner repo path, env VENUS_MINER_PATH"), }, }, Commands: append(local, lcli.CommonCommands...), } app.Setup() - app.Metadata["repoType"] = repo.Miner lcli.RunApp(app) } diff --git a/cmd/run.go b/cmd/run.go index eea0f7f..c466753 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -26,8 +26,8 @@ import ( "github.com/filecoin-project/venus-miner/lib/tracing" "github.com/filecoin-project/venus-miner/node" "github.com/filecoin-project/venus-miner/node/config" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" "github.com/filecoin-project/venus-miner/node/repo" + "github.com/filecoin-project/venus-miner/types" "github.com/filecoin-project/venus/pkg/constants" "github.com/filecoin-project/venus/venus-shared/api" @@ -50,7 +50,6 @@ var runCmd = &cli.Command{ Name: "api-max-req-size", Usage: "maximum API request size accepted by the JSON RPC server", }, - node.CLIFLAGBlockRecord, }, Action: func(cctx *cli.Context) error { log.Info("Initializing build params") @@ -71,7 +70,7 @@ var runCmd = &cli.Command{ return fmt.Errorf("repo at '%s' is not initialized, run 'venus-miner init' to set it up", minerRepoPath) } - lr, err := r.Lock(repo.Miner) + lr, err := r.Lock() if err != nil { return err } @@ -118,12 +117,11 @@ var runCmd = &cli.Command{ var minerAPI lapi.MinerAPI stop, err := node.New(ctx, node.MinerAPI(&minerAPI), - node.Override(new(dtypes.ShutdownChan), shutdownChan), - node.Online(), node.Repo(cctx, r), + node.Override(new(types.ShutdownChan), shutdownChan), node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("miner-api") }, - node.Override(new(dtypes.APIEndpoint), func() (dtypes.APIEndpoint, error) { + node.Override(new(types.APIEndpoint), func() (types.APIEndpoint, error) { return multiaddr.NewMultiaddr("/ip4/0.0.0.0/tcp/" + cctx.String("miner-api")) })), node.Override(new(v1.FullNode), nodeApi), diff --git a/lib/tracing/setup.go b/lib/tracing/setup.go index e648786..09f0336 100644 --- a/lib/tracing/setup.go +++ b/lib/tracing/setup.go @@ -1,10 +1,11 @@ package tracing import ( + "os" + "contrib.go.opencensus.io/exporter/jaeger" logging "github.com/ipfs/go-log/v2" "go.opencensus.io/trace" - "os" "github.com/filecoin-project/venus-miner/node/config" ) diff --git a/miner/api.go b/miner/api.go index 45a9f08..4821e30 100644 --- a/miner/api.go +++ b/miner/api.go @@ -2,12 +2,11 @@ package miner import ( "context" + "github.com/filecoin-project/venus-miner/types" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/venus-miner/node/modules/dtypes" ) type MiningAPI interface { @@ -23,11 +22,10 @@ type IMinerMining interface { } type IMinerManager interface { - UpdateAddress(context.Context, int64, int64) ([]dtypes.MinerInfo, error) - ListAddress(context.Context) ([]dtypes.MinerInfo, error) - StatesForMining(context.Context, []address.Address) ([]dtypes.MinerState, error) - CountWinners(context.Context, []address.Address, abi.ChainEpoch, abi.ChainEpoch) ([]dtypes.CountWinners, error) - AddAddress(ctx context.Context, mi dtypes.MinerInfo) error + UpdateAddress(context.Context, int64, int64) ([]types.MinerInfo, error) + ListAddress(context.Context) ([]types.MinerInfo, error) + StatesForMining(context.Context, []address.Address) ([]types.MinerState, error) + CountWinners(context.Context, []address.Address, abi.ChainEpoch, abi.ChainEpoch) ([]types.CountWinners, error) } type MockMinerMgr struct { @@ -35,22 +33,18 @@ type MockMinerMgr struct { var _ IMinerManager = &MockMinerMgr{} -func (m MockMinerMgr) UpdateAddress(context.Context, int64, int64) ([]dtypes.MinerInfo, error) { +func (m MockMinerMgr) UpdateAddress(context.Context, int64, int64) ([]types.MinerInfo, error) { return nil, nil } -func (m MockMinerMgr) ListAddress(context.Context) ([]dtypes.MinerInfo, error) { +func (m MockMinerMgr) ListAddress(context.Context) ([]types.MinerInfo, error) { return nil, nil } -func (m MockMinerMgr) StatesForMining(context.Context, []address.Address) ([]dtypes.MinerState, error) { +func (m MockMinerMgr) StatesForMining(context.Context, []address.Address) ([]types.MinerState, error) { return nil, nil } -func (m MockMinerMgr) CountWinners(context.Context, []address.Address, abi.ChainEpoch, abi.ChainEpoch) ([]dtypes.CountWinners, error) { +func (m MockMinerMgr) CountWinners(context.Context, []address.Address, abi.ChainEpoch, abi.ChainEpoch) ([]types.CountWinners, error) { return nil, nil } - -func (m MockMinerMgr) AddAddress(context.Context, dtypes.MinerInfo) error { - return nil -} diff --git a/miner/minerwpp.go b/miner/minerwpp.go index 6da890e..3b88df9 100644 --- a/miner/minerwpp.go +++ b/miner/minerwpp.go @@ -3,6 +3,7 @@ package miner import ( "context" "fmt" + types2 "github.com/filecoin-project/venus-miner/types" "time" "github.com/filecoin-project/go-address" @@ -12,8 +13,6 @@ import ( "github.com/filecoin-project/venus-miner/api/client" "github.com/filecoin-project/venus-miner/build" "github.com/filecoin-project/venus-miner/node/config" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" - "github.com/filecoin-project/venus/pkg/constants" "github.com/filecoin-project/venus/pkg/util/ffiwrapper" "github.com/filecoin-project/venus/venus-shared/actors/builtin" @@ -22,7 +21,7 @@ import ( ) type MiningWpp struct { - minerInfo dtypes.MinerInfo + minerInfo types2.MinerInfo gatewayNode *config.GatewayNode verifier ffiwrapper.Verifier @@ -30,7 +29,7 @@ type MiningWpp struct { winnRpt abi.RegisteredPoStProof } -func NewWinningPoStProver(api v1.FullNode, gatewayNode *config.GatewayNode, minerInfo dtypes.MinerInfo, verifier ffiwrapper.Verifier) (*MiningWpp, error) { +func NewWinningPoStProver(api v1.FullNode, gatewayNode *config.GatewayNode, minerInfo types2.MinerInfo, verifier ffiwrapper.Verifier) (*MiningWpp, error) { mi, err := api.StateMinerInfo(context.TODO(), minerInfo.Addr, types.EmptyTSK) if err != nil { return nil, fmt.Errorf("getting sector size: %w", err) diff --git a/miner/multiminer.go b/miner/multiminer.go index 71e053e..5030f98 100644 --- a/miner/multiminer.go +++ b/miner/multiminer.go @@ -6,10 +6,12 @@ import ( "crypto/rand" "encoding/binary" "fmt" - types3 "github.com/filecoin-project/venus-miner/types" + "github.com/filecoin-project/venus-miner/node/modules/miner-manager" "sync" "time" + types3 "github.com/filecoin-project/venus-miner/types" + logging "github.com/ipfs/go-log/v2" "github.com/pkg/errors" "go.opencensus.io/trace" @@ -23,11 +25,8 @@ import ( "github.com/filecoin-project/venus-miner/api/client" "github.com/filecoin-project/venus-miner/build" "github.com/filecoin-project/venus-miner/lib/journal" - "github.com/filecoin-project/venus-miner/miner/slashfilter" "github.com/filecoin-project/venus-miner/node/config" - "github.com/filecoin-project/venus-miner/node/modules/block_recorder" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" - "github.com/filecoin-project/venus-miner/node/modules/minermanage" + "github.com/filecoin-project/venus-miner/node/modules/slashfilter" "github.com/filecoin-project/venus/pkg/chain" "github.com/filecoin-project/venus/pkg/constants" @@ -64,8 +63,8 @@ func randTimeOffset(width time.Duration) time.Duration { // NewMiner instantiates a miner with a concrete WinningPoStProver and a miner // address (which can be different from the worker's address). -func NewMiner(api v1api.FullNode, gtNode *config.GatewayNode, verifier ffiwrapper.Verifier, minerManager minermanage.MinerManageAPI, - sf slashfilter.SlashFilterAPI, j journal.Journal, blockRecord block_recorder.IBlockRecord) *Miner { +func NewMiner(api v1api.FullNode, gtNode *config.GatewayNode, verifier ffiwrapper.Verifier, minerManager miner_manager.MinerManageAPI, + sf slashfilter.SlashFilterAPI, j journal.Journal) *Miner { networkParams, err := api.StateGetNetworkParams(context.TODO()) if err != nil { return nil @@ -96,8 +95,7 @@ func NewMiner(api v1api.FullNode, gtNode *config.GatewayNode, verifier ffiwrappe return func(bool, abi.ChainEpoch, error) {}, 0, nil }, - sf: sf, - blockRecord: blockRecord, + sf: sf, evtTypes: [...]journal.EventType{ evtTypeBlockMined: j.RegisterEventType("miner", "block_mined"), @@ -144,8 +142,7 @@ type Miner struct { lastWork *MiningBase - sf slashfilter.SlashFilterAPI - blockRecord block_recorder.IBlockRecord + sf slashfilter.SlashFilterAPI evtTypes [1]journal.EventType journal journal.Journal @@ -154,7 +151,7 @@ type Miner struct { lkWPP sync.Mutex minerWPPMap map[address.Address]*minerWPP - minerManager minermanage.MinerManageAPI + minerManager miner_manager.MinerManageAPI verifier ffiwrapper.Verifier @@ -481,24 +478,26 @@ minerLoop: // broadcast all blocks for _, b := range blks { go func(bm *types2.BlockMsg) { - if err := m.sf.MinedBlock(ctx, bm.Header, base.TipSet.Height()+base.NullRounds); err != nil { + if exists, err := m.sf.HasBlock(ctx, bm.Header); err != nil { log.Errorf(" SLASH FILTER ERROR: %s", err) return - } - - has := m.blockRecord.Has(bm.Header.Miner, uint64(bm.Header.Height)) - if has { - log.Warnw("Created a block at the same height as another block we've created", "height", bm.Header.Height, "miner", bm.Header.Miner, "parents", bm.Header.Parents) + } else if exists { + log.Error(" SLASH FILTER ERROR: double-fork mining faults") return } - err = m.blockRecord.MarkAsProduced(bm.Header.Miner, uint64(bm.Header.Height)) - if err != nil { - log.Errorf("failed to write db: %s", err) + if err := m.sf.MinedBlock(ctx, bm.Header, base.TipSet.Height()+base.NullRounds); err != nil { + log.Errorf(" SLASH FILTER ERROR: %s", err) + return } if err := m.api.SyncSubmitBlock(ctx, bm); err != nil { log.Errorf("failed to submit newly mined block: %s", err) + return + } + + if err := m.sf.PutBlock(ctx, bm.Header, base.TipSet.Height()+base.NullRounds); err != nil { + log.Errorf(" SLASH FILTER ERROR: %s", err) } }(b) } @@ -884,7 +883,7 @@ func (m *Miner) ManualStop(ctx context.Context, addrs []address.Address) error { return nil } -func (m *Miner) UpdateAddress(ctx context.Context, skip, limit int64) ([]dtypes.MinerInfo, error) { +func (m *Miner) UpdateAddress(ctx context.Context, skip, limit int64) ([]types3.MinerInfo, error) { miners, err := m.minerManager.Update(ctx, skip, limit) if err != nil { return nil, err @@ -907,38 +906,27 @@ func (m *Miner) UpdateAddress(ctx context.Context, skip, limit int64) ([]dtypes. return miners, nil } -func (m *Miner) AddAddress(ctx context.Context, mi dtypes.MinerInfo) error { - epp, err := NewWinningPoStProver(m.api, m.gatewayNode, mi, m.verifier) - if err != nil { - return err - } - - m.minerWPPMap[mi.Addr] = &minerWPP{epp: epp, account: mi.Name, isMining: true} - - return m.minerManager.Put(ctx, mi) -} - -func (m *Miner) ListAddress(ctx context.Context) ([]dtypes.MinerInfo, error) { +func (m *Miner) ListAddress(ctx context.Context) ([]types3.MinerInfo, error) { m.lkWPP.Lock() defer m.lkWPP.Unlock() return m.minerManager.List(ctx) } -func (m *Miner) StatesForMining(ctx context.Context, addrs []address.Address) ([]dtypes.MinerState, error) { +func (m *Miner) StatesForMining(ctx context.Context, addrs []address.Address) ([]types3.MinerState, error) { m.lkWPP.Lock() defer m.lkWPP.Unlock() - res := make([]dtypes.MinerState, 0) + res := make([]types3.MinerState, 0) if len(addrs) > 0 { for _, addr := range addrs { if val, ok := m.minerWPPMap[addr]; ok { - res = append(res, dtypes.MinerState{Addr: addr, IsMining: val.isMining, Err: val.err}) + res = append(res, types3.MinerState{Addr: addr, IsMining: val.isMining, Err: val.err}) } } } else { for k, v := range m.minerWPPMap { - res = append(res, dtypes.MinerState{Addr: k, IsMining: v.isMining, Err: v.err}) + res = append(res, types3.MinerState{Addr: k, IsMining: v.isMining, Err: v.err}) } } @@ -972,20 +960,20 @@ func (m *Miner) winCountInRound(ctx context.Context, account string, mAddr addre return IsRoundWinner(ctx, ts.Height()+1, account, mAddr, rbase, mbi, api) } -func (m *Miner) CountWinners(ctx context.Context, addrs []address.Address, start abi.ChainEpoch, end abi.ChainEpoch) ([]dtypes.CountWinners, error) { +func (m *Miner) CountWinners(ctx context.Context, addrs []address.Address, start abi.ChainEpoch, end abi.ChainEpoch) ([]types3.CountWinners, error) { log.Infof("count winners, addrs: %v, start: %v, end: %v", addrs, start, end) ts, err := m.api.ChainHead(ctx) if err != nil { log.Error("get chain head", err) - return []dtypes.CountWinners{}, err + return []types3.CountWinners{}, err } if start > ts.Height() || end > ts.Height() { - return []dtypes.CountWinners{}, fmt.Errorf("start or end greater than cur tipset height: %v", ts.Height()) + return []types3.CountWinners{}, fmt.Errorf("start or end greater than cur tipset height: %v", ts.Height()) } - res := make([]dtypes.CountWinners, 0) + res := make([]types3.CountWinners, 0) wg := sync.WaitGroup{} mAddrs := make([]address.Address, 0) @@ -995,7 +983,7 @@ func (m *Miner) CountWinners(ctx context.Context, addrs []address.Address, start if _, ok := m.minerWPPMap[addr]; ok { mAddrs = append(mAddrs, addr) } else { - res = append(res, dtypes.CountWinners{Msg: "miner not exist", Miner: addr}) + res = append(res, types3.CountWinners{Msg: "miner not exist", Miner: addr}) } } } else { @@ -1013,7 +1001,7 @@ func (m *Miner) CountWinners(ctx context.Context, addrs []address.Address, start go func() { defer wg.Done() - winInfo := make([]dtypes.SimpleWinInfo, 0) + winInfo := make([]types3.SimpleWinInfo, 0) totalWinCount := int64(0) var sign SignFunc = nil @@ -1023,13 +1011,13 @@ func (m *Miner) CountWinners(ctx context.Context, addrs []address.Address, start walletAPI, closer, err := client.NewGatewayRPC(ctx, m.gatewayNode) if err != nil { log.Errorf("[%v] create wallet RPC failed: %w", tAddr, err) - res = append(res, dtypes.CountWinners{Msg: err.Error(), Miner: tAddr}) + res = append(res, types3.CountWinners{Msg: err.Error(), Miner: tAddr}) return } defer closer() sign = walletAPI.WalletSign } else { - res = append(res, dtypes.CountWinners{Msg: "miner not exist", Miner: tAddr}) + res = append(res, types3.CountWinners{Msg: "miner not exist", Miner: tAddr}) return } @@ -1048,12 +1036,12 @@ func (m *Miner) CountWinners(ctx context.Context, addrs []address.Address, start if winner != nil { totalWinCount += winner.WinCount - winInfo = append(winInfo, dtypes.SimpleWinInfo{Epoch: epoch + 1, WinCount: winner.WinCount}) + winInfo = append(winInfo, types3.SimpleWinInfo{Epoch: epoch + 1, WinCount: winner.WinCount}) } }(epoch) } wgWin.Wait() - res = append(res, dtypes.CountWinners{Miner: tAddr, TotalWinCount: totalWinCount, WinEpochList: winInfo}) + res = append(res, types3.CountWinners{Miner: tAddr, TotalWinCount: totalWinCount, WinEpochList: winInfo}) }() } wg.Wait() diff --git a/node/builder.go b/node/builder.go index 475bb66..da3f280 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,32 +3,25 @@ package node import ( "context" "encoding/json" - "errors" "fmt" - types2 "github.com/filecoin-project/venus-miner/types" + "github.com/filecoin-project/venus-miner/node/modules/miner-manager" logging "github.com/ipfs/go-log/v2" metricsi "github.com/ipfs/go-metrics-interface" - "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" "go.uber.org/fx" "github.com/filecoin-project/venus-miner/api" "github.com/filecoin-project/venus-miner/lib/journal" "github.com/filecoin-project/venus-miner/miner" - "github.com/filecoin-project/venus-miner/miner/slashfilter" "github.com/filecoin-project/venus-miner/node/config" "github.com/filecoin-project/venus-miner/node/impl" "github.com/filecoin-project/venus-miner/node/impl/common" "github.com/filecoin-project/venus-miner/node/modules" - "github.com/filecoin-project/venus-miner/node/modules/block_recorder" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" "github.com/filecoin-project/venus-miner/node/modules/helpers" - "github.com/filecoin-project/venus-miner/node/modules/minermanage" - "github.com/filecoin-project/venus-miner/node/modules/minermanage/auth" - "github.com/filecoin-project/venus-miner/node/modules/minermanage/local" - "github.com/filecoin-project/venus-miner/node/modules/minermanage/mysql" + "github.com/filecoin-project/venus-miner/node/modules/slashfilter" "github.com/filecoin-project/venus-miner/node/repo" + "github.com/filecoin-project/venus-miner/types" "github.com/filecoin-project/venus/pkg/util/ffiwrapper" fwpimpl "github.com/filecoin-project/venus/pkg/util/ffiwrapper/impl" @@ -50,9 +43,6 @@ const ( // the system starts, so that it's available for all other components. InitJournalKey = invoke(iota) - // System processes. - InitMemoryWatchdog - // daemon ExtractApiKey @@ -72,11 +62,6 @@ type Settings struct { // invokes are separate from modules as they can't be referenced by return // type, and must be applied in correct order invokes []fx.Option - - nodeType repo.RepoType - - Online bool // Online option applied - Config bool // Config option applied } func defaults() []Option { @@ -89,46 +74,13 @@ func defaults() []Option { return metricsi.CtxScope(context.Background(), "venus-miner") }), - Override(new(dtypes.ShutdownChan), make(chan struct{})), + Override(new(types.ShutdownChan), make(chan struct{})), } } -func isType(t repo.RepoType) func(s *Settings) bool { - return func(s *Settings) bool { return s.nodeType == t } -} - -// Online sets up basic libp2p node -func Online() Option { - - return Options( - // make sure that online is applied before Config. - // This is important because Config overrides some of Online units - func(s *Settings) error { s.Online = true; return nil }, - ApplyIf(func(s *Settings) bool { return s.Config }, - Error(errors.New("the Online option must be set before Config option")), - ), - - // miner - ApplyIf(isType(repo.Miner)), - ) -} - -// Config sets up constructors based on the provided Config -func ConfigCommon(cfg *config.Common) Option { - return Options( - func(s *Settings) error { s.Config = true; return nil }, - Override(new(dtypes.APIEndpoint), func() (dtypes.APIEndpoint, error) { - return multiaddr.NewMultiaddr(cfg.API.ListenAddress) - }), - Override(SetApiEndpointKey, func(lr repo.LockedRepo, e dtypes.APIEndpoint) error { - return lr.SetAPIEndpoint(e) - }), - ) -} - func Repo(cctx *cli.Context, r repo.Repo) Option { return func(settings *Settings) error { - lr, err := r.Lock(settings.nodeType) + lr, err := r.Lock() if err != nil { return err } @@ -140,104 +92,68 @@ func Repo(cctx *cli.Context, r repo.Repo) Option { return Options( Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing - Override(new(dtypes.MetadataDS), modules.Datastore), + Override(new(types.MetadataDS), modules.Datastore), - Override(new(types2.KeyStore), modules.KeyStore), + Override(new(types.KeyStore), modules.KeyStore), - Override(new(*dtypes.APIAlg), modules.APISecret), + Override(new(*types.APIAlg), modules.APISecret), - ApplyIf(isType(repo.Miner), ConfigPostOptions(cctx, c)), + ConfigMinerOptions(cctx, c), )(settings) } } -var ( - CLIFLAGBlockRecord = &cli.StringFlag{ - Name: "block_record", - Usage: "the way to record the blocks that have been generated, optional: local, cache", - EnvVars: []string{"BLOCK_RECORD_WAY"}, - Value: "", - } -) +func newSlashFilterAPI(sfCfg *config.SlashFilterConfig) Option { + return func(settings *Settings) error { + switch slashfilter.BlockStoreType(sfCfg.Type) { + case slashfilter.Local: + Override(new(slashfilter.SlashFilterAPI), slashfilter.NewLocal) + case slashfilter.MySQL: + Override(new(slashfilter.SlashFilterAPI), slashfilter.NewMysql(&sfCfg.MySQL)) + default: -func ConfigPostConfig(cctx *cli.Context, cfg *config.MinerConfig) (*config.MinerConfig, error) { - if cctx.IsSet(CLIFLAGBlockRecord.Name) { - cfg.BlockRecord = cctx.String(CLIFLAGBlockRecord.Name) - } + } - configStr, _ := json.MarshalIndent(cfg, "", "\t") - log.Warnf("final config: \n%v", string(configStr)) - return cfg, nil + return fmt.Errorf("unsupported block store type [%s]", sfCfg.Type) + } } -func ConfigPostOptions(cctx *cli.Context, c interface{}) Option { - postCfg, ok := c.(*config.MinerConfig) +func ConfigMinerOptions(cctx *cli.Context, c interface{}) Option { + cfg, ok := c.(*config.MinerConfig) if !ok { return Error(fmt.Errorf("invalid config from repo, got: %T", c)) } - scfg, err := ConfigPostConfig(cctx, postCfg) - if err != nil { - return Error(fmt.Errorf("error to parse config and flag %v", err)) - } + configStr, _ := json.MarshalIndent(cfg, "", "\t") + log.Infof("final config: \n%v", string(configStr)) + shareOps := Options( - Override(new(*config.MinerConfig), scfg), + Override(new(*config.MinerConfig), cfg), Override(new(api.Common), From(new(common.CommonAPI))), Override(new(ffiwrapper.Verifier), fwpimpl.ProofVerifier), ) - opt, err := PostWinningOptions(scfg) - if err != nil { - return Error(fmt.Errorf("error to constructor poster %v", err)) - } - - return Options( - ConfigCommon(&postCfg.Common), - shareOps, - opt, - ) -} - -func PostWinningOptions(postCfg *config.MinerConfig) (Option, error) { - blockRecordOp, err := newBlockRecord(postCfg.BlockRecord) - if err != nil { - return nil, err - } + minerOps := Options( + newSlashFilterAPI(cfg.SlashFilter), - minerManageAPIOp, err := newMinerManageAPI(postCfg.Db) - if err != nil { - return nil, err - } + Override(new(*config.GatewayNode), cfg.Gateway), - slashFilterAPIOp, err := newSlashFilterAPI(postCfg.Db) - if err != nil { - return nil, err - } + Override(new(miner_manager.MinerManageAPI), miner_manager.NewMinerManager(cfg.Auth.Addr, cfg.Auth.Token)), + Override(new(miner.MiningAPI), modules.NewMinerProcessor), + ) return Options( - blockRecordOp, - minerManageAPIOp, - slashFilterAPIOp, - Override(new(*config.GatewayNode), postCfg.Gateway), - Override(new(miner.MiningAPI), modules.NewWiningPoster), - ), nil + shareOps, + minerOps, + Override(SetApiEndpointKey, func(lr repo.LockedRepo, e types.APIEndpoint) error { + return lr.SetAPIEndpoint(e) + }), + ) } func MinerAPI(out *api.MinerAPI) Option { return Options( - ApplyIf(func(s *Settings) bool { return s.Config }, - Error(errors.New("the Poster option must be set before Config option")), - ), - ApplyIf(func(s *Settings) bool { return s.Online }, - Error(errors.New("the Poster option must be set before Online option")), - ), - - func(s *Settings) error { - s.nodeType = repo.Miner - return nil - }, - func(s *Settings) error { resAPI := &impl.MinerAPI{} s.invokes[ExtractApiKey] = fx.Populate(resAPI) @@ -247,44 +163,6 @@ func MinerAPI(out *api.MinerAPI) Option { ) } -func newMinerManageAPI(dbConfig *config.MinerDbConfig) (Option, error) { - switch dbConfig.Type { - case minermanage.Local: - return Override(new(minermanage.MinerManageAPI), local.NewMinerManger), nil - case minermanage.MySQL: - return Override(new(minermanage.MinerManageAPI), mysql.NewMinerManger(&dbConfig.MySQL)), nil - case minermanage.Auth: - return Override(new(minermanage.MinerManageAPI), auth.NewMinerManager(dbConfig.Auth.ListenAPI, dbConfig.Auth.Token)), nil - default: - - } - - return nil, fmt.Errorf("unsupported db type [%s]", dbConfig.Type) -} - -func newSlashFilterAPI(dbConfig *config.MinerDbConfig) (Option, error) { - switch dbConfig.SFType { - case minermanage.Local: - return Override(new(slashfilter.SlashFilterAPI), slashfilter.NewLocal), nil - case minermanage.MySQL: - return Override(new(slashfilter.SlashFilterAPI), slashfilter.NewMysqlSlashFilter(&dbConfig.MySQL)), nil - default: - - } - - return nil, fmt.Errorf("unsupported db type [%s]", dbConfig.SFType) -} - -func newBlockRecord(t string) (Option, error) { - if t == block_recorder.Local { - return Override(new(block_recorder.IBlockRecord), block_recorder.NewLocalDBRecord), nil - } else if t == block_recorder.Cache { - return Override(new(block_recorder.IBlockRecord), block_recorder.NewCacheRecord), nil - } else { - return nil, fmt.Errorf("unsupport block record type") - } -} - type FullOption = Option type StopFunc func(context.Context) error diff --git a/cli/util/apiinfo.go b/node/config/apiinfo.go similarity index 66% rename from cli/util/apiinfo.go rename to node/config/apiinfo.go index df6c5ff..ea267d4 100644 --- a/cli/util/apiinfo.go +++ b/node/config/apiinfo.go @@ -1,38 +1,22 @@ -package cliutil +package config import ( "net/http" "net/url" - "regexp" - "strings" - logging "github.com/ipfs/go-log/v2" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) -var log = logging.Logger("cliutil") - -var ( - infoWithToken = regexp.MustCompile(`^[a-zA-Z0-9\\-_]+?\\.[a-zA-Z0-9\\-_]+?\\.([a-zA-Z0-9\\-_]+)?:.+$`) -) - type APIInfo struct { Addr string - Token []byte + Token string } -func ParseApiInfo(s string) APIInfo { - var tok []byte - if infoWithToken.Match([]byte(s)) { - sp := strings.SplitN(s, ":", 2) - tok = []byte(sp[0]) - s = sp[1] - } - - return APIInfo{ - Addr: s, - Token: tok, +func defaultAPIInfo() *APIInfo { + return &APIInfo{ + Addr: "/ip4/0.0.0.0/tcp/12308/http", + Token: "", } } @@ -78,6 +62,6 @@ func (a APIInfo) AuthHeader() http.Header { headers.Add("Authorization", "Bearer "+string(a.Token)) return headers } - log.Warn("API Token not set and requested, capabilities might be limited.") + log.Warn("Sealer API Token not set and requested, capabilities might be limited.") return nil } diff --git a/node/config/authdef.go b/node/config/authdef.go deleted file mode 100644 index c6581eb..0000000 --- a/node/config/authdef.go +++ /dev/null @@ -1,13 +0,0 @@ -package config - -type AuthConfig struct { - ListenAPI string // http://[ip]:[port] - Token string -} - -func newDefaultAuthConfig() *AuthConfig { - return &AuthConfig{ - ListenAPI: "", - Token: "", - } -} diff --git a/node/config/def.go b/node/config/def.go index a8656a9..d5fb8d5 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -1,70 +1,13 @@ package config import ( - "encoding" - "net/http" - "net/url" "time" logging "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" ) var log = logging.Logger("config") -// Common is common config between full node and miner -type Common struct { - API API -} - -// API contains configs for API endpoint -type API struct { - ListenAddress string - RemoteListenAddress string - Timeout Duration -} - -type FullNode struct { - ListenAPI string - Token string -} - -func (sn FullNode) DialArgs(version string) (string, error) { - ma, err := multiaddr.NewMultiaddr(sn.ListenAPI) - if err == nil { - _, addr, err := manet.DialArgs(ma) - if err != nil { - return "", err - } - - return "ws://" + addr + "/rpc/" + version, nil - } - - _, err = url.Parse(sn.ListenAPI) - if err != nil { - return "", err - } - return sn.ListenAPI + "/rpc/" + version, nil -} - -func (sn FullNode) AuthHeader() http.Header { - if len(sn.Token) != 0 { - headers := http.Header{} - headers.Add("Authorization", "Bearer "+string(sn.Token)) - return headers - } - log.Warn("Sealer API Token not set and requested, capabilities might be limited.") - return nil -} - -func defFullNode() FullNode { - return FullNode{ - ListenAPI: "/ip4/127.0.0.1/tcp/1234/http", - Token: "", - } -} - type MySQLConfig struct { Conn string `json:"conn"` MaxOpenConn int `json:"maxOpenConn"` // 100 @@ -73,17 +16,19 @@ type MySQLConfig struct { Debug bool `json:"debug"` } -type MinerDbConfig struct { - Type string `json:"type"` - SFType string `json:"sfType"` - MySQL MySQLConfig `json:"mysql"` - Auth *AuthConfig `json:"auth"` +type BlockRecordConfig struct { + Type string `json:"type"` + MySQL MySQLConfig `json:"mysql"` } -func newDefaultMinerDbConfig() *MinerDbConfig { - return &MinerDbConfig{ - Type: "auth", - SFType: "mysql", +type SlashFilterConfig struct { + Type string `json:"type"` + MySQL MySQLConfig `json:"mysql"` +} + +func newSlashFilterConfig() *SlashFilterConfig { + return &SlashFilterConfig{ + Type: "local", MySQL: MySQLConfig{ Conn: "", MaxOpenConn: 100, @@ -91,7 +36,6 @@ func newDefaultMinerDbConfig() *MinerDbConfig { ConnMaxLifeTime: 60, Debug: false, }, - Auth: newDefaultAuthConfig(), } } @@ -118,61 +62,23 @@ func newDefaultTraceConfig() *TraceConfig { } type MinerConfig struct { - Common - - FullNode - - BlockRecord string - - Gateway *GatewayNode `json:"gateway"` + FullNode *APIInfo `json:"fullnode"` + Gateway *GatewayNode `json:"gateway"` + Auth *APIInfo `json:"auth"` - Db *MinerDbConfig `json:"db"` + SlashFilter *SlashFilterConfig Tracing *TraceConfig `json:"tracing"` } -func defCommon() Common { - return Common{ - API: API{ - ListenAddress: "/ip4/127.0.0.1/tcp/1234/http", - Timeout: Duration(30 * time.Second), - }, - } -} - func DefaultMinerConfig() *MinerConfig { minerCfg := &MinerConfig{ - Common: defCommon(), - FullNode: defFullNode(), - BlockRecord: "cache", + FullNode: defaultAPIInfo(), Gateway: newDefaultGatewayNode(), - Db: newDefaultMinerDbConfig(), + Auth: defaultAPIInfo(), + SlashFilter: newSlashFilterConfig(), Tracing: newDefaultTraceConfig(), } - minerCfg.Common.API.ListenAddress = "/ip4/0.0.0.0/tcp/12308/http" //change default address - return minerCfg } - -var _ encoding.TextMarshaler = (*Duration)(nil) -var _ encoding.TextUnmarshaler = (*Duration)(nil) - -// Duration is a wrapper type for time.Duration -// for decoding and encoding from/to TOML -type Duration time.Duration - -// UnmarshalText implements interface for TOML decoding -func (dur *Duration) UnmarshalText(text []byte) error { - d, err := time.ParseDuration(string(text)) - if err != nil { - return err - } - *dur = Duration(d) - return err -} - -func (dur Duration) MarshalText() ([]byte, error) { - d := time.Duration(dur) - return []byte(d.String()), nil -} diff --git a/node/config/load.go b/node/config/load.go index 67ce1bf..54aa1e6 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -7,7 +7,6 @@ import ( "os" "github.com/BurntSushi/toml" - "github.com/kelseyhightower/envconfig" ) // FromFile loads config from a specified file overriding defaults specified in @@ -33,11 +32,6 @@ func FromReader(reader io.Reader, def interface{}) (interface{}, error) { return nil, err } - err = envconfig.Process("LOTUS", cfg) - if err != nil { - return nil, fmt.Errorf("processing env vars overrides: %s", err) - } - return cfg, nil } diff --git a/node/config/load_test.go b/node/config/load_test.go index 73f9533..a50dbe9 100644 --- a/node/config/load_test.go +++ b/node/config/load_test.go @@ -2,12 +2,10 @@ package config import ( "bytes" + "github.com/stretchr/testify/assert" "io/ioutil" "os" "testing" - "time" - - "github.com/stretchr/testify/assert" ) func TestDecodeNothing(t *testing.T) { @@ -31,11 +29,9 @@ func TestDecodeNothing(t *testing.T) { func TestParitalConfig(t *testing.T) { assert := assert.New(t) cfgString := ` - [API] - Timeout = "10s" + [FullNode] ` expected := DefaultMinerConfig() - expected.API.Timeout = Duration(10 * time.Second) { cfg, err := FromReader(bytes.NewReader([]byte(cfgString)), DefaultMinerConfig()) diff --git a/node/impl/common/common.go b/node/impl/common/common.go index 8638d77..9c77356 100644 --- a/node/impl/common/common.go +++ b/node/impl/common/common.go @@ -3,6 +3,7 @@ package common import ( "context" "fmt" + "github.com/filecoin-project/venus-miner/types" "github.com/gbrlsnchs/jwt/v3" "github.com/google/uuid" @@ -13,8 +14,6 @@ import ( "github.com/filecoin-project/venus-miner/api" "github.com/filecoin-project/venus-miner/build" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" - vapi "github.com/filecoin-project/venus/venus-shared/api" ) @@ -23,8 +22,8 @@ var session = uuid.New() type CommonAPI struct { fx.In - APISecret *dtypes.APIAlg - ShutdownChan dtypes.ShutdownChan + APISecret *types.APIAlg + ShutdownChan types.ShutdownChan } type jwtPayload struct { diff --git a/node/impl/miner.go b/node/impl/miner.go index 4ec5bb0..caa28e9 100644 --- a/node/impl/miner.go +++ b/node/impl/miner.go @@ -8,30 +8,28 @@ import ( "github.com/filecoin-project/venus-miner/miner" "github.com/filecoin-project/venus-miner/node/impl/common" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" - "github.com/filecoin-project/venus-miner/node/modules/minermanage" + "github.com/filecoin-project/venus-miner/types" ) type MinerAPI struct { common.CommonAPI - MinerManager minermanage.MinerManageAPI miner.MiningAPI } -func (m *MinerAPI) UpdateAddress(ctx context.Context, skip int64, limit int64) ([]dtypes.MinerInfo, error) { +func (m *MinerAPI) UpdateAddress(ctx context.Context, skip int64, limit int64) ([]types.MinerInfo, error) { return m.MiningAPI.UpdateAddress(ctx, skip, limit) } -func (m *MinerAPI) ListAddress(ctx context.Context) ([]dtypes.MinerInfo, error) { +func (m *MinerAPI) ListAddress(ctx context.Context) ([]types.MinerInfo, error) { return m.MiningAPI.ListAddress(ctx) } -func (m *MinerAPI) StatesForMining(ctx context.Context, addrs []address.Address) ([]dtypes.MinerState, error) { +func (m *MinerAPI) StatesForMining(ctx context.Context, addrs []address.Address) ([]types.MinerState, error) { return m.MiningAPI.StatesForMining(ctx, addrs) } -func (m *MinerAPI) CountWinners(ctx context.Context, addrs []address.Address, start abi.ChainEpoch, end abi.ChainEpoch) ([]dtypes.CountWinners, error) { +func (m *MinerAPI) CountWinners(ctx context.Context, addrs []address.Address, start abi.ChainEpoch, end abi.ChainEpoch) ([]types.CountWinners, error) { return m.MiningAPI.CountWinners(ctx, addrs, start, end) } @@ -42,7 +40,3 @@ func (m *MinerAPI) Start(ctx context.Context, addrs []address.Address) error { func (m *MinerAPI) Stop(ctx context.Context, addrs []address.Address) error { return m.MiningAPI.ManualStop(ctx, addrs) } - -func (s *MinerAPI) AddAddress(ctx context.Context, mi dtypes.MinerInfo) error { - return s.MiningAPI.AddAddress(ctx, mi) -} diff --git a/node/modules/block_recorder/block_record.go b/node/modules/block_recorder/block_record.go deleted file mode 100644 index a54399a..0000000 --- a/node/modules/block_recorder/block_record.go +++ /dev/null @@ -1,69 +0,0 @@ -package block_recorder - -import ( - "context" - "fmt" - - "github.com/filecoin-project/go-address" - lru "github.com/hashicorp/golang-lru" - "github.com/ipfs/go-datastore" - - "github.com/filecoin-project/venus-miner/node/modules/dtypes" -) - -const ( - Local = "local" - Cache = "cache" - MySQL = "mysql" -) - -type IBlockRecord interface { - MarkAsProduced(miner address.Address, height uint64) error - Has(miner address.Address, height uint64) bool -} - -type LocalDBRecord struct { - da dtypes.MetadataDS -} - -func NewLocalDBRecord(da dtypes.MetadataDS) *LocalDBRecord { - return &LocalDBRecord{da: da} -} - -func (l *LocalDBRecord) MarkAsProduced(ctx context.Context, miner address.Address, height uint64) error { - blkKey := datastore.NewKey(fmt.Sprintf("%s-%d", miner, height)) - return l.da.Put(ctx, blkKey, []byte{1}) -} - -func (l *LocalDBRecord) Has(ctx context.Context, miner address.Address, height uint64) bool { - blkKey := datastore.NewKey(fmt.Sprintf("%s-%d", miner, height)) - has, err := l.da.Has(ctx, blkKey) - if err != nil { - return false - } - return has -} - -type CacheRecord struct { - cache *lru.ARCCache -} - -func NewCacheRecord() (*CacheRecord, error) { - cache, err := lru.NewARC(100000) - if err != nil { - return nil, err - } - return &CacheRecord{cache: cache}, nil -} - -func (c *CacheRecord) MarkAsProduced(miner address.Address, height uint64) error { - blkKey := datastore.NewKey(fmt.Sprintf("%s-%d", miner, height)) - c.cache.Add(blkKey, true) - return nil -} - -func (c *CacheRecord) Has(miner address.Address, height uint64) bool { - blkKey := datastore.NewKey(fmt.Sprintf("%s-%d", miner, height)) - _, has := c.cache.Get(blkKey) - return has -} diff --git a/node/modules/core.go b/node/modules/core.go index 6c3ea85..fa6d49c 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -4,16 +4,16 @@ import ( "crypto/rand" "errors" "fmt" - types2 "github.com/filecoin-project/venus-miner/types" "io" "io/ioutil" + types2 "github.com/filecoin-project/venus-miner/types" + "github.com/filecoin-project/go-jsonrpc/auth" "github.com/gbrlsnchs/jwt/v3" logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/venus-miner/api" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" "github.com/filecoin-project/venus-miner/node/repo" ) @@ -30,7 +30,7 @@ type JwtPayload struct { Allow []auth.Permission } -func APISecret(keystore types2.KeyStore, lr repo.LockedRepo) (*dtypes.APIAlg, error) { +func APISecret(keystore types2.KeyStore, lr repo.LockedRepo) (*types2.APIAlg, error) { key, err := keystore.Get(JWTSecretName) if errors.Is(err, types2.ErrKeyInfoNotFound) { @@ -67,5 +67,5 @@ func APISecret(keystore types2.KeyStore, lr repo.LockedRepo) (*dtypes.APIAlg, er return nil, fmt.Errorf("could not get JWT Token: %w", err) } - return (*dtypes.APIAlg)(jwt.NewHS256(key.PrivateKey)), nil + return (*types2.APIAlg)(jwt.NewHS256(key.PrivateKey)), nil } diff --git a/node/modules/miner-manager/api.go b/node/modules/miner-manager/api.go new file mode 100644 index 0000000..f71df73 --- /dev/null +++ b/node/modules/miner-manager/api.go @@ -0,0 +1,17 @@ +package miner_manager + +import ( + "context" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/venus-miner/types" +) + +type MinerManageAPI interface { + Has(ctx context.Context, checkAddr address.Address) bool + Get(ctx context.Context, checkAddr address.Address) *types.MinerInfo + List(ctx context.Context) ([]types.MinerInfo, error) + Update(ctx context.Context, skip, limit int64) ([]types.MinerInfo, error) + Count(ctx context.Context) int +} diff --git a/node/modules/minermanage/auth/auth.go b/node/modules/miner-manager/auth_manager.go similarity index 60% rename from node/modules/minermanage/auth/auth.go rename to node/modules/miner-manager/auth_manager.go index 77bd88d..e7add11 100644 --- a/node/modules/minermanage/auth/auth.go +++ b/node/modules/miner-manager/auth_manager.go @@ -1,4 +1,4 @@ -package auth +package miner_manager import ( "context" @@ -7,30 +7,29 @@ import ( "strconv" "sync" + "github.com/filecoin-project/go-address" "github.com/go-resty/resty/v2" logging "github.com/ipfs/go-log/v2" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" - "github.com/filecoin-project/venus-miner/node/modules/minermanage" + "github.com/filecoin-project/venus-miner/types" ) const CoMinersLimit = 200 var log = logging.Logger("auth-miners") -type MinerManagerForAuth struct { +type MinerManage struct { cli *resty.Client token string - miners []dtypes.MinerInfo + miners []types.MinerInfo lk sync.Mutex } -func NewMinerManager(url, token string) func() (minermanage.MinerManageAPI, error) { - return func() (minermanage.MinerManageAPI, error) { +func NewMinerManager(url, token string) func() (MinerManageAPI, error) { + return func() (MinerManageAPI, error) { cli := resty.New().SetHostURL(url).SetHeader("Accept", "application/json") - m := &MinerManagerForAuth{cli: cli, token: token} + m := &MinerManage{cli: cli, token: token} miners, err := m.Update(context.TODO(), 0, 0) if err != nil { @@ -42,20 +41,7 @@ func NewMinerManager(url, token string) func() (minermanage.MinerManageAPI, erro } } -func (m *MinerManagerForAuth) Put(ctx context.Context, mi dtypes.MinerInfo) error { - m.lk.Lock() - defer m.lk.Unlock() - - if m.Has(ctx, mi.Addr) { - log.Warnf("addr %s has exit", mi.Addr) - return nil - } - - m.miners = append(m.miners, mi) - return nil -} - -func (m *MinerManagerForAuth) Has(ctx context.Context, addr address.Address) bool { +func (m *MinerManage) Has(ctx context.Context, addr address.Address) bool { m.lk.Lock() defer m.lk.Unlock() @@ -68,7 +54,7 @@ func (m *MinerManagerForAuth) Has(ctx context.Context, addr address.Address) boo return false } -func (m *MinerManagerForAuth) Get(ctx context.Context, addr address.Address) *dtypes.MinerInfo { +func (m *MinerManage) Get(ctx context.Context, addr address.Address) *types.MinerInfo { m.lk.Lock() defer m.lk.Unlock() @@ -81,14 +67,14 @@ func (m *MinerManagerForAuth) Get(ctx context.Context, addr address.Address) *dt return nil } -func (m *MinerManagerForAuth) List(ctx context.Context) ([]dtypes.MinerInfo, error) { +func (m *MinerManage) List(ctx context.Context) ([]types.MinerInfo, error) { m.lk.Lock() defer m.lk.Unlock() return m.miners, nil } -func (m *MinerManagerForAuth) Update(ctx context.Context, skip, limit int64) ([]dtypes.MinerInfo, error) { +func (m *MinerManage) Update(ctx context.Context, skip, limit int64) ([]types.MinerInfo, error) { if limit == 0 { limit = CoMinersLimit } @@ -96,7 +82,7 @@ func (m *MinerManagerForAuth) Update(ctx context.Context, skip, limit int64) ([] if err != nil { return nil, err } - var mInfos = make([]dtypes.MinerInfo, 0) + var mInfos = make([]types.MinerInfo, 0) for _, u := range users { if u.State != 1 { @@ -115,7 +101,7 @@ func (m *MinerManagerForAuth) Update(ctx context.Context, skip, limit int64) ([] continue } - mInfos = append(mInfos, dtypes.MinerInfo{ + mInfos = append(mInfos, types.MinerInfo{ Addr: addr, Id: u.ID, Name: u.Name, @@ -128,17 +114,15 @@ func (m *MinerManagerForAuth) Update(ctx context.Context, skip, limit int64) ([] return m.miners, nil } -func (m *MinerManagerForAuth) Count(ctx context.Context) int { +func (m *MinerManage) Count(ctx context.Context) int { m.lk.Lock() defer m.lk.Unlock() return len(m.miners) } -var _ minermanage.MinerManageAPI = &MinerManagerForAuth{} - -func (m *MinerManagerForAuth) listUsers(skip, limit int64) ([]*dtypes.User, error) { - var users []*dtypes.User +func (m *MinerManage) listUsers(skip, limit int64) ([]*types.User, error) { + var users []*types.User resp, err := m.cli.R().SetQueryParams(map[string]string{ "skip": strconv.FormatInt(skip, 10), "limit": strconv.FormatInt(limit, 10), @@ -147,20 +131,20 @@ func (m *MinerManagerForAuth) listUsers(skip, limit int64) ([]*dtypes.User, erro return nil, err } if resp.StatusCode() == http.StatusOK { - return *(resp.Result().(*[]*dtypes.User)), nil + return *(resp.Result().(*[]*types.User)), nil } return nil, resp.Error().(*apiErr).Err() } -func (m *MinerManagerForAuth) listMiners(user string) ([]*dtypes.Miner, error) { - var res []*dtypes.Miner +func (m *MinerManage) listMiners(user string) ([]*types.Miner, error) { + var res []*types.Miner resp, err := m.cli.R().SetQueryParams(map[string]string{"user": user}). SetResult(&res).SetError(&apiErr{}).Get("/miner/list-by-user") if err != nil { return nil, err } if resp.StatusCode() == http.StatusOK { - return *(resp.Result().(*[]*dtypes.Miner)), nil + return *(resp.Result().(*[]*types.Miner)), nil } return nil, resp.Error().(*apiErr).Err() } @@ -172,3 +156,5 @@ type apiErr struct { func (err *apiErr) Err() error { return fmt.Errorf(err.Error) } + +var _ MinerManageAPI = &MinerManage{} diff --git a/node/modules/miner.go b/node/modules/miner.go index 2416f6c..97b8f41 100644 --- a/node/modules/miner.go +++ b/node/modules/miner.go @@ -2,30 +2,28 @@ package modules import ( "context" + "github.com/filecoin-project/venus-miner/node/modules/miner-manager" "go.uber.org/fx" "github.com/filecoin-project/venus-miner/lib/journal" "github.com/filecoin-project/venus-miner/miner" - "github.com/filecoin-project/venus-miner/miner/slashfilter" "github.com/filecoin-project/venus-miner/node/config" - "github.com/filecoin-project/venus-miner/node/modules/block_recorder" - "github.com/filecoin-project/venus-miner/node/modules/minermanage" + "github.com/filecoin-project/venus-miner/node/modules/slashfilter" "github.com/filecoin-project/venus/pkg/util/ffiwrapper" - v1api "github.com/filecoin-project/venus/venus-shared/api/chain/v1" + fullnode "github.com/filecoin-project/venus/venus-shared/api/chain/v1" ) -func NewWiningPoster(lc fx.Lifecycle, - api v1api.FullNode, +func NewMinerProcessor(lc fx.Lifecycle, + api fullnode.FullNode, gtNode *config.GatewayNode, sfAPI slashfilter.SlashFilterAPI, verifier ffiwrapper.Verifier, - minerManager minermanage.MinerManageAPI, + minerManager miner_manager.MinerManageAPI, j journal.Journal, - blockRecord block_recorder.IBlockRecord, ) (miner.MiningAPI, error) { - m := miner.NewMiner(api, gtNode, verifier, minerManager, sfAPI, j, blockRecord) + m := miner.NewMiner(api, gtNode, verifier, minerManager, sfAPI, j) lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { diff --git a/node/modules/minermanage/local/local.go b/node/modules/minermanage/local/local.go deleted file mode 100644 index 7abc40c..0000000 --- a/node/modules/minermanage/local/local.go +++ /dev/null @@ -1,225 +0,0 @@ -package local - -import ( - "context" - "encoding/json" - "fmt" - "sync" - - "github.com/ipfs/go-datastore" - logging "github.com/ipfs/go-log/v2" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/venus-miner/node/modules/dtypes" - "github.com/filecoin-project/venus-miner/node/modules/minermanage" -) - -var log = logging.Logger("local_minermgr") - -const actorKey = "miner-actors" -const defaultKey = "default-actor" - -var ErrNoDefault = fmt.Errorf("not set default key") - -type MinerManager struct { - miners []dtypes.MinerInfo - - da dtypes.MetadataDS - lk sync.Mutex -} - -func NewMinerManger(ds dtypes.MetadataDS) (minermanage.MinerManageAPI, error) { - addrBytes, err := ds.Get(context.TODO(), datastore.NewKey(actorKey)) - if err != nil && err != datastore.ErrNotFound { - return nil, err - } - - var miners []dtypes.MinerInfo - - if err != datastore.ErrNotFound { - err = json.Unmarshal(addrBytes, &miners) - if err != nil { - return nil, err - } - } - - return &MinerManager{da: ds, miners: miners}, nil -} - -func (m *MinerManager) Put(ctx context.Context, miner dtypes.MinerInfo) error { - m.lk.Lock() - defer m.lk.Unlock() - - if m.Has(ctx, miner.Addr) { - log.Warnf("addr %s has exit", miner.Addr) - return nil - } - - newMiner := append(m.miners, miner) - addrBytes, err := json.Marshal(newMiner) - if err != nil { - return err - } - err = m.da.Put(ctx, datastore.NewKey(actorKey), addrBytes) - if err != nil { - return err - } - - m.miners = newMiner - return nil -} - -func (m *MinerManager) Set(ctx context.Context, miner dtypes.MinerInfo) error { - m.lk.Lock() - defer m.lk.Unlock() - - for _, addr := range m.miners { - if addr.Addr.String() == miner.Addr.String() { - //if miner.Sealer.ListenAPI != "" && miner.Sealer.ListenAPI != m.miners[k].Sealer.ListenAPI { - // m.miners[k].Sealer.ListenAPI = miner.Sealer.ListenAPI - //} - // - //if miner.Sealer.Token != "" && miner.Sealer.Token != m.miners[k].Sealer.Token { - // m.miners[k].Sealer.Token = miner.Sealer.Token - //} - // - //if miner.Wallet.ListenAPI != "" && miner.Wallet.ListenAPI != m.miners[k].Wallet.ListenAPI { - // m.miners[k].Wallet.ListenAPI = miner.Wallet.ListenAPI - //} - // - //if miner.Wallet.Token != "" && miner.Wallet.Token != m.miners[k].Wallet.Token { - // m.miners[k].Wallet.Token = miner.Wallet.Token - //} - - addrBytes, err := json.Marshal(m.miners) - if err != nil { - return err - } - - err = m.da.Put(ctx, datastore.NewKey(actorKey), addrBytes) - if err != nil { - return err - } - - break - } - } - - return nil -} - -func (m *MinerManager) Has(ctx context.Context, addr address.Address) bool { - for _, miner := range m.miners { - if miner.Addr.String() == addr.String() { - return true - } - } - - return false -} - -func (m *MinerManager) Get(ctx context.Context, addr address.Address) *dtypes.MinerInfo { - m.lk.Lock() - defer m.lk.Unlock() - - for k := range m.miners { - if m.miners[k].Addr.String() == addr.String() { - return &m.miners[k] - } - } - - return nil -} - -func (m *MinerManager) List(ctx context.Context) ([]dtypes.MinerInfo, error) { - m.lk.Lock() - defer m.lk.Unlock() - - return m.miners, nil -} - -func (m *MinerManager) Remove(ctx context.Context, rmAddr address.Address) error { - m.lk.Lock() - defer m.lk.Unlock() - - if !m.Has(ctx, rmAddr) { - return nil - } - - var newMiners []dtypes.MinerInfo - for _, miner := range m.miners { - if miner.Addr.String() != rmAddr.String() { - newMiners = append(newMiners, miner) - } - } - - addrBytes, err := json.Marshal(newMiners) - if err != nil { - return err - } - err = m.da.Put(ctx, datastore.NewKey(actorKey), addrBytes) - if err != nil { - return err - } - - m.miners = newMiners - - //rm default if rmAddr == defaultAddr - defaultAddr, err := m.Default(ctx) - if err != nil { - if err == ErrNoDefault { - return nil - } - return err - } - - if rmAddr == defaultAddr { - err := m.rmDefault(ctx) - if err != nil { - return err - } - } - - return nil -} - -func (m *MinerManager) rmDefault(ctx context.Context) error { - return m.da.Delete(ctx, datastore.NewKey(defaultKey)) -} - -func (m *MinerManager) SetDefault(ctx context.Context, addr address.Address) error { - return m.da.Put(ctx, datastore.NewKey(defaultKey), addr.Bytes()) -} - -func (m *MinerManager) Default(ctx context.Context) (address.Address, error) { - bytes, err := m.da.Get(ctx, datastore.NewKey(defaultKey)) - if err != nil { - // set the address with index 0 as the default address - if len(m.miners) == 0 { - return address.Undef, ErrNoDefault - } - - err = m.SetDefault(ctx, m.miners[0].Addr) - if err != nil { - return address.Undef, err - } - - return m.miners[0].Addr, nil - } - - return address.NewFromBytes(bytes) -} - -func (m *MinerManager) Update(ctx context.Context, skip, limit int64) ([]dtypes.MinerInfo, error) { - return nil, nil -} - -func (m *MinerManager) Count(ctx context.Context) int { - m.lk.Lock() - defer m.lk.Unlock() - - return len(m.miners) -} - -var _ minermanage.MinerManageAPI = &MinerManager{} diff --git a/node/modules/minermanage/minermanger.go b/node/modules/minermanage/minermanger.go deleted file mode 100644 index baf085d..0000000 --- a/node/modules/minermanage/minermanger.go +++ /dev/null @@ -1,26 +0,0 @@ -package minermanage - -import ( - "context" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/venus-miner/node/modules/dtypes" -) - -const ( - Local = "local" - MySQL = "mysql" - Auth = "auth" -) - -type MinerManageAPI interface { - Put(ctx context.Context, addr dtypes.MinerInfo) error - //Set(ctx context.Context, addr dtypes.MinerInfo) error - Has(ctx context.Context, checkAddr address.Address) bool - Get(ctx context.Context, checkAddr address.Address) *dtypes.MinerInfo - List(ctx context.Context) ([]dtypes.MinerInfo, error) - Update(ctx context.Context, skip, limit int64) ([]dtypes.MinerInfo, error) - //Remove(ctx context.Context, rmAddr address.Address) error - Count(ctx context.Context) int -} diff --git a/node/modules/minermanage/mysql/mysql.go b/node/modules/minermanage/mysql/mysql.go deleted file mode 100644 index 305858f..0000000 --- a/node/modules/minermanage/mysql/mysql.go +++ /dev/null @@ -1,204 +0,0 @@ -package mysql - -import ( - "context" - "fmt" - "sync" - "time" - - logging "github.com/ipfs/go-log/v2" - "gorm.io/driver/mysql" - "gorm.io/gorm" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/venus-miner/node/config" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" - "github.com/filecoin-project/venus-miner/node/modules/minermanage" -) - -var log = logging.Logger("mysql") - -type mysqlMinerInfo struct { - Addr string `gorm:"column:addr;type:varchar(20);uniqueIndex;NOT NULL"` - Name string `gorm:"column:name;type:varchar(50);"` - Comment string `gorm:"column:comment;type:varchar(50);"` - State int `gorm:"column:state;type:int(10);comment:'0 for init,1 for active'"` - CreateTime uint64 `gorm:"column:create_time;not null;type:bigint(20) unsigned" json:"createTime"` - UpdateTime uint64 `gorm:"column:update_time;not null;type:bigint(20) unsigned" json:"updateTime"` -} - -func (m *mysqlMinerInfo) TableName() string { - return "miner_infos" -} - -type MinerManagerForMySQL struct { - miners []dtypes.MinerInfo - - _db *gorm.DB - lk sync.Mutex -} - -func NewMinerManger(cfg *config.MySQLConfig) func() (minermanage.MinerManageAPI, error) { - return func() (minermanage.MinerManageAPI, error) { - // root:123456@tcp(ip:3306)/venus_miner?charset=utf8mb4&parseTime=True&loc=Local&timeout=10s - db, err := gorm.Open(mysql.Open(cfg.Conn), &gorm.Config{ - //Logger: logger.Default.LogMode(logger.Info), - }) - if err != nil { - return nil, fmt.Errorf("[db connection failed] conn: %s %w", cfg.Conn, err) - } - - db.Set("gorm:table_options", "CHARSET=utf8mb4") - if cfg.Debug { - db = db.Debug() - } - - if err := db.AutoMigrate(mysqlMinerInfo{}); err != nil { - return nil, err - } - - sqlDB, err := db.DB() - if err != nil { - return nil, err - } - - // Set the maximum number of idle connections in the connection pool. - sqlDB.SetMaxIdleConns(cfg.MaxIdleConn) - // Set the maximum number of open database connections. - sqlDB.SetMaxOpenConns(cfg.MaxOpenConn) - // The maximum time that the connection can be reused is set. - sqlDB.SetConnMaxLifetime(time.Second * cfg.ConnMaxLifeTime) - - log.Info("init mysql success for MinerManger!") - m := &MinerManagerForMySQL{ - _db: db, - } - - if err := m.init(); err != nil { - return nil, err - } - - return m, nil - } -} - -func (m *MinerManagerForMySQL) init() error { - var res []mysqlMinerInfo - if err := m._db.Find(&res).Error; err != nil { - return err - } - - for _, val := range res { - addr, err := address.NewFromString(val.Addr) - if err != nil { - return err - } - m.miners = append(m.miners, dtypes.MinerInfo{ - Addr: addr, - }) - } - - return nil -} - -func (m *MinerManagerForMySQL) Put(ctx context.Context, miner dtypes.MinerInfo) error { - m.lk.Lock() - defer m.lk.Unlock() - - err := m._db.Create(&mysqlMinerInfo{Addr: miner.Addr.String()}).Error - if err != nil { - return err - } - - m.miners = append(m.miners, miner) - return nil -} - -func (m *MinerManagerForMySQL) Set(ctx context.Context, miner dtypes.MinerInfo) error { - m.lk.Lock() - defer m.lk.Unlock() - - for _, addr := range m.miners { - if addr.Addr.String() == miner.Addr.String() { - // ToDo other changes - - err := m.Put(ctx, miner) - if err != nil { - return err - } - break - } - } - - return nil -} - -func (m *MinerManagerForMySQL) Has(ctx context.Context, addr address.Address) bool { - for _, miner := range m.miners { - if miner.Addr.String() == addr.String() { - return true - } - } - - return false -} - -func (m *MinerManagerForMySQL) Get(ctx context.Context, addr address.Address) *dtypes.MinerInfo { - m.lk.Lock() - defer m.lk.Unlock() - - for k := range m.miners { - if m.miners[k].Addr.String() == addr.String() { - return &m.miners[k] - } - } - - return nil -} - -func (m *MinerManagerForMySQL) List(ctx context.Context) ([]dtypes.MinerInfo, error) { - m.lk.Lock() - defer m.lk.Unlock() - - return m.miners, nil -} - -func (m *MinerManagerForMySQL) Remove(ctx context.Context, rmAddr address.Address) error { - m.lk.Lock() - defer m.lk.Unlock() - - if !m.Has(ctx, rmAddr) { - return nil - } - - var res mysqlMinerInfo - m._db.Where("`addr` = ?", rmAddr.String()).Take(&res) - - if err := m._db.Delete(&res).Error; err != nil { - return err - } - - var newMiners []dtypes.MinerInfo - for _, miner := range m.miners { - if miner.Addr.String() != rmAddr.String() { - newMiners = append(newMiners, miner) - } - } - - m.miners = newMiners - - return nil -} - -func (m *MinerManagerForMySQL) Update(ctx context.Context, skip, limit int64) ([]dtypes.MinerInfo, error) { - return nil, nil -} - -func (m *MinerManagerForMySQL) Count(ctx context.Context) int { - m.lk.Lock() - defer m.lk.Unlock() - - return len(m.miners) -} - -var _ minermanage.MinerManageAPI = &MinerManagerForMySQL{} diff --git a/node/modules/services.go b/node/modules/services.go index 46abd2f..5781ce1 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -2,6 +2,7 @@ package modules import ( "context" + "go.uber.org/fx" "github.com/filecoin-project/venus-miner/lib/journal" diff --git a/node/modules/slashfilter/api.go b/node/modules/slashfilter/api.go new file mode 100644 index 0000000..efabcae --- /dev/null +++ b/node/modules/slashfilter/api.go @@ -0,0 +1,22 @@ +package slashfilter + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + + vtypes "github.com/filecoin-project/venus/venus-shared/types" +) + +type BlockStoreType string + +const ( + Local BlockStoreType = "local" + MySQL BlockStoreType = "mysql" +) + +type SlashFilterAPI interface { + HasBlock(ctx context.Context, bh *vtypes.BlockHeader) (bool, error) + MinedBlock(ctx context.Context, bh *vtypes.BlockHeader, parentEpoch abi.ChainEpoch) error + PutBlock(ctx context.Context, bh *vtypes.BlockHeader, parentEpoch abi.ChainEpoch) error +} diff --git a/miner/slashfilter/slashfilter.go b/node/modules/slashfilter/local.go similarity index 55% rename from miner/slashfilter/slashfilter.go rename to node/modules/slashfilter/local.go index 093fee8..567ee36 100644 --- a/miner/slashfilter/slashfilter.go +++ b/node/modules/slashfilter/local.go @@ -5,45 +5,52 @@ import ( "fmt" "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" - "github.com/filecoin-project/venus/venus-shared/types" + vtypes "github.com/filecoin-project/venus/venus-shared/types" + + "github.com/filecoin-project/venus-miner/types" ) -type SlashFilterAPI interface { - MinedBlock(context.Context, *types.BlockHeader, abi.ChainEpoch) error +type localSlashFilter struct { + byEpoch datastore.Datastore // double-fork mining faults, parent-grinding fault + byParents datastore.Datastore // time-offset mining faults } -type SlashFilter struct { - byEpoch ds.Datastore // double-fork mining faults, parent-grinding fault - byParents ds.Datastore // time-offset mining faults +func NewLocal(ds types.MetadataDS) SlashFilterAPI { + return &localSlashFilter{ + byEpoch: namespace.Wrap(ds, datastore.NewKey("/slashfilter/epoch")), + byParents: namespace.Wrap(ds, datastore.NewKey("/slashfilter/parents")), + } } -func NewLocal(ds dtypes.MetadataDS) SlashFilterAPI { - return New(ds) +func (f *localSlashFilter) HasBlock(ctx context.Context, bh *vtypes.BlockHeader) (bool, error) { + epochKey := datastore.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, bh.Height)) + + return f.byEpoch.Has(ctx, epochKey) } -func New(dstore ds.Batching) *SlashFilter { - return &SlashFilter{ - byEpoch: namespace.Wrap(dstore, ds.NewKey("/slashfilter/epoch")), - byParents: namespace.Wrap(dstore, ds.NewKey("/slashfilter/parents")), +func (f *localSlashFilter) PutBlock(ctx context.Context, bh *vtypes.BlockHeader, parentEpoch abi.ChainEpoch) error { + parentsKey := datastore.NewKey(fmt.Sprintf("/%s/%x", bh.Miner, vtypes.NewTipSetKey(bh.Parents...).Bytes())) + if err := f.byParents.Put(ctx, parentsKey, bh.Cid().Bytes()); err != nil { + return fmt.Errorf("putting byEpoch entry: %w", err) } -} -func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) error { - epochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, bh.Height)) - { - // double-fork mining (2 blocks at one epoch) - if err := checkFault(ctx, f.byEpoch, epochKey, bh, "double-fork mining faults"); err != nil { - return err - } + epochKey := datastore.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, bh.Height)) + if err := f.byEpoch.Put(ctx, epochKey, bh.Cid().Bytes()); err != nil { + return fmt.Errorf("putting byEpoch entry: %w", err) } - parentsKey := ds.NewKey(fmt.Sprintf("/%s/%x", bh.Miner, types.NewTipSetKey(bh.Parents...).Bytes())) + return nil +} + +func (f *localSlashFilter) MinedBlock(ctx context.Context, bh *vtypes.BlockHeader, parentEpoch abi.ChainEpoch) error { + // double-fork mining (2 blocks at one epoch) --> HasBlock + + parentsKey := datastore.NewKey(fmt.Sprintf("/%s/%x", bh.Miner, vtypes.NewTipSetKey(bh.Parents...).Bytes())) { // time-offset mining faults (2 blocks with the same parents) if err := checkFault(ctx, f.byParents, parentsKey, bh, "time-offset mining faults"); err != nil { @@ -55,7 +62,7 @@ func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, par // parent-grinding fault (didn't mine on top of our own block) // First check if we have mined a block on the parent epoch - parentEpochKey := ds.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, parentEpoch)) + parentEpochKey := datastore.NewKey(fmt.Sprintf("/%s/%d", bh.Miner, parentEpoch)) have, err := f.byEpoch.Has(ctx, parentEpochKey) if err != nil { return err @@ -86,18 +93,10 @@ func (f *SlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, par } } - if err := f.byParents.Put(ctx, parentsKey, bh.Cid().Bytes()); err != nil { - return fmt.Errorf("putting byEpoch entry: %w", err) - } - - if err := f.byEpoch.Put(ctx, epochKey, bh.Cid().Bytes()); err != nil { - return fmt.Errorf("putting byEpoch entry: %w", err) - } - return nil } -func checkFault(ctx context.Context, t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType string) error { +func checkFault(ctx context.Context, t datastore.Datastore, key datastore.Key, bh *vtypes.BlockHeader, faultType string) error { fault, err := t.Has(ctx, key) if err != nil { return err diff --git a/miner/slashfilter/mysql_slashfilter.go b/node/modules/slashfilter/mysql.go similarity index 74% rename from miner/slashfilter/mysql_slashfilter.go rename to node/modules/slashfilter/mysql.go index c479696..1ba1e1d 100644 --- a/miner/slashfilter/mysql_slashfilter.go +++ b/node/modules/slashfilter/mysql.go @@ -19,7 +19,7 @@ import ( var log = logging.Logger("mysql_slashFilter") -type MysqlSlashFilter struct { +type mysqlSlashFilter struct { _db *gorm.DB } @@ -27,18 +27,18 @@ type MinedBlock struct { ParentEpoch int64 `gorm:"column:parent_epoch;type:bigint(20);NOT NULL"` ParentKey string `gorm:"column:parent_key;type:varchar(256);NOT NULL"` - Epoch int64 `gorm:"column:epoch;type:bigint(20);NOT NULL"` - Miner string `gorm:"column:miner;type:varchar(256);NOT NULL"` - Cid string `gorm:"column:cid;type:varchar(256);NOT NULL"` + Epoch int64 `gorm:"column:epoch;type:bigint(20);NOT NULL;primary_key"` + Miner string `gorm:"column:miner;type:varchar(256);NOT NULL;primary_key"` + Cid string `gorm:"column:cid;type:varchar(256)"` } func (m *MinedBlock) TableName() string { return "miner_blocks" } -var _ SlashFilterAPI = (*MysqlSlashFilter)(nil) +var _ SlashFilterAPI = (*mysqlSlashFilter)(nil) -func NewMysqlSlashFilter(cfg *config.MySQLConfig) func() (SlashFilterAPI, error) { +func NewMysql(cfg *config.MySQLConfig) func() (SlashFilterAPI, error) { return func() (SlashFilterAPI, error) { db, err := gorm.Open(mysql.Open(cfg.Conn)) if err != nil { @@ -67,19 +67,22 @@ func NewMysqlSlashFilter(cfg *config.MySQLConfig) func() (SlashFilterAPI, error) sqlDB.SetConnMaxLifetime(time.Second * cfg.ConnMaxLifeTime) log.Info("init mysql success for mysqlSlashFilter!") - return &MysqlSlashFilter{ + return &mysqlSlashFilter{ _db: db, }, nil } } // double-fork mining (2 blocks at one epoch) -func (f *MysqlSlashFilter) checkSameHeightFault(bh *types.BlockHeader) error { +func (f *mysqlSlashFilter) checkSameHeightFault(bh *types.BlockHeader) error { // nolint: unused var bk MinedBlock err := f._db.Model(&MinedBlock{}).Take(&bk, "miner=? and epoch=?", bh.Miner.String(), bh.Height).Error if err == gorm.ErrRecordNotFound { return nil } + if err != nil { + return err + } other, err := cid.Decode(bk.Cid) if err != nil { @@ -91,16 +94,18 @@ func (f *MysqlSlashFilter) checkSameHeightFault(bh *types.BlockHeader) error { } return fmt.Errorf("produced block would trigger double-fork mining faults consensus fault; miner: %s; bh: %s, other: %s", bh.Miner, bh.Cid(), other) - } // time-offset mining faults (2 blocks with the same parents) -func (f *MysqlSlashFilter) checkSameParentFault(bh *types.BlockHeader) error { +func (f *mysqlSlashFilter) checkSameParentFault(bh *types.BlockHeader) error { var bk MinedBlock err := f._db.Model(&MinedBlock{}).Take(&bk, "miner=? and parent_key=?", bh.Miner.String(), types.NewTipSetKey(bh.Parents...).String()).Error if err == gorm.ErrRecordNotFound { return nil } + if err != nil { + return err + } other, err := cid.Decode(bk.Cid) if err != nil { @@ -112,13 +117,36 @@ func (f *MysqlSlashFilter) checkSameParentFault(bh *types.BlockHeader) error { } return fmt.Errorf("produced block would trigger time-offset mining faults consensus fault; miner: %s; bh: %s, other: %s", bh.Miner, bh.Cid(), other) - } -func (f *MysqlSlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) error { - if err := f.checkSameHeightFault(bh); err != nil { - return err +func (f *mysqlSlashFilter) HasBlock(ctx context.Context, bh *types.BlockHeader) (bool, error) { + var bk MinedBlock + err := f._db.Model(&MinedBlock{}).Take(&bk, "miner=? and epoch=?", bh.Miner.String(), bh.Height).Error + if err == gorm.ErrRecordNotFound { + return false, nil } + if err != nil { + return false, err + } + + return true, nil +} + +func (f *mysqlSlashFilter) PutBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) error { + return f._db.Save(&MinedBlock{ + ParentEpoch: int64(parentEpoch), + ParentKey: types.NewTipSetKey(bh.Parents...).String(), + Epoch: int64(bh.Height), + Miner: bh.Miner.String(), + Cid: bh.Cid().String(), + }).Error +} + +func (f *mysqlSlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader, parentEpoch abi.ChainEpoch) error { + // double-fork mining (2 blocks at one epoch) --> HasBlock + //if err := f.checkSameHeightFault(bh); err != nil { + // return err + //} if err := f.checkSameParentFault(bh); err != nil { return err @@ -154,11 +182,5 @@ func (f *MysqlSlashFilter) MinedBlock(ctx context.Context, bh *types.BlockHeader //if not exit good block } - return f._db.Create(&MinedBlock{ - ParentEpoch: int64(parentEpoch), - ParentKey: types.NewTipSetKey(bh.Parents...).String(), - Epoch: int64(bh.Height), - Miner: bh.Miner.String(), - Cid: bh.Cid().String(), - }).Error + return nil } diff --git a/node/modules/storage.go b/node/modules/storage.go index e9cb0b1..41fcf13 100644 --- a/node/modules/storage.go +++ b/node/modules/storage.go @@ -2,11 +2,11 @@ package modules import ( "context" + types2 "github.com/filecoin-project/venus-miner/types" "go.uber.org/fx" - "github.com/filecoin-project/venus-miner/node/modules/dtypes" "github.com/filecoin-project/venus-miner/node/modules/helpers" "github.com/filecoin-project/venus-miner/node/repo" ) @@ -27,7 +27,7 @@ func KeyStore(lr repo.LockedRepo) (types2.KeyStore, error) { return lr.KeyStore() } -func Datastore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.MetadataDS, error) { +func Datastore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (types2.MetadataDS, error) { ctx := helpers.LifecycleCtx(mctx, lc) mds, err := r.Datastore(ctx, "/metadata") if err != nil { diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index b30b309..3265d7b 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - types2 "github.com/filecoin-project/venus-miner/types" "io" "io/ioutil" "os" @@ -22,6 +21,7 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/venus-miner/node/config" + "github.com/filecoin-project/venus-miner/types" ) const ( @@ -33,22 +33,6 @@ const ( fsKeystore = "keystore" ) -type RepoType int - -const ( - _ = iota // Default is invalid - Miner RepoType = iota -) - -func defConfForType(t RepoType) interface{} { - switch t { - case Miner: - return config.DefaultMinerConfig() - default: - panic(fmt.Sprintf("unknown RepoType(%d)", int(t))) - } -} - var log = logging.Logger("repo") var ErrRepoExists = errors.New("repo exists") @@ -92,7 +76,7 @@ func (fsr *FsRepo) Exists() (bool, error) { //nolint return !notexist, err } -func (fsr *FsRepo) Init(t RepoType) error { +func (fsr *FsRepo) Init() error { exist, err := fsr.Exists() if err != nil { return err @@ -107,7 +91,7 @@ func (fsr *FsRepo) Init(t RepoType) error { return err } - if err := fsr.initConfig(t); err != nil { + if err := fsr.initConfig(); err != nil { return fmt.Errorf("init config: %w", err) } @@ -115,7 +99,7 @@ func (fsr *FsRepo) Init(t RepoType) error { } -func (fsr *FsRepo) initConfig(t RepoType) error { +func (fsr *FsRepo) initConfig() error { _, err := os.Stat(fsr.configPath) if err == nil { // exists @@ -129,7 +113,7 @@ func (fsr *FsRepo) initConfig(t RepoType) error { return err } - comm, err := config.ConfigComment(defConfForType(t)) + comm, err := config.ConfigComment(config.DefaultMinerConfig()) if err != nil { return fmt.Errorf("comment: %w", err) } @@ -225,11 +209,11 @@ func (fsr *FsRepo) APIToken() ([]byte, error) { } func (fsr *FsRepo) Config() (interface{}, error) { - return config.FromFile(fsr.configPath, defConfForType(Miner)) + return config.FromFile(fsr.configPath, config.DefaultMinerConfig()) } // Lock acquires exclusive lock on this repo -func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) { +func (fsr *FsRepo) Lock() (LockedRepo, error) { locked, err := fslock.Locked(fsr.path, fsLock) if err != nil { return nil, fmt.Errorf("could not check lock status: %w", err) @@ -245,26 +229,13 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) { return &fsLockedRepo{ path: fsr.path, configPath: fsr.configPath, - repoType: repoType, closer: closer, }, nil } -// Like Lock, except datastores will work in read-only mode -func (fsr *FsRepo) LockRO(repoType RepoType) (LockedRepo, error) { - lr, err := fsr.Lock(repoType) - if err != nil { - return nil, err - } - - lr.(*fsLockedRepo).readonly = true - return lr, nil -} - type fsLockedRepo struct { path string configPath string - repoType RepoType closer io.Closer readonly bool @@ -321,7 +292,7 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) { } func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) { - return config.FromFile(fsr.configPath, defConfForType(fsr.repoType)) + return config.FromFile(fsr.configPath, config.DefaultMinerConfig()) } func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error { @@ -372,7 +343,7 @@ func (fsr *fsLockedRepo) SetAPIToken(token []byte) error { return ioutil.WriteFile(fsr.join(fsAPIToken), token, 0600) } -func (fsr *fsLockedRepo) KeyStore() (types2.KeyStore, error) { +func (fsr *fsLockedRepo) KeyStore() (types.KeyStore, error) { if err := fsr.stillValid(); err != nil { return nil, err } @@ -413,9 +384,9 @@ func (fsr *fsLockedRepo) List() ([]string, error) { } // Get gets a key out of keystore and returns types.KeyInfo coresponding to named key -func (fsr *fsLockedRepo) Get(name string) (types2.KeyInfo, error) { +func (fsr *fsLockedRepo) Get(name string) (types.KeyInfo, error) { if err := fsr.stillValid(); err != nil { - return types2.KeyInfo{}, err + return types.KeyInfo{}, err } encName := base32.RawStdEncoding.EncodeToString([]byte(name)) @@ -423,37 +394,37 @@ func (fsr *fsLockedRepo) Get(name string) (types2.KeyInfo, error) { fstat, err := os.Stat(keyPath) if os.IsNotExist(err) { - return types2.KeyInfo{}, fmt.Errorf("opening key '%s': %w", name, types2.ErrKeyInfoNotFound) + return types.KeyInfo{}, fmt.Errorf("opening key '%s': %w", name, types.ErrKeyInfoNotFound) } else if err != nil { - return types2.KeyInfo{}, fmt.Errorf("opening key '%s': %w", name, err) + return types.KeyInfo{}, fmt.Errorf("opening key '%s': %w", name, err) } if fstat.Mode()&0077 != 0 { - return types2.KeyInfo{}, fmt.Errorf(kstrPermissionMsg, name, fstat.Mode()) + return types.KeyInfo{}, fmt.Errorf(kstrPermissionMsg, name, fstat.Mode()) } file, err := os.Open(keyPath) if err != nil { - return types2.KeyInfo{}, fmt.Errorf("opening key '%s': %w", name, err) + return types.KeyInfo{}, fmt.Errorf("opening key '%s': %w", name, err) } defer file.Close() //nolint: errcheck // read only op data, err := ioutil.ReadAll(file) if err != nil { - return types2.KeyInfo{}, fmt.Errorf("reading key '%s': %w", name, err) + return types.KeyInfo{}, fmt.Errorf("reading key '%s': %w", name, err) } - var res types2.KeyInfo + var res types.KeyInfo err = json.Unmarshal(data, &res) if err != nil { - return types2.KeyInfo{}, fmt.Errorf("decoding key '%s': %w", name, err) + return types.KeyInfo{}, fmt.Errorf("decoding key '%s': %w", name, err) } return res, nil } // Put saves key info under given name -func (fsr *fsLockedRepo) Put(name string, info types2.KeyInfo) error { +func (fsr *fsLockedRepo) Put(name string, info types.KeyInfo) error { if err := fsr.stillValid(); err != nil { return err } @@ -463,7 +434,7 @@ func (fsr *fsLockedRepo) Put(name string, info types2.KeyInfo) error { _, err := os.Stat(keyPath) if err == nil { - return fmt.Errorf("checking key before put '%s': %w", name, types2.ErrKeyExists) + return fmt.Errorf("checking key before put '%s': %w", name, types.ErrKeyExists) } else if !os.IsNotExist(err) { return fmt.Errorf("checking key before put '%s': %w", name, err) } @@ -490,7 +461,7 @@ func (fsr *fsLockedRepo) Delete(name string) error { _, err := os.Stat(keyPath) if os.IsNotExist(err) { - return fmt.Errorf("checking key before delete '%s': %w", name, types2.ErrKeyInfoNotFound) + return fmt.Errorf("checking key before delete '%s': %w", name, types.ErrKeyInfoNotFound) } else if err != nil { return fmt.Errorf("checking key before delete '%s': %w", name, err) } diff --git a/node/repo/fsrepo_ds.go b/node/repo/fsrepo_ds.go index d47fe40..5972b3c 100644 --- a/node/repo/fsrepo_ds.go +++ b/node/repo/fsrepo_ds.go @@ -19,11 +19,6 @@ type dsCtor func(path string, readonly bool) (datastore.Batching, error) var fsDatastores = map[string]dsCtor{ "metadata": levelDs, - - // Those need to be fast for large writes... but also need a really good GC :c - "staging": badgerDs, // miner specific - - "client": badgerDs, // client specific } func badgerDs(path string, readonly bool) (datastore.Batching, error) { diff --git a/node/repo/fsrepo_test.go b/node/repo/fsrepo_test.go index 03677be..c9e37db 100644 --- a/node/repo/fsrepo_test.go +++ b/node/repo/fsrepo_test.go @@ -17,7 +17,7 @@ func genFsRepo(t *testing.T) (*FsRepo, func()) { t.Fatal(err) } - err = repo.Init(Miner) + err = repo.Init() if err != ErrRepoExists && err != nil { t.Fatal(err) } diff --git a/node/repo/interface.go b/node/repo/interface.go index d74e862..8f7570f 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -3,10 +3,11 @@ package repo import ( "context" "errors" - types2 "github.com/filecoin-project/venus-miner/types" "github.com/ipfs/go-datastore" "github.com/multiformats/go-multiaddr" + + "github.com/filecoin-project/venus-miner/types" ) var ( @@ -22,7 +23,7 @@ type Repo interface { APIToken() ([]byte, error) // Lock locks the repo for exclusive use. - Lock(RepoType) (LockedRepo, error) + Lock() (LockedRepo, error) } type LockedRepo interface { @@ -47,7 +48,7 @@ type LockedRepo interface { SetAPIToken([]byte) error // KeyStore returns store of private keys for Filecoin transactions - KeyStore() (types2.KeyStore, error) + KeyStore() (types.KeyStore, error) // Path returns absolute path of the repo Path() string diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go index 8e1665b..325918a 100644 --- a/node/repo/repo_test.go +++ b/node/repo/repo_test.go @@ -2,9 +2,10 @@ package repo import ( "errors" - types2 "github.com/filecoin-project/venus-miner/types" "testing" + types2 "github.com/filecoin-project/venus-miner/types" + "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" @@ -18,12 +19,12 @@ func basicTest(t *testing.T, repo Repo) { } assert.Nil(t, apima, "with no api endpoint, return should be nil") - lrepo, err := repo.Lock(Miner) + lrepo, err := repo.Lock() assert.NoError(t, err, "should be able to lock once") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") { - lrepo2, err := repo.Lock(Miner) + lrepo2, err := repo.Lock() if assert.Error(t, err) { assert.Equal(t, ErrRepoAlreadyLocked, err) } @@ -33,7 +34,7 @@ func basicTest(t *testing.T, repo Repo) { err = lrepo.Close() assert.NoError(t, err, "should be able to unlock") - lrepo, err = repo.Lock(Miner) + lrepo, err = repo.Lock() assert.NoError(t, err, "should be able to relock") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") @@ -64,7 +65,7 @@ func basicTest(t *testing.T, repo Repo) { k1 := types2.KeyInfo{Type: "foo"} k2 := types2.KeyInfo{Type: "bar"} - lrepo, err = repo.Lock(Miner) + lrepo, err = repo.Lock() assert.NoError(t, err, "should be able to relock") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") diff --git a/node/modules/dtypes/api.go b/types/api.go similarity index 90% rename from node/modules/dtypes/api.go rename to types/api.go index d57b05c..09ee68c 100644 --- a/node/modules/dtypes/api.go +++ b/types/api.go @@ -1,4 +1,4 @@ -package dtypes +package types import ( "github.com/gbrlsnchs/jwt/v3" diff --git a/node/modules/dtypes/miner.go b/types/miner.go similarity index 98% rename from node/modules/dtypes/miner.go rename to types/miner.go index 32d818f..01cc462 100644 --- a/node/modules/dtypes/miner.go +++ b/types/miner.go @@ -1,4 +1,4 @@ -package dtypes +package types import ( "time" diff --git a/node/modules/dtypes/shutdown.go b/types/shutdown.go similarity index 92% rename from node/modules/dtypes/shutdown.go rename to types/shutdown.go index d87c212..d8adcb0 100644 --- a/node/modules/dtypes/shutdown.go +++ b/types/shutdown.go @@ -1,4 +1,4 @@ -package dtypes +package types // ShutdownChan is a channel to which you send a value if you intend to shut // down the daemon (or miner), including the node and RPC server. diff --git a/node/modules/dtypes/storage.go b/types/storage.go similarity index 92% rename from node/modules/dtypes/storage.go rename to types/storage.go index 241a2d8..2f78e02 100644 --- a/node/modules/dtypes/storage.go +++ b/types/storage.go @@ -1,4 +1,4 @@ -package dtypes +package types import ( "github.com/ipfs/go-datastore"