Skip to content

Commit

Permalink
lotus-pcr: find miners command
Browse files Browse the repository at this point in the history
  • Loading branch information
@travisperson
travisperson committed Sep 14, 2020
1 parent 96ae7c1 commit 7b39a6e
Showing 1 changed file with 223 additions and 27 deletions.
250 changes: 223 additions & 27 deletions cmd/lotus-pcr/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package main

import (
"bufio"
"bytes"
"context"
"encoding/csv"
"fmt"
"io/ioutil"
"math"
"net/http"
_ "net/http/pprof"
"os"
Expand All @@ -20,13 +23,14 @@ import (

"golang.org/x/xerrors"

"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/miner"

"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
Expand All @@ -39,6 +43,7 @@ func main() {
local := []*cli.Command{
runCmd,
recoverMinersCmd,
findMinersCmd,
versionCmd,
}

Expand Down Expand Up @@ -102,6 +107,90 @@ var versionCmd = &cli.Command{
},
}

var findMinersCmd = &cli.Command{
Name: "find-miners",
Usage: "find miners with a desired minimum balance",
Description: `Find miners returns a list of miners and their balances that are below a
threhold value. By default only the miner actor available balance is considered but other
account balances can be included by enabling them through the flags.
Examples
Return all miners with an available balance below 100 FIL
lotus-pcr find-miners --threshold 100
Find all miners with a balance below zero, which includes the owner and worker
lotus-pcr find-miners --threshold 0 --owner --worker
`,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-sync",
EnvVars: []string{"LOTUS_PCR_NO_SYNC"},
Usage: "do not wait for chain sync to complete",
},
&cli.IntFlag{
Name: "threshold",
EnvVars: []string{"LOTUS_PCR_THRESHOLD"},
Usage: "balance below this limit will be printed",
Value: 0,
},
&cli.BoolFlag{
Name: "owner",
Usage: "include owner balance",
Value: false,
},
&cli.BoolFlag{
Name: "worker",
Usage: "include worker balance",
Value: false,
},
&cli.BoolFlag{
Name: "control",
Usage: "include control balance",
Value: false,
},
},
Action: func(cctx *cli.Context) error {
ctx := context.Background()
api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path"))
if err != nil {
log.Fatal(err)
}
defer closer()

if !cctx.Bool("no-sync") {
if err := stats.WaitForSyncComplete(ctx, api); err != nil {
log.Fatal(err)
}
}

owner := cctx.Bool("owner")
worker := cctx.Bool("worker")
control := cctx.Bool("control")
threshold := uint64(cctx.Int("threshold"))

rf := &refunder{
api: api,
threshold: types.FromFil(threshold),
}

refundTipset, err := api.ChainHead(ctx)
if err != nil {
return err
}

balanceRefund, err := rf.FindMiners(ctx, refundTipset, NewMinersRefund(), owner, worker, control)
if err != nil {
return err
}

for _, maddr := range balanceRefund.Miners() {
fmt.Printf("%s\t%s\n", maddr, types.FIL(balanceRefund.GetRefund(maddr)))
}

return nil
},
}

var recoverMinersCmd = &cli.Command{
Name: "recover-miners",
Usage: "Ensure all miners with a negative available balance have a FIL surplus across accounts",
Expand All @@ -123,11 +212,15 @@ var recoverMinersCmd = &cli.Command{
Value: false,
},
&cli.IntFlag{
Name: "miner-total-funds-threashold",
EnvVars: []string{"LOTUS_PCR_MINER_TOTAL_FUNDS_THREASHOLD"},
Usage: "total filecoin across all accounts that should be met, if the miner balancer drops below zero",
Name: "threshold",
EnvVars: []string{"LOTUS_PCR_THRESHOLD"},
Usage: "total filecoin across all accounts that should be met, if the miner balance drops below zero",
Value: 0,
},
&cli.StringFlag{
Name: "output",
Usage: "dump data as a csv format to this file",
},
},
Action: func(cctx *cli.Context) error {
ctx := context.Background()
Expand All @@ -149,26 +242,26 @@ var recoverMinersCmd = &cli.Command{
}

dryRun := cctx.Bool("dry-run")
minerTotalFundsThreashold := uint64(cctx.Int("miner-total-funds-threashold"))
threshold := uint64(cctx.Int("threshold"))

rf := &refunder{
api: api,
wallet: from,
dryRun: dryRun,
minerTotalFundsThreashold: types.FromFil(minerTotalFundsThreashold),
api: api,
wallet: from,
dryRun: dryRun,
threshold: types.FromFil(threshold),
}

refundTipset, err := api.ChainHead(ctx)
if err != nil {
return err
}

negativeBalancerRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund())
balanceRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund(), cctx.String("output"))
if err != nil {
return err
}

if err := rf.Refund(ctx, "refund negative balancer miner", refundTipset, negativeBalancerRefund, 0); err != nil {
if err := rf.Refund(ctx, "refund to recover miner", refundTipset, balanceRefund, 0); err != nil {
return err
}

Expand Down Expand Up @@ -397,6 +490,8 @@ type refunderNodeApi interface {
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error)
StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error)
StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error)
StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error)
StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error)
StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error)
MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error)
Expand All @@ -405,16 +500,16 @@ type refunderNodeApi interface {
}

type refunder struct {
api refunderNodeApi
wallet address.Address
percentExtra int
dryRun bool
preCommitEnabled bool
proveCommitEnabled bool
minerTotalFundsThreashold big.Int
api refunderNodeApi
wallet address.Address
percentExtra int
dryRun bool
preCommitEnabled bool
proveCommitEnabled bool
threshold big.Int
}

func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) {
func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) {
miners, err := r.api.StateListMiners(ctx, tipset.Key())
if err != nil {
return nil, err
Expand All @@ -437,8 +532,91 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet
continue
}

if minerAvailableBalance.GreaterThanEqual(big.Zero()) {
log.Debugw("skipping over miner with positive balance", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
// Look up and find all addresses associated with the miner
minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key())

allAddresses := []address.Address{}

if worker {
allAddresses = append(allAddresses, minerInfo.Worker)
}

if owner {
allAddresses = append(allAddresses, minerInfo.Owner)
}

if control {
allAddresses = append(allAddresses, minerInfo.ControlAddresses...)
}

// Sum the balancer of all the addresses
addrSum := big.Zero()
addrCheck := make(map[address.Address]struct{}, len(allAddresses))
for _, addr := range allAddresses {
if _, found := addrCheck[addr]; !found {
balance, err := r.api.WalletBalance(ctx, addr)
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}

addrSum = big.Add(addrSum, balance)
addrCheck[addr] = struct{}{}
}
}

totalAvailableBalance := big.Add(addrSum, minerAvailableBalance)

if totalAvailableBalance.GreaterThanEqual(r.threshold) {
continue
}

refunds.Track(maddr, totalAvailableBalance)

log.Debugw("processing miner", "miner", maddr, "sectors", "available_balance", totalAvailableBalance)
}

return refunds, nil
}

func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, output string) (*MinersRefund, error) {
miners, err := r.api.StateListMiners(ctx, tipset.Key())
if err != nil {
return nil, err
}

w := ioutil.Discard
if len(output) != 0 {
f, err := os.Create(output)
if err != nil {
return nil, err
}

defer f.Close()

w = bufio.NewWriter(f)
}

csvOut := csv.NewWriter(w)
defer csvOut.Flush()
if err := csvOut.Write([]string{"MinerID", "Sectors", "CombinedBalance", "ProposedSend"}); err != nil {
return nil, err
}

for _, maddr := range miners {
mact, err := r.api.StateGetActor(ctx, maddr, types.EmptyTSK)
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}

if !mact.Balance.GreaterThan(big.Zero()) {
continue
}

minerAvailableBalance, err := r.api.StateMinerAvailableBalance(ctx, maddr, tipset.Key())
if err != nil {
log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}

Expand All @@ -464,19 +642,37 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet
}
}

sectorInfo, err := r.api.StateMinerSectors(ctx, maddr, nil, false, tipset.Key())
if err != nil {
log.Errorw("failed to look up miner sectors", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}

if len(sectorInfo) == 0 {
log.Debugw("skipping miner with zero sectors", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr)
continue
}

totalAvailableBalance := big.Add(addrSum, minerAvailableBalance)

// If the miner has available balance they should use it
if totalAvailableBalance.GreaterThanEqual(r.minerTotalFundsThreashold) {
log.Debugw("skipping over miner with enough funds cross all accounts", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "miner_available_balance", minerAvailableBalance, "wallet_total_balances", addrSum)
numSectorInfo := float64(len(sectorInfo))
filAmount := uint64(math.Ceil(math.Max(math.Log(numSectorInfo)*math.Sqrt(numSectorInfo)/4, 20)))
attoFilAmount := big.Mul(big.NewIntUnsigned(filAmount), big.NewIntUnsigned(build.FilecoinPrecision))

if totalAvailableBalance.GreaterThanEqual(attoFilAmount) {
log.Debugw("skipping over miner with total available balance larger than refund", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "available_balance", totalAvailableBalance, "possible_refund", attoFilAmount)
continue
}

// Calculate the required FIL to bring the miner up to the minimum across all of their accounts
refundValue := big.Add(totalAvailableBalance.Abs(), r.minerTotalFundsThreashold)
refundValue := big.Sub(attoFilAmount, totalAvailableBalance)

refunds.Track(maddr, refundValue)
record := []string{maddr.String(), fmt.Sprintf("%d", len(sectorInfo)), totalAvailableBalance.String(), refundValue.String()}
if err := csvOut.Write(record); err != nil {
return nil, err
}

log.Debugw("processing negative balance miner", "miner", maddr, "refund", refundValue)
log.Debugw("processing miner", "miner", maddr, "sectors", len(sectorInfo), "available_balance", totalAvailableBalance, "refund", refundValue)
}

return refunds, nil
Expand Down

0 comments on commit 7b39a6e

Please sign in to comment.