From b03c4bc26b68c8e23a33694953699ea770f5ff71 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 28 Nov 2020 17:29:15 +0800 Subject: [PATCH 1/8] add bitfield hex encode --- cmd/lotus-shed/bitfield.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 442cbef4846..e377dba1e2c 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -17,6 +17,7 @@ import ( var bitFieldCmd = &cli.Command{ Name: "bitfield", + Usage: "Analyze tool", Description: "analyze bitfields", Flags: []cli.Flag{ &cli.StringFlag{ @@ -305,15 +306,9 @@ var bitFieldSubCmd = &cli.Command{ var bitFieldEncodeCmd = &cli.Command{ Name: "encode", + Usage: "Decimal number to bitfield", Description: "encode a series of decimal numbers into a bitfield", ArgsUsage: "[infile]", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "enc", - Value: "base64", - Usage: "specify input encoding to parse", - }, - }, Action: func(cctx *cli.Context) error { f, err := os.Open(cctx.Args().First()) if err != nil { @@ -340,8 +335,14 @@ var bitFieldEncodeCmd = &cli.Command{ if err != nil { return err } - - fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + switch cctx.String("enc") { + case "base64": + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + case "hex": + fmt.Println(hex.EncodeToString(bytes)) + default: + return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + } return nil }, From 055795c4cc925f3c061ca471262da34acdada5f9 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 28 Nov 2020 19:14:48 +0800 Subject: [PATCH 2/8] optimize bitfield decode --- cmd/lotus-shed/bitfield.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index e377dba1e2c..be41d85afa0 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -196,14 +196,8 @@ var bitFieldStatCmd = &cli.Command{ var bitFieldDecodeCmd = &cli.Command{ Name: "decode", + Usage: "Bitfield to decimal number", Description: "decode bitfield and print all numbers in it", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "enc", - Value: "base64", - Usage: "specify input encoding to parse", - }, - }, Action: func(cctx *cli.Context) error { rle, err := decode(cctx, 0) if err != nil { @@ -348,16 +342,16 @@ var bitFieldEncodeCmd = &cli.Command{ }, } -func decode(cctx *cli.Context, a int) (bitfield.BitField, error) { +func decode(cctx *cli.Context, i int) (bitfield.BitField, error) { var val string if cctx.Args().Present() { - if a >= cctx.NArg() { - return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + if i >= cctx.NArg() { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", i) } - val = cctx.Args().Get(a) + val = cctx.Args().Get(i) } else { - if a > 0 { - return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + if i > 0 { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", i) } b, err := ioutil.ReadAll(os.Stdin) if err != nil { From 8007c1f9ae7027b42c821e95c6ab496190486ef7 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 28 Nov 2020 20:03:26 +0800 Subject: [PATCH 3/8] feat bitfield stat --- cmd/lotus-shed/bitfield.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index be41d85afa0..c76a3f8501d 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -110,14 +110,8 @@ var bitFieldRunsCmd = &cli.Command{ var bitFieldStatCmd = &cli.Command{ Name: "stat", + Usage: "Bitfield stats", Description: "print bitfield stats", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "enc", - Value: "base64", - Usage: "specify input encoding to parse", - }, - }, Action: func(cctx *cli.Context) error { var val string if cctx.Args().Present() { @@ -147,6 +141,7 @@ var bitFieldStatCmd = &cli.Command{ default: return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) } + fmt.Printf("Raw length: %d bits (%d bytes)\n", len(dec)*8, len(dec)) rle, err := rlepluslazy.FromBuf(dec) if err != nil { @@ -158,10 +153,7 @@ var bitFieldStatCmd = &cli.Command{ return xerrors.Errorf("getting run iterator: %w", err) } - fmt.Printf("Raw length: %d bits (%d bytes)\n", len(dec)*8, len(dec)) - var ones, zeros, oneRuns, zeroRuns, invalid uint64 - for rit.HasNext() { r, err := rit.NextRun() if err != nil { From 21835c785bd2b241d7048f715519f2c14a7b086a Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 28 Nov 2020 20:17:56 +0800 Subject: [PATCH 4/8] optimize bitfield runs --- cmd/lotus-shed/bitfield.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index c76a3f8501d..7f2d4c160be 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -38,14 +38,8 @@ var bitFieldCmd = &cli.Command{ var bitFieldRunsCmd = &cli.Command{ Name: "runs", + Usage: "Bitfield bit runs", Description: "print bit runs in a bitfield", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "enc", - Value: "base64", - Usage: "specify input encoding to parse", - }, - }, Action: func(cctx *cli.Context) error { var val string if cctx.Args().Present() { @@ -99,7 +93,7 @@ var bitFieldRunsCmd = &cli.Command{ s = "FALSE" } - fmt.Printf("@%d %s * %d\n", idx, s, r.Len) + fmt.Printf("@%08d %s * %d\n", idx, s, r.Len) idx += r.Len } From c016e34dbce3b67de6dfd4d36918b3105dcb13dd Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 28 Nov 2020 20:56:16 +0800 Subject: [PATCH 5/8] bitfield add encode --- cmd/lotus-shed/bitfield.go | 41 +++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 7f2d4c160be..b7949016b1b 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -306,28 +306,41 @@ var bitFieldEncodeCmd = &cli.Command{ out.Set(i) } - s, err := out.RunIterator() + str, err := encode(cctx, out) if err != nil { return err } - - bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) - if err != nil { - return err - } - switch cctx.String("enc") { - case "base64": - fmt.Println(base64.StdEncoding.EncodeToString(bytes)) - case "hex": - fmt.Println(hex.EncodeToString(bytes)) - default: - return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) - } + fmt.Println(str) return nil }, } +func encode(cctx *cli.Context, field bitfield.BitField) (string, error) { + s, err := field.RunIterator() + if err != nil { + return "", err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return "", err + } + + var str string + switch cctx.String("enc") { + case "base64": + str = base64.StdEncoding.EncodeToString(bytes) + case "hex": + str = hex.EncodeToString(bytes) + default: + return "", fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + } + + return str, nil + +} + func decode(cctx *cli.Context, i int) (bitfield.BitField, error) { var val string if cctx.Args().Present() { From f4adf03ce5ab2709a3cd0eb39754660219df0fe6 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 28 Nov 2020 21:07:14 +0800 Subject: [PATCH 6/8] optimize bitfield intersect --- cmd/lotus-shed/bitfield.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index b7949016b1b..418c0b58c08 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -202,21 +202,15 @@ var bitFieldDecodeCmd = &cli.Command{ var bitFieldIntersectCmd = &cli.Command{ Name: "intersect", + Usage: "Two bitfields intersect", Description: "intersect 2 bitfields and print the resulting bitfield as base64", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "enc", - Value: "base64", - Usage: "specify input encoding to parse", - }, - }, Action: func(cctx *cli.Context) error { - b, err := decode(cctx, 1) + a, err := decode(cctx, 0) if err != nil { return err } - a, err := decode(cctx, 0) + b, err := decode(cctx, 1) if err != nil { return err } @@ -226,17 +220,11 @@ var bitFieldIntersectCmd = &cli.Command{ return xerrors.Errorf("intersect: %w", err) } - s, err := o.RunIterator() - if err != nil { - return err - } - - bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + str, err := encode(cctx, o) if err != nil { return err } - - fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + fmt.Println(str) return nil }, From a7ed639c218be1cdf1ca185f1fa0a08dc50f9a37 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 28 Nov 2020 21:44:12 +0800 Subject: [PATCH 7/8] optimize bitfield use func --- cmd/lotus-shed/bitfield.go | 121 +++++++++++-------------------------- 1 file changed, 34 insertions(+), 87 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 418c0b58c08..15f1e544324 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -17,7 +17,7 @@ import ( var bitFieldCmd = &cli.Command{ Name: "bitfield", - Usage: "Analyze tool", + Usage: "Bitfield analyze tool", Description: "analyze bitfields", Flags: []cli.Flag{ &cli.StringFlag{ @@ -29,9 +29,9 @@ var bitFieldCmd = &cli.Command{ Subcommands: []*cli.Command{ bitFieldRunsCmd, bitFieldStatCmd, + bitFieldEncodeCmd, bitFieldDecodeCmd, bitFieldIntersectCmd, - bitFieldEncodeCmd, bitFieldSubCmd, }, } @@ -41,33 +41,9 @@ var bitFieldRunsCmd = &cli.Command{ Usage: "Bitfield bit runs", Description: "print bit runs in a bitfield", Action: func(cctx *cli.Context) error { - var val string - if cctx.Args().Present() { - val = cctx.Args().Get(0) - } else { - b, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return err - } - val = string(b) - } - - var dec []byte - switch cctx.String("enc") { - case "base64": - d, err := base64.StdEncoding.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding base64 value: %w", err) - } - dec = d - case "hex": - d, err := hex.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding hex value: %w", err) - } - dec = d - default: - return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + dec, err := decodeToByte(cctx, 0) + if err != nil { + return err } rle, err := rlepluslazy.FromBuf(dec) @@ -107,33 +83,9 @@ var bitFieldStatCmd = &cli.Command{ Usage: "Bitfield stats", Description: "print bitfield stats", Action: func(cctx *cli.Context) error { - var val string - if cctx.Args().Present() { - val = cctx.Args().Get(0) - } else { - b, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return err - } - val = string(b) - } - - var dec []byte - switch cctx.String("enc") { - case "base64": - d, err := base64.StdEncoding.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding base64 value: %w", err) - } - dec = d - case "hex": - d, err := hex.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding hex value: %w", err) - } - dec = d - default: - return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + dec, err := decodeToByte(cctx, 0) + if err != nil { + return err } fmt.Printf("Raw length: %d bits (%d bytes)\n", len(dec)*8, len(dec)) @@ -202,8 +154,8 @@ var bitFieldDecodeCmd = &cli.Command{ var bitFieldIntersectCmd = &cli.Command{ Name: "intersect", - Usage: "Two bitfields intersect", - Description: "intersect 2 bitfields and print the resulting bitfield as base64", + Usage: "Intersect 2 bitfields", + Description: "intersect 2 bitfields and print the resulting bitfield", Action: func(cctx *cli.Context) error { a, err := decode(cctx, 0) if err != nil { @@ -232,41 +184,29 @@ var bitFieldIntersectCmd = &cli.Command{ var bitFieldSubCmd = &cli.Command{ Name: "sub", - Description: "subtract 2 bitfields and print the resulting bitfield as base64", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "enc", - Value: "base64", - Usage: "specify input encoding to parse", - }, - }, + Usage: "Subtract 2 bitfields", + Description: "subtract 2 bitfields and print the resulting bitfield", Action: func(cctx *cli.Context) error { - b, err := decode(cctx, 1) + a, err := decode(cctx, 0) if err != nil { return err } - a, err := decode(cctx, 0) + b, err := decode(cctx, 1) if err != nil { return err } o, err := bitfield.SubtractBitField(a, b) if err != nil { - return xerrors.Errorf("intersect: %w", err) + return xerrors.Errorf("subtract: %w", err) } - s, err := o.RunIterator() - if err != nil { - return err - } - - bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + str, err := encode(cctx, o) if err != nil { return err } - - fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + fmt.Println(str) return nil }, @@ -328,23 +268,30 @@ func encode(cctx *cli.Context, field bitfield.BitField) (string, error) { return str, nil } - func decode(cctx *cli.Context, i int) (bitfield.BitField, error) { + b, err := decodeToByte(cctx, i) + if err != nil { + return bitfield.BitField{}, err + } + return bitfield.NewFromBytes(b) +} + +func decodeToByte(cctx *cli.Context, i int) ([]byte, error) { var val string if cctx.Args().Present() { if i >= cctx.NArg() { - return bitfield.BitField{}, xerrors.Errorf("need more than %d args", i) + return nil, xerrors.Errorf("need more than %d args", i) } val = cctx.Args().Get(i) } else { if i > 0 { - return bitfield.BitField{}, xerrors.Errorf("need more than %d args", i) + return nil, xerrors.Errorf("need more than %d args", i) } - b, err := ioutil.ReadAll(os.Stdin) + r, err := ioutil.ReadAll(os.Stdin) if err != nil { - return bitfield.BitField{}, err + return nil, err } - val = string(b) + val = string(r) } var dec []byte @@ -352,18 +299,18 @@ func decode(cctx *cli.Context, i int) (bitfield.BitField, error) { case "base64": d, err := base64.StdEncoding.DecodeString(val) if err != nil { - return bitfield.BitField{}, fmt.Errorf("decoding base64 value: %w", err) + return nil, fmt.Errorf("decoding base64 value: %w", err) } dec = d case "hex": d, err := hex.DecodeString(val) if err != nil { - return bitfield.BitField{}, fmt.Errorf("decoding hex value: %w", err) + return nil, fmt.Errorf("decoding hex value: %w", err) } dec = d default: - return bitfield.BitField{}, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + return nil, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) } - return bitfield.NewFromBytes(dec) + return dec, nil } From 253e22d6eabe53e9840073493290e65b7a27d4c7 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Sat, 28 Nov 2020 21:54:30 +0800 Subject: [PATCH 8/8] add bitfield merge func --- cmd/lotus-shed/bitfield.go | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 15f1e544324..f0824de4f2a 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -27,10 +27,11 @@ var bitFieldCmd = &cli.Command{ }, }, Subcommands: []*cli.Command{ - bitFieldRunsCmd, - bitFieldStatCmd, bitFieldEncodeCmd, bitFieldDecodeCmd, + bitFieldRunsCmd, + bitFieldStatCmd, + bitFieldMergeCmd, bitFieldIntersectCmd, bitFieldSubCmd, }, @@ -152,6 +153,36 @@ var bitFieldDecodeCmd = &cli.Command{ }, } +var bitFieldMergeCmd = &cli.Command{ + Name: "merge", + Usage: "Merge 2 bitfields", + Description: "Merge 2 bitfields and print the resulting bitfield", + Action: func(cctx *cli.Context) error { + a, err := decode(cctx, 0) + if err != nil { + return err + } + + b, err := decode(cctx, 1) + if err != nil { + return err + } + + o, err := bitfield.MergeBitFields(a, b) + if err != nil { + return xerrors.Errorf("merge: %w", err) + } + + str, err := encode(cctx, o) + if err != nil { + return err + } + fmt.Println(str) + + return nil + }, +} + var bitFieldIntersectCmd = &cli.Command{ Name: "intersect", Usage: "Intersect 2 bitfields",