Skip to content
This repository has been archived by the owner on Apr 15, 2024. It is now read-only.

feat: support json output for query signers command #338

Merged
merged 9 commits into from
May 2, 2023
Merged

feat: support json output for query signers command #338

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
275f4dd
feat: support json output for query signers command
rach-id Apr 29, 2023
b174b36
chore: lint
rach-id Apr 29, 2023
9531b71
chore: lint
rach-id Apr 29, 2023
e75850a
chore: use logger instead of println
rach-id May 1, 2023
0cad2db
chore: better printing
rach-id May 2, 2023
aebc2d7
chore: remove unused function
rach-id May 2, 2023
fdb2969
chore: add done message to printing
rach-id May 2, 2023
a8abfb2
chore: better logging underscore
rach-id May 2, 2023
60e2419
fix: use the correct signature
rach-id May 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 88 additions & 16 deletions cmd/qgb/query/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package query

import (
"context"
"encoding/json"
"fmt"
"math/big"
"os"
Expand Down Expand Up @@ -119,7 +120,7 @@ func Signers() *cobra.Command {
return fmt.Errorf("nonce 1 doesn't need to be signed. signatures start from nonce 2")
}

err = getSignaturesAndPrintThem(ctx, logger, appQuerier, tmQuerier, p2pQuerier, nonce)
err = getSignaturesAndPrintThem(ctx, logger, appQuerier, tmQuerier, p2pQuerier, nonce, config.outputFile)
if err != nil {
return err
}
Expand All @@ -129,7 +130,15 @@ func Signers() *cobra.Command {
return addFlags(command)
}

func getSignaturesAndPrintThem(ctx context.Context, logger tmlog.Logger, appQuerier *rpc.AppQuerier, tmQuerier *rpc.TmQuerier, p2pQuerier *p2p.Querier, nonce uint64) error {
func getSignaturesAndPrintThem(
ctx context.Context,
logger tmlog.Logger,
appQuerier *rpc.AppQuerier,
tmQuerier *rpc.TmQuerier,
p2pQuerier *p2p.Querier,
nonce uint64,
outputFile string,
) error {
logger.Info("getting signatures for nonce", "nonce", nonce)

lastValset, err := appQuerier.QueryLastValsetBeforeNonce(ctx, nonce)
Expand Down Expand Up @@ -163,7 +172,14 @@ func getSignaturesAndPrintThem(ctx context.Context, logger tmlog.Logger, appQuer
for _, confirm := range confirms {
confirmsMap[confirm.EthAddress] = confirm.Signature
}
printConfirms(logger, confirmsMap, lastValset)
if outputFile == "" {
printConfirms(logger, confirmsMap, lastValset)
} else {
err := writeConfirmsToJSONFile(logger, confirmsMap, lastValset, outputFile)
if err != nil {
return err
}
}
case celestiatypes.DataCommitmentRequestType:
dc, ok := att.(*celestiatypes.DataCommitment)
if !ok {
Expand All @@ -186,7 +202,14 @@ func getSignaturesAndPrintThem(ctx context.Context, logger tmlog.Logger, appQuer
for _, confirm := range confirms {
confirmsMap[confirm.EthAddress] = confirm.Signature
}
printConfirms(logger, confirmsMap, lastValset)
if outputFile == "" {
printConfirms(logger, confirmsMap, lastValset)
} else {
err := writeConfirmsToJSONFile(logger, confirmsMap, lastValset, outputFile)
if err != nil {
return err
}
}
default:
return errors.Wrap(types.ErrUnknownAttestationType, strconv.FormatUint(nonce, 10))
}
Expand All @@ -203,18 +226,7 @@ func parseNonce(ctx context.Context, querier *rpc.AppQuerier, nonce string) (uin
}

func printConfirms(logger tmlog.Logger, confirmsMap map[string]string, valset *celestiatypes.Valset) {
signers := make(map[string]string)
missingSigners := make([]string, 0)

for _, validator := range valset.Members {
val, ok := confirmsMap[validator.EvmAddress]
if ok {
signers[validator.EvmAddress] = val
continue
}
missingSigners = append(missingSigners, validator.EvmAddress)
}

signers, missingSigners := getSignersAndMissingSigners(confirmsMap, valset)
logger.Info("orchestrators that signed the attestation", "count", len(signers))
i := 0
for addr, sig := range signers {
Expand All @@ -227,3 +239,63 @@ func printConfirms(logger tmlog.Logger, confirmsMap map[string]string, valset *c
logger.Info(addr, "number", i)
}
}

func writeConfirmsToJSONFile(logger tmlog.Logger, confirmsMap map[string]string, valset *celestiatypes.Valset, outputFile string) error {
signers, missingSigners := getSignersAndMissingSigners(confirmsMap, valset)
logger.Info("writing confirms json file", "path", outputFile)

file, err := os.Create(outputFile)
if err != nil {
return errors.Wrap(err, "error creating file")
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[blocking question]
this is likely fine for most cases, but does this require one file per query? should we instead use something like

file, err := os.OpenFile(logFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
if err != nil {
	return err
}

this way, we can continually append to a single file

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense to me, @mojtaba-esk what do you think? Which would make your life easier?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the above suggestion will work both ways I believe, where if we specify a different file name with each query, it will create different files. If you use the file name, only then will it append.

Its likely that the standard library is doing something similar under the hood of os.Create, but this is more explicit and safer

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm thinking about this, and i wonder if the output file, if we keep appending to it, will it be a correct json output?

{
   // first output
}
{
   // second output
}

Do we care about that?
@evan-forbes

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Example output from updated implementation:

{
  "signatures": [
    {
      "evmAddress": "0x3EE99606625E740D8b29C8570d855Eb387F3c790",
      "signature": "",
      "signed": false
    },
    {
      "evmAddress": "0x3d22f0C38251ebdBE92e14BBF1bd2067F1C3b7D7",
      "signature": "fdf4bbb3cf7a709c507cbc61b50db49237f5e2d74989da9681a0b117e73568692d417965e86cdaf43968709aeb0f192089c1ef8f8fee7fa65f4f31b21ec2c2af01",
      "signed": true
    },
    {
      "evmAddress": "0x91DEd26b5f38B065FC0204c7929Da1b2A21877Ad",
      "signature": "d82b4b7d096a17eef02b650f658ed4e474464f42884bd6db9995e487e6596a5073f39a817cd0ee10bfaf4e9b6d4c558252e6747e8c52173797bb46f5f75b57fa00",
      "signed": true
    },
    {
      "evmAddress": "0x966e6f22781EF6a6A82BBB4DB3df8E225DfD9488",
      "signature": "7362e738b918cc3a899ca6b4c55ec94a1e33f03f90f6e1b452a89c527aa32d8a61de6dac396e51c58425c927abb5ced6e9a71a925e519b490e5b515f6396a6c201",
      "signed": true
    }
  ],
  "nonce": 11,
  "majority_threshold": 2863311532,
  "current_threshold": 3221225472,
  "can_relay": true
}
{
  "signatures": [
    {
      "evmAddress": "0x966e6f22781EF6a6A82BBB4DB3df8E225DfD9488",
      "signature": "e279a29f9ffb16fb814a828fbd06a75f5de31edb96c138995784a162cd61bb13167914babfb12e03609f8e27530ffeb7ae7e2b090d35234b122673971f4aca5501",
      "signed": true
    }
  ],
  "nonce": 2,
  "majority_threshold": 2863311532,
  "current_threshold": 4294967296,
  "can_relay": true
}

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @sweexordious This output makes it easy to process with the knack-updater tool.
Just a question: we need to run the command once to get all the evm addresses and their signing status up to the time right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Cool 👍

we need to run the command once to get all the evm addresses and their signing status up to the time right?

You need to run it for different nonces. The way the QGB works is via having multiple attestations, referenced by their nonce, signed by the orchestrators (which are run by the validators).
Thus, we need to check each nonce apart to know who signed it and who didn't

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

0cad2db

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm thinking about this, and i wonder if the output file, if we keep appending to it, will it be a correct json output?

yeah good point, it is purely dependant on what @mojtaba-esk is okay with atm, but in the future we might want to add some delimiter to make it easier to parse. either that or use something simlar to json lines.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The delimiter is already \n, so we should be good

defer func(file *os.File) {
err := file.Close()
if err != nil {
fmt.Println(err)
evan-forbes marked this conversation as resolved.
Show resolved Hide resolved
}
}(file)

type output struct {
EvmAddress string `json:"evmAddress"`
Signature string `json:"signature"`
Signed bool `json:"signed"`
}
jsonOutput := make([]output, 0)
for key, val := range signers {
jsonOutput = append(jsonOutput, output{
Signed: true,
EvmAddress: key,
Signature: val,
})
}
for _, val := range missingSigners {
jsonOutput = append(jsonOutput, output{
Signed: false,
EvmAddress: val,
})
}

encoder := json.NewEncoder(file)
err = encoder.Encode(jsonOutput)
if err != nil {
return err
}

logger.Info("output written to file successfully", "path", outputFile)
return nil
}

func getSignersAndMissingSigners(confirmsMap map[string]string, valset *celestiatypes.Valset) (map[string]string, []string) {
signers := make(map[string]string)
missingSigners := make([]string, 0)

for _, validator := range valset.Members {
val, ok := confirmsMap[validator.EvmAddress]
if ok {
signers[validator.EvmAddress] = val
continue
}
missingSigners = append(missingSigners, validator.EvmAddress)
}
return signers, missingSigners
}
12 changes: 11 additions & 1 deletion cmd/qgb/query/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,24 @@ import (
"github.com/spf13/cobra"
)

const FlagP2PNode = "p2p-node"
const (
FlagP2PNode = "p2p-node"
FlagOutputFile = "output-file"
)

func addFlags(cmd *cobra.Command) *cobra.Command {
cmd.Flags().StringP(relayer.FlagCelesGRPC, "c", "localhost:9090", "Specify the grpc address")
cmd.Flags().StringP(relayer.FlagTendermintRPC, "t", "http://localhost:26657", "Specify the rest rpc address")
cmd.Flags().StringP(FlagP2PNode, "n", "", "P2P target node multiaddress (eg. /ip4/127.0.0.1/tcp/30000/p2p/12D3KooWBSMasWzRSRKXREhediFUwABNZwzJbkZcYz5rYr9Zdmfn)")
cmd.Flags().StringP(FlagOutputFile, "o", "", "Path to an output file path if the results need to be written to a json file. Leaving it as empty will result in printing the result to stdout")

return cmd
}

type Config struct {
celesGRPC, tendermintRPC string
targetNode string
outputFile string
}

func parseFlags(cmd *cobra.Command) (Config, error) {
Expand All @@ -33,10 +38,15 @@ func parseFlags(cmd *cobra.Command) (Config, error) {
if err != nil {
return Config{}, err
}
outputFile, err := cmd.Flags().GetString(FlagOutputFile)
if err != nil {
return Config{}, err
}

return Config{
celesGRPC: celesGRPC,
tendermintRPC: tendermintRPC,
targetNode: targetNode,
outputFile: outputFile,
}, nil
}