Skip to content

Commit

Permalink
refact "cscli explain" (#2835)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mmetc
mmetc authored Feb 12, 2024
1 parent 2853410 commit 5c83695
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 122 deletions.
184 changes: 75 additions & 109 deletions cmd/crowdsec-cli/explain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,33 +16,53 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)

func GetLineCountForFile(filepath string) (int, error) {
func getLineCountForFile(filepath string) (int, error) {
f, err := os.Open(filepath)
if err != nil {
return 0, err
}
defer f.Close()

lc := 0
fs := bufio.NewReader(f)

for {
input, err := fs.ReadBytes('\n')
if len(input) > 1 {
lc++
}

if err != nil && err == io.EOF {
break
}
}

return lc, nil
}

type cliExplain struct{}
type cliExplain struct {
cfg configGetter
flags struct {
logFile string
dsn string
logLine string
logType string
details bool
skipOk bool
onlySuccessfulParsers bool
noClean bool
crowdsec string
labels string
}
}

func NewCLIExplain() *cliExplain {
return &cliExplain{}
func NewCLIExplain(cfg configGetter) *cliExplain {
return &cliExplain{
cfg: cfg,
}
}

func (cli cliExplain) NewCommand() *cobra.Command {
func (cli *cliExplain) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Expand All @@ -57,140 +77,76 @@ tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: cli.run,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()

logFile, err := flags.GetString("file")
if err != nil {
return err
}

dsn, err := flags.GetString("dsn")
if err != nil {
return err
}

logLine, err := flags.GetString("log")
if err != nil {
return err
}

logType, err := flags.GetString("type")
if err != nil {
return err
}

if logLine == "" && logFile == "" && dsn == "" {
printHelp(cmd)
fmt.Println()
return fmt.Errorf("please provide --log, --file or --dsn flag")
}
if logType == "" {
printHelp(cmd)
fmt.Println()
return fmt.Errorf("please provide --type flag")
}
RunE: func(_ *cobra.Command, _ []string) error {
return cli.run()
},
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
fileInfo, _ := os.Stdin.Stat()
if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
if cli.flags.logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return fmt.Errorf("the option -f - is intended to work with pipes")
}

return nil
},
}

flags := cmd.Flags()

flags.StringP("file", "f", "", "Log file to test")
flags.StringP("dsn", "d", "", "DSN to test")
flags.StringP("log", "l", "", "Log line to test")
flags.StringP("type", "t", "", "Type of the acquisition to test")
flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolP("verbose", "v", false, "Display individual changes")
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")
flags.String("crowdsec", "crowdsec", "Path to crowdsec")
flags.Bool("no-clean", false, "Don't clean runtime environment after tests")
flags.StringVarP(&cli.flags.logFile, "file", "f", "", "Log file to test")
flags.StringVarP(&cli.flags.dsn, "dsn", "d", "", "DSN to test")
flags.StringVarP(&cli.flags.logLine, "log", "l", "", "Log line to test")
flags.StringVarP(&cli.flags.logType, "type", "t", "", "Type of the acquisition to test")
flags.StringVar(&cli.flags.labels, "labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolVarP(&cli.flags.details, "verbose", "v", false, "Display individual changes")
flags.BoolVar(&cli.flags.skipOk, "failures", false, "Only show failed lines")
flags.BoolVar(&cli.flags.onlySuccessfulParsers, "only-successful-parsers", false, "Only show successful parsers")
flags.StringVar(&cli.flags.crowdsec, "crowdsec", "crowdsec", "Path to crowdsec")
flags.BoolVar(&cli.flags.noClean, "no-clean", false, "Don't clean runtime environment after tests")

cmd.MarkFlagRequired("type")
cmd.MarkFlagsOneRequired("log", "file", "dsn")

return cmd
}

func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()

logFile, err := flags.GetString("file")
if err != nil {
return err
}
func (cli *cliExplain) run() error {
logFile := cli.flags.logFile
logLine := cli.flags.logLine
logType := cli.flags.logType
dsn := cli.flags.dsn
labels := cli.flags.labels
crowdsec := cli.flags.crowdsec

dsn, err := flags.GetString("dsn")
if err != nil {
return err
}

logLine, err := flags.GetString("log")
if err != nil {
return err
}

logType, err := flags.GetString("type")
if err != nil {
return err
}

opts := dumps.DumpOpts{}

opts.Details, err = flags.GetBool("verbose")
if err != nil {
return err
}

no_clean, err := flags.GetBool("no-clean")
if err != nil {
return err
}

opts.SkipOk, err = flags.GetBool("failures")
if err != nil {
return err
}

opts.ShowNotOkParsers, err = flags.GetBool("only-successful-parsers")
opts.ShowNotOkParsers = !opts.ShowNotOkParsers
if err != nil {
return err
}

crowdsec, err := flags.GetString("crowdsec")
if err != nil {
return err
}

labels, err := flags.GetString("labels")
if err != nil {
return err
opts := dumps.DumpOpts{
Details: cli.flags.details,
SkipOk: cli.flags.skipOk,
ShowNotOkParsers: !cli.flags.onlySuccessfulParsers,
}

var f *os.File

// using empty string fallback to /tmp
dir, err := os.MkdirTemp("", "cscli_explain")
if err != nil {
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %w", err)
}

defer func() {
if no_clean {
if cli.flags.noClean {
return
}

if _, err := os.Stat(dir); !os.IsNotExist(err) {
if err := os.RemoveAll(dir); err != nil {
log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
}
}
}()

// we create a temporary log file if a log line/stdin has been provided
if logLine != "" || logFile == "-" {
tmpFile := filepath.Join(dir, "cscli_test_tmp.log")

f, err = os.Create(tmpFile)
if err != nil {
return err
Expand Down Expand Up @@ -220,6 +176,7 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
}
}

f.Close()
// this is the file that was going to be read by crowdsec anyway
logFile = tmpFile
Expand All @@ -230,15 +187,20 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
}

dsn = fmt.Sprintf("file://%s", absolutePath)
lineCount, err := GetLineCountForFile(absolutePath)

lineCount, err := getLineCountForFile(absolutePath)
if err != nil {
return err
}

log.Debugf("file %s has %d lines", absolutePath, lineCount)

if lineCount == 0 {
return fmt.Errorf("the log file is empty: %s", absolutePath)
}

if lineCount > 100 {
log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
}
Expand All @@ -249,28 +211,32 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
}

cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}

if labels != "" {
log.Debugf("adding labels %s", labels)
cmdArgs = append(cmdArgs, "-label", labels)
}

crowdsecCmd := exec.Command(crowdsec, cmdArgs...)

output, err := crowdsecCmd.CombinedOutput()
if err != nil {
fmt.Println(string(output))
return fmt.Errorf("fail to run crowdsec for test: %v", err)

return fmt.Errorf("fail to run crowdsec for test: %w", err)
}

parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)

parserDump, err := dumps.LoadParserDump(parserDumpFile)
if err != nil {
return fmt.Errorf("unable to load parser dump result: %s", err)
return fmt.Errorf("unable to load parser dump result: %w", err)
}

bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
if err != nil {
return fmt.Errorf("unable to load bucket dump result: %s", err)
return fmt.Errorf("unable to load bucket dump result: %w", err)
}

dumps.DumpTree(*parserDump, *bucketStateDump, opts)
Expand Down
2 changes: 1 addition & 1 deletion cmd/crowdsec-cli/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
cmd.AddCommand(NewCLILapi(cli.cfg).NewCommand())
cmd.AddCommand(NewCompletionCmd())
cmd.AddCommand(NewConsoleCmd())
cmd.AddCommand(NewCLIExplain().NewCommand())
cmd.AddCommand(NewCLIExplain(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIHubTest().NewCommand())
cmd.AddCommand(NewCLINotifications().NewCommand())
cmd.AddCommand(NewCLISupport().NewCommand())
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ require (
github.com/shirou/gopsutil/v3 v3.23.5
github.com/sirupsen/logrus v1.9.3
github.com/slack-go/slack v0.12.2
github.com/spf13/cobra v1.7.0
github.com/spf13/cobra v1.8.0
github.com/stretchr/testify v1.8.4
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
github.com/wasilibs/go-re2 v1.3.0
Expand Down Expand Up @@ -108,7 +108,7 @@ require (
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/corazawaf/libinjection-go v0.1.2 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/creack/pty v1.1.18 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/go-units v0.5.0 // indirect
Expand Down
14 changes: 4 additions & 10 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,21 +91,17 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f h1:FkOB9aDw0xzDd14pTarGRLsUNAymONq3dc7zhvsXElg=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f/go.mod h1:TrU7Li+z2RHNrPy0TKJ6R65V6Yzpan2sTIRryJJyJso=
github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h8clMcRL2u5ooZ3tmwnmJftmhb9Ws1MKmavvI=
github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/go-cs-lib v0.0.6 h1:Ef6MylXe0GaJE9vrfvxEdbHb31+JUP1os+murPz7Pos=
github.com/crowdsecurity/go-cs-lib v0.0.6/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
Expand Down Expand Up @@ -640,8 +636,8 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
Expand Down Expand Up @@ -809,8 +805,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down

0 comments on commit 5c83695

Please sign in to comment.