Dependency updates and tests

.github/workflows/functional_tests.yaml

@@ -11,24 +11,33 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
+
     - name: Set up Go 1.19
       uses: actions/setup-go@v3
       with:
         go-version: 1.19
       id: go
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
-    - id: cache-pipenv
+      with:
+        fetch-depth: 0
+
+    - name: Cache virtualenvs
+      id: cache-pipenv
       uses: actions/cache@v3
       with:
         path: ~/.local/share/virtualenvs
         key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
+
     - name: Install deps for tests
       run: |
         sudo apt install -y nftables iptables ipset
         python3 -m pip install --upgrade pipenv wheel
+        docker network create net-test
+
     - name: Run tests
       run: |
         make build
         pipenv install --deploy
-        pipenv run pytest
+        pipenv run pytest --durations=0 --color=yes

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 crowdsecurity
+Copyright (c) 2020-2023 crowdsecurity
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -20,10 +20,7 @@ export LD_OPTS=-ldflags "-s -w -X github.com/crowdsecurity/cs-custom-bouncer/pkg
 
 RELDIR = "crowdsec-custom-bouncer-${BUILD_VERSION}"
 
-PYTHON=python3
-PIP=pip
-
-all: clean test build
+all: clean build test
 
 static: clean
 	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v -a -tags netgo -ldflags '-w -extldflags "-static"'

Pipfile

@@ -1,8 +1,9 @@
 [packages]
-pytest = "*"
+pytest-dotenv = "*"
 flask = "*"
 pytimeparse = "*"
 psutil = "*"
+pytest-cs = {ref = "main", git = "https://github.com/crowdsecurity/pytest-cs.git"}
 
 [dev-packages]
 gnureadline = "*"

config.go

@@ -22,6 +22,7 @@ type PrometheusConfig struct {
 
 type bouncerConfig struct {
 	BinPath                    string           `yaml:"bin_path"` // path to binary
+	BinArgs                    []string         `yaml:"bin_args"` // arguments for binary
 	PidDir                     string           `yaml:"piddir"`
 	UpdateFrequency            string           `yaml:"update_frequency"`
 	IncludeScenariosContaining []string         `yaml:"include_scenarios_containing"`
@@ -82,7 +83,7 @@ func newConfig(reader io.Reader) (*bouncerConfig, error) {
 
 	/*Configure logging*/
 	if err := types.SetDefaultLoggerConfig(config.LogMode, config.LogDir, config.LogLevel, config.LogMaxSize, config.LogMaxFiles, config.LogMaxAge, config.CompressLogs, false); err != nil {
-		log.Fatal(err.Error())
+		log.Fatal(err)
 	}
 	if config.LogMode == "file" {
 		if config.LogDir == "" {

default.env

@@ -0,0 +1,3 @@
+CROWDSEC_TEST_VERSION="dev"
+CROWDSEC_TEST_FLAVORS="full"
+CROWDSEC_TEST_NETWORK="net-test"

go.mod

@@ -1,18 +1,21 @@
 module github.com/crowdsecurity/cs-custom-bouncer
 
-go 1.19
+go 1.20
 
 require (
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
-	github.com/crowdsecurity/crowdsec v1.4.1
-	github.com/crowdsecurity/go-cs-bouncer v0.0.2
+	github.com/crowdsecurity/crowdsec v1.4.6
+	github.com/crowdsecurity/go-cs-bouncer v0.0.3-0.20230302160125-5325225f5785
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/prometheus/client_golang v1.14.0
 	github.com/sirupsen/logrus v1.9.0
-	golang.org/x/net v0.4.0 // indirect
-	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
+)
+
+require (
+	golang.org/x/sync v0.1.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -44,5 +47,6 @@ require (
 	github.com/prometheus/procfs v0.8.0 // indirect
 	go.mongodb.org/mongo-driver v1.11.1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

main.go

@@ -18,18 +18,17 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"github.com/sirupsen/logrus/hooks/writer"
+	"golang.org/x/sync/errgroup"
 
+	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/cs-custom-bouncer/pkg/version"
 	csbouncer "github.com/crowdsecurity/go-cs-bouncer"
-	"gopkg.in/tomb.v2"
 )
 
 const (
 	name = "crowdsec-custom-bouncer"
 )
 
-var t tomb.Tomb
-
 func termHandler(sig os.Signal, custom *customBouncer) error {
 	if err := custom.ShutDown(); err != nil {
 		return err
@@ -62,6 +61,38 @@ func HandleSignals(custom *customBouncer) {
 	os.Exit(code)
 }
 
+func deleteDecisions(custom *customBouncer, decisions []*models.Decision) {
+	if len(decisions) == 1 {
+		log.Infof("deleting 1 decision")
+	} else {
+		log.Infof("deleting %d decisions", len(decisions))
+	}
+	for _, d := range decisions {
+		if err := custom.Delete(d); err != nil {
+			log.Errorf("unable to delete decision for '%s': %s", *d.Value, err)
+			continue
+		}
+		log.Debugf("deleted '%s'", *d.Value)
+	}
+}
+
+
+func addDecisions(custom *customBouncer, decisions []*models.Decision) {
+	if len(decisions) == 1 {
+		log.Infof("adding 1 decision")
+	} else {
+		log.Infof("adding %d decisions", len(decisions))
+	}
+	for _, d := range decisions {
+		if err := custom.Add(d); err != nil {
+			log.Errorf("unable to insert decision for '%s': %s", *d.Value, err)
+			continue
+		}
+		log.Debugf("Adding '%s' for '%s'", *d.Value, *d.Duration)
+	}
+}
+
+
 func main() {
 	var err error
 	var promServer *http.Server
@@ -104,31 +135,33 @@ func main() {
 
 	custom, err := newCustomBouncer(config)
 	if err != nil {
-		log.Fatalf(err.Error())
+		log.Fatal(err)
 	}
 
 	if err := custom.Init(); err != nil {
-		log.Fatalf(err.Error())
+		log.Fatal(err)
 	}
 
 	bouncer := &csbouncer.StreamBouncer{}
 	bouncer.UserAgent = fmt.Sprintf("%s/%s", name, version.VersionStr())
 
 	err = bouncer.ConfigReader(bytes.NewReader(configBytes))
 	if err != nil {
-		log.Errorf("unable to configure bouncer: %s", err)
-		return
+		log.Fatalf("unable to configure bouncer: %s", err)
 	}
 
 	if err := bouncer.Init(); err != nil {
-		log.Error(err.Error())
-		return
+		log.Fatal(err)
 	}
 	cacheResetTicker := time.NewTicker(config.CacheRetentionDuration)
-	go func() {
-		bouncer.Run()
-		t.Kill(fmt.Errorf("stream init failed"))
-	}()
+
+	g, ctx := errgroup.WithContext(context.Background())
+
+	g.Go(func() error {
+		bouncer.Run(ctx)
+		return fmt.Errorf("stream init failed")
+	})
+
 	if config.PrometheusConfig.Enabled {
 		listenOn := net.JoinHostPort(
 			config.PrometheusConfig.ListenAddress,
@@ -147,49 +180,46 @@ func main() {
 		go func() {
 			log.Infof("Serving metrics at %s", listenOn+"/metrics")
 			log.Error(promServer.ListenAndServe())
+			// don't need to cancel context here, prometheus is not critical
 		}()
 	}
 	if config.FeedViaStdin {
-		t.Go(
-			func() error {
-				f := func() error {
-					c := exec.Command(config.BinPath)
-					s, err := c.StdinPipe()
-					if err != nil {
-						return err
-					}
-					custom.binaryStdin = s
-					if err := c.Start(); err != nil {
-						return err
-					}
-
-					return c.Wait()
+		g.Go(func() error {
+			f := func() error {
+				log.Debugf("Starting binary %s %s", config.BinPath, config.BinArgs)
+				c := exec.CommandContext(ctx, config.BinPath, config.BinArgs...)
+				s, err := c.StdinPipe()
+				if err != nil {
+					return err
 				}
-				var err error
-				if config.TotalRetries == -1 {
-					for {
-						err := f()
-						log.Errorf("Binary exited: %s", err)
-					}
-				} else {
-					for i := 0; i <= config.TotalRetries; i++ {
-						err = f()
-						log.Errorf("Binary exited (retry %d/%d): %s", i, config.TotalRetries, err)
-					}
+				custom.binaryStdin = s
+				if err := c.Start(); err != nil {
+					return err
 				}
-				log.Error("maximum retries exceeded for binary. Exiting")
-				t.Kill(err)
-				return err
-			},
-		)
 
+				return c.Wait()
+			}
+			var err error
+			if config.TotalRetries == -1 {
+				for {
+					err := f()
+					log.Errorf("Binary exited: %s", err)
+				}
+			} else {
+				for i := 1; i <= config.TotalRetries; i++ {
+					err = f()
+					log.Errorf("Binary exited (retry %d/%d): %s", i, config.TotalRetries, err)
+				}
+			}
+			return fmt.Errorf("maximum retries exceeded for binary. Exiting")
+		})
 	}
 
-	t.Go(func() error {
-		log.Printf("Processing new and deleted decisions . . .")
+	g.Go(func() error {
+		log.Infof("Processing new and deleted decisions . . .")
 		for {
 			select {
-			case <-t.Dying():
+			case <-ctx.Done():
 				log.Infoln("terminating bouncer process")
 				if config.PrometheusConfig.Enabled {
 					log.Infoln("terminating prometheus server")
@@ -199,23 +229,11 @@ func main() {
 				}
 				return nil
 			case decisions := <-bouncer.Stream:
-				log.Infof("deleting '%d' decisions", len(decisions.Deleted))
-				for _, decision := range decisions.Deleted {
-					if err := custom.Delete(decision); err != nil {
-						log.Errorf("unable to delete decision for '%s': %s", *decision.Value, err)
-					} else {
-						log.Debugf("deleted '%s'", *decision.Value)
-					}
-
-				}
-				log.Infof("adding '%d' decisions", len(decisions.New))
-				for _, decision := range decisions.New {
-					if err := custom.Add(decision); err != nil {
-						log.Errorf("unable to insert decision for '%s': %s", *decision.Value, err)
-					} else {
-						log.Debugf("Adding '%s' for '%s'", *decision.Value, *decision.Duration)
-					}
+				if decisions == nil {
+					continue
 				}
+				deleteDecisions(custom, decisions.Deleted)
+				addDecisions(custom, decisions.New)
 			case <-cacheResetTicker.C:
 				custom.ResetCache()
 			}
@@ -230,7 +248,7 @@ func main() {
 		go HandleSignals(custom)
 	}
 
-	if err := t.Wait(); err != nil {
-		log.Errorf("process return with error: %s", err)
+	if err := g.Wait(); err != nil {
+		log.Fatal(err)
 	}
 }

pytest-debug.ini

@@ -1,3 +1,6 @@
 [pytest]
 # drop to pdb on first failure
 addopts = --pdb --pdbcls=IPython.terminal.debugger:Pdb
+env_files =
+    .env
+    default.env

pytest.ini

@@ -1,2 +1,5 @@
 [pytest]
 addopts =
+env_files =
+    .env
+    default.env