diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 3276dbb1bfd..44abbbe24a3 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -61,6 +61,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.62 + version: v1.63 args: --issues-exit-code=1 --timeout 10m only-new-issues: false diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index d882f88580e..649c47ebd26 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -189,6 +189,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.62 + version: v1.63 args: --issues-exit-code=1 --timeout 10m only-new-issues: false diff --git a/.golangci.yml b/.golangci.yml index 68420581971..fe77aec2d3c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -223,7 +223,6 @@ linters-settings: - unnamedResult - sloppyReassign - appendCombine - - captLocal - typeUnparen - commentFormatting - deferInLoop # @@ -487,3 +486,14 @@ issues: - revive path: "pkg/types/utils.go" text: "argument-limit: .*" + + # need some cleanup first: to create db in memory and share the client, not the config + - linters: + - usetesting + path: "pkg/apiserver/(.+)_test.go" + text: "os.MkdirTemp.* could be replaced by t.TempDir.*" + + - linters: + - usetesting + path: "pkg/apiserver/(.+)_test.go" + text: "os.CreateTemp.* could be replaced by os.CreateTemp.*" diff --git a/cmd/crowdsec-cli/clidecision/decisions.go b/cmd/crowdsec-cli/clidecision/decisions.go index 307cabffe51..b5865bab6e0 100644 --- a/cmd/crowdsec-cli/clidecision/decisions.go +++ b/cmd/crowdsec-cli/clidecision/decisions.go @@ -170,7 +170,7 @@ func (cli *cliDecisions) NewCommand() *cobra.Command { return cmd } -func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOpts, NoSimu *bool, contained *bool, printMachine bool) error { +func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOpts, noSimu *bool, contained *bool, printMachine bool) error { var err error *filter.ScopeEquals, err = clialert.SanitizeScope(*filter.ScopeEquals, *filter.IPEquals, *filter.RangeEquals) @@ -181,7 +181,7 @@ func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOp filter.ActiveDecisionEquals = new(bool) *filter.ActiveDecisionEquals = true - if NoSimu != nil && *NoSimu { + if noSimu != nil && *noSimu { filter.IncludeSimulated = new(bool) } /* nullify the empty entries to avoid bad filter */ diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index faac786ebdc..0a58a8c1ab3 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -1,234 +1,18 @@ package main import ( - "encoding/json" - "errors" "fmt" - "os" - "path/filepath" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func (cli *cliConfig) backupHub(dirPath string) error { - hub, err := require.Hub(cli.cfg(), nil) - if err != nil { - return err - } - - for _, itemType := range cwhub.ItemTypes { - clog := log.WithField("type", itemType) - - itemMap := hub.GetItemMap(itemType) - if itemMap == nil { - clog.Infof("No %s to backup.", itemType) - continue - } - - itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itemType) - if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil { - return fmt.Errorf("error while creating %s: %w", itemDirectory, err) - } - - upstreamParsers := []string{} - - for k, v := range itemMap { - clog = clog.WithField("file", v.Name) - if !v.State.Installed { // only backup installed ones - clog.Debugf("[%s]: not installed", k) - continue - } - - // for the local/tainted ones, we back up the full file - if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate { - // we need to backup stages for parsers - if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { - fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) - if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil { - return fmt.Errorf("error while creating stage dir %s: %w", fstagedir, err) - } - } - - clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate) - - tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = CopyFile(v.State.LocalPath, tfile); err != nil { - return fmt.Errorf("failed copy %s %s to %s: %w", itemType, v.State.LocalPath, tfile, err) - } - - clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile) - - continue - } - - clog.Debugf("[%s]: from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate) - clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate) - upstreamParsers = append(upstreamParsers, v.Name) - } - // write the upstream items - upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType) - - upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ") - if err != nil { - return fmt.Errorf("failed to serialize upstream parsers: %w", err) - } - - err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0o644) - if err != nil { - return fmt.Errorf("unable to write to %s %s: %w", itemType, upstreamParsersFname, err) - } - - clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname) - } - - return nil -} - -/* - Backup crowdsec configurations to directory : - -- Main config (config.yaml) -- Profiles config (profiles.yaml) -- Simulation config (simulation.yaml) -- Backup of API credentials (local API and online API) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Acquisition files (acquis.yaml, acquis.d/*.yaml) -*/ -func (cli *cliConfig) backup(dirPath string) error { - var err error - - cfg := cli.cfg() - - if dirPath == "" { - return errors.New("directory path can't be empty") - } - - log.Infof("Starting configuration backup") - - /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ - parentDir := filepath.Dir(dirPath) - if _, err = os.Stat(parentDir); err != nil { - return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) - } - - if err = os.Mkdir(dirPath, 0o700); err != nil { - return fmt.Errorf("while creating %s: %w", dirPath, err) - } - - if cfg.ConfigPaths.SimulationFilePath != "" { - backupSimulation := filepath.Join(dirPath, "simulation.yaml") - if err = CopyFile(cfg.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.ConfigPaths.SimulationFilePath, backupSimulation, err) - } - - log.Infof("Saved simulation to %s", backupSimulation) - } - - /* - - backup AcquisitionFilePath - - backup the other files of acquisition directory - */ - if cfg.Crowdsec != nil && cfg.Crowdsec.AcquisitionFilePath != "" { - backupAcquisition := filepath.Join(dirPath, "acquis.yaml") - if err = CopyFile(cfg.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.Crowdsec.AcquisitionFilePath, backupAcquisition, err) - } - } - - acquisBackupDir := filepath.Join(dirPath, "acquis") - if err = os.Mkdir(acquisBackupDir, 0o700); err != nil { - return fmt.Errorf("error while creating %s: %w", acquisBackupDir, err) - } - - if cfg.Crowdsec != nil && len(cfg.Crowdsec.AcquisitionFiles) > 0 { - for _, acquisFile := range cfg.Crowdsec.AcquisitionFiles { - /*if it was the default one, it was already backup'ed*/ - if cfg.Crowdsec.AcquisitionFilePath == acquisFile { - continue - } - - targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) - if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) - } - - if err = CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) - } - - log.Infof("Saved acquis %s to %s", acquisFile, targetFname) - } - } - - if ConfigFilePath != "" { - backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if err = CopyFile(ConfigFilePath, backupMain); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", ConfigFilePath, backupMain, err) - } - - log.Infof("Saved default yaml to %s", backupMain) - } - - if cfg.API != nil && cfg.API.Server != nil && cfg.API.Server.OnlineClient != nil && cfg.API.Server.OnlineClient.CredentialsFilePath != "" { - backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if err = CopyFile(cfg.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) - } - - log.Infof("Saved online API credentials to %s", backupCAPICreds) - } - - if cfg.API != nil && cfg.API.Client != nil && cfg.API.Client.CredentialsFilePath != "" { - backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if err = CopyFile(cfg.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Client.CredentialsFilePath, backupLAPICreds, err) - } - - log.Infof("Saved local API credentials to %s", backupLAPICreds) - } - - if cfg.API != nil && cfg.API.Server != nil && cfg.API.Server.ProfilesPath != "" { - backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if err = CopyFile(cfg.API.Server.ProfilesPath, backupProfiles); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Server.ProfilesPath, backupProfiles, err) - } - - log.Infof("Saved profiles to %s", backupProfiles) - } - - if err = cli.backupHub(dirPath); err != nil { - return fmt.Errorf("failed to backup hub config: %w", err) - } - - return nil -} - func (cli *cliConfig) newBackupCmd() *cobra.Command { cmd := &cobra.Command{ - Use: `backup "directory"`, - Short: "Backup current config", - Long: `Backup the current crowdsec configuration including : - -- Main config (config.yaml) -- Simulation config (simulation.yaml) -- Profiles config (profiles.yaml) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Backup of API credentials (local API and online API)`, - Example: `cscli config backup ./my-backup`, - Args: cobra.ExactArgs(1), + Use: "backup", DisableAutoGenTag: true, - RunE: func(_ *cobra.Command, args []string) error { - if err := cli.backup(args[0]); err != nil { - return fmt.Errorf("failed to backup config: %w", err) - } - - return nil + RunE: func(_ *cobra.Command, _ []string) error { + configDir := cli.cfg().ConfigPaths.ConfigDir + return fmt.Errorf("'cscli config backup' has been removed, you can manually backup/restore %s instead", configDir) }, } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index b5fbf36b2b4..75373475ed9 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -1,285 +1,18 @@ package main import ( - "context" - "encoding/json" "fmt" - "os" - "path/filepath" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/hubops" ) -func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error { - cfg := cli.cfg() - - hub, err := require.Hub(cfg, nil) - if err != nil { - return err - } - - contentProvider := require.HubDownloader(ctx, cfg) - - for _, itype := range cwhub.ItemTypes { - itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) - if _, err = os.Stat(itemDirectory); err != nil { - log.Infof("no %s in backup", itype) - continue - } - /*restore the upstream items*/ - upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype) - - file, err := os.ReadFile(upstreamListFN) - if err != nil { - return fmt.Errorf("error while opening %s: %w", upstreamListFN, err) - } - - var upstreamList []string - - err = json.Unmarshal(file, &upstreamList) - if err != nil { - return fmt.Errorf("error parsing %s: %w", upstreamListFN, err) - } - - for _, toinstall := range upstreamList { - item := hub.GetItem(itype, toinstall) - if item == nil { - log.Errorf("Item %s/%s not found in hub", itype, toinstall) - continue - } - - plan := hubops.NewActionPlan(hub) - - if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, false)); err != nil { - return err - } - - if err = plan.AddCommand(hubops.NewEnableCommand(item, false)); err != nil { - return err - } - - if err = plan.Execute(ctx, true, false, false); err != nil { - log.Errorf("Error while installing %s : %s", toinstall, err) - } - } - - /*restore the local and tainted items*/ - files, err := os.ReadDir(itemDirectory) - if err != nil { - return fmt.Errorf("failed enumerating files of %s: %w", itemDirectory, err) - } - - for _, file := range files { - // this was the upstream data - if file.Name() == fmt.Sprintf("upstream-%s.json", itype) { - continue - } - - if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS { - // we expect a stage here - if !file.IsDir() { - continue - } - - stage := file.Name() - stagedir := fmt.Sprintf("%s/%s/%s/", cfg.ConfigPaths.ConfigDir, itype, stage) - log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir) - - if err = os.MkdirAll(stagedir, os.ModePerm); err != nil { - return fmt.Errorf("error while creating stage directory %s: %w", stagedir, err) - } - - // find items - ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/") - if err != nil { - return fmt.Errorf("failed enumerating files of %s: %w", itemDirectory+"/"+stage, err) - } - - // finally copy item - for _, tfile := range ifiles { - log.Infof("Going to restore local/tainted [%s]", tfile.Name()) - sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) - - destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) - if err = CopyFile(sourceFile, destinationFile); err != nil { - return fmt.Errorf("failed copy %s %s to %s: %w", itype, sourceFile, destinationFile, err) - } - - log.Infof("restored %s to %s", sourceFile, destinationFile) - } - } else { - log.Infof("Going to restore local/tainted [%s]", file.Name()) - sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) - destinationFile := fmt.Sprintf("%s/%s/%s", cfg.ConfigPaths.ConfigDir, itype, file.Name()) - - if err = CopyFile(sourceFile, destinationFile); err != nil { - return fmt.Errorf("failed copy %s %s to %s: %w", itype, sourceFile, destinationFile, err) - } - - log.Infof("restored %s to %s", sourceFile, destinationFile) - } - } - } - - return nil -} - -/* - Restore crowdsec configurations to directory : - -- Main config (config.yaml) -- Profiles config (profiles.yaml) -- Simulation config (simulation.yaml) -- Backup of API credentials (local API and online API) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Acquisition files (acquis.yaml, acquis.d/*.yaml) -*/ -func (cli *cliConfig) restore(ctx context.Context, dirPath string) error { - var err error - - cfg := cli.cfg() - - backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if _, err = os.Stat(backupMain); err == nil { - if cfg.ConfigPaths != nil && cfg.ConfigPaths.ConfigDir != "" { - if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", cfg.ConfigPaths.ConfigDir)); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupMain, cfg.ConfigPaths.ConfigDir, err) - } - } - } - - // Now we have config.yaml, we should regenerate config struct to have rights paths etc - ConfigFilePath = fmt.Sprintf("%s/config.yaml", cfg.ConfigPaths.ConfigDir) - - log.Debug("Reloading configuration") - - csConfig, _, err = loadConfigFor("config") - if err != nil { - return fmt.Errorf("failed to reload configuration: %w", err) - } - - cfg = cli.cfg() - - backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if _, err = os.Stat(backupCAPICreds); err == nil { - if err = CopyFile(backupCAPICreds, cfg.API.Server.OnlineClient.CredentialsFilePath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupCAPICreds, cfg.API.Server.OnlineClient.CredentialsFilePath, err) - } - } - - backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if _, err = os.Stat(backupLAPICreds); err == nil { - if err = CopyFile(backupLAPICreds, cfg.API.Client.CredentialsFilePath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupLAPICreds, cfg.API.Client.CredentialsFilePath, err) - } - } - - backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if _, err = os.Stat(backupProfiles); err == nil { - if err = CopyFile(backupProfiles, cfg.API.Server.ProfilesPath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupProfiles, cfg.API.Server.ProfilesPath, err) - } - } - - backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath) - if _, err = os.Stat(backupSimulation); err == nil { - if err = CopyFile(backupSimulation, cfg.ConfigPaths.SimulationFilePath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupSimulation, cfg.ConfigPaths.SimulationFilePath, err) - } - } - - /*if there is a acquisition dir, restore its content*/ - if cfg.Crowdsec.AcquisitionDirPath != "" { - if err = os.MkdirAll(cfg.Crowdsec.AcquisitionDirPath, 0o700); err != nil { - return fmt.Errorf("error while creating %s: %w", cfg.Crowdsec.AcquisitionDirPath, err) - } - } - - // if there was a single one - backupAcquisition := fmt.Sprintf("%s/acquis.yaml", dirPath) - if _, err = os.Stat(backupAcquisition); err == nil { - log.Debugf("restoring backup'ed %s", backupAcquisition) - - if err = CopyFile(backupAcquisition, cfg.Crowdsec.AcquisitionFilePath); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", backupAcquisition, cfg.Crowdsec.AcquisitionFilePath, err) - } - } - - // if there are files in the acquis backup dir, restore them - acquisBackupDir := filepath.Join(dirPath, "acquis", "*.yaml") - if acquisFiles, err := filepath.Glob(acquisBackupDir); err == nil { - for _, acquisFile := range acquisFiles { - targetFname, err := filepath.Abs(cfg.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile)) - if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err) - } - - log.Debugf("restoring %s to %s", acquisFile, targetFname) - - if err = CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) - } - } - } - - if cfg.Crowdsec != nil && len(cfg.Crowdsec.AcquisitionFiles) > 0 { - for _, acquisFile := range cfg.Crowdsec.AcquisitionFiles { - log.Infof("backup filepath from dir -> %s", acquisFile) - - // if it was the default one, it has already been backed up - if cfg.Crowdsec.AcquisitionFilePath == acquisFile { - log.Infof("skip this one") - continue - } - - targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) - if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) - } - - if err = CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) - } - - log.Infof("Saved acquis %s to %s", acquisFile, targetFname) - } - } - - if err = cli.restoreHub(ctx, dirPath); err != nil { - return fmt.Errorf("failed to restore hub config: %w", err) - } - - return nil -} - func (cli *cliConfig) newRestoreCmd() *cobra.Command { cmd := &cobra.Command{ - Use: `restore "directory"`, - Short: `Restore config in backup "directory"`, - Long: `Restore the crowdsec configuration from specified backup "directory" including: - -- Main config (config.yaml) -- Simulation config (simulation.yaml) -- Profiles config (profiles.yaml) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Backup of API credentials (local API and online API)`, - Args: cobra.ExactArgs(1), + Use: "restore", DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - dirPath := args[0] - - if err := cli.restore(cmd.Context(), dirPath); err != nil { - return fmt.Errorf("failed to restore config from %s: %w", dirPath, err) - } - - return nil + RunE: func(cmd *cobra.Command, _ []string) error { + configDir := cli.cfg().ConfigPaths.ConfigDir + return fmt.Errorf("'cscli config restore' has been removed, you can manually backup/restore %s instead", configDir) }, } diff --git a/cmd/crowdsec-cli/copyfile.go b/cmd/crowdsec-cli/copyfile.go deleted file mode 100644 index 272fb3f7851..00000000000 --- a/cmd/crowdsec-cli/copyfile.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "fmt" - "io" - "os" - "path/filepath" - - log "github.com/sirupsen/logrus" -) - -/*help to copy the file, ioutil doesn't offer the feature*/ - -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return - } - - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - - if _, err = io.Copy(out, in); err != nil { - return - } - - err = out.Sync() - - return -} - -/*copy the file, ioutile doesn't offer the feature*/ -func CopyFile(sourceSymLink, destinationFile string) error { - sourceFile, err := filepath.EvalSymlinks(sourceSymLink) - if err != nil { - log.Infof("Not a symlink : %s", err) - - sourceFile = sourceSymLink - } - - sourceFileStat, err := os.Stat(sourceFile) - if err != nil { - return err - } - - if !sourceFileStat.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) - } - - destinationFileStat, err := os.Stat(destinationFile) - if err != nil { - if !os.IsNotExist(err) { - return err - } - } else { - if !(destinationFileStat.Mode().IsRegular()) { - return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) - } - - if os.SameFile(sourceFileStat, destinationFileStat) { - return err - } - } - - if err = os.Link(sourceFile, destinationFile); err != nil { - err = copyFileContents(sourceFile, destinationFile) - } - - return err -} diff --git a/debian/postinst b/debian/postinst index 77f2511f556..d50a7c0bfe2 100644 --- a/debian/postinst +++ b/debian/postinst @@ -11,14 +11,6 @@ if [ "$1" = configure ]; then mkdir -p /var/lib/crowdsec/data fi - if [[ -d /var/lib/crowdsec/backup ]]; then - cscli config restore /var/lib/crowdsec/backup/backup.config - rm -rf /var/lib/crowdsec/backup - /usr/bin/cscli hub update - /usr/bin/cscli hub upgrade - systemctl start crowdsec - fi - . /usr/share/crowdsec/wizard.sh -n if ! [[ -f /etc/crowdsec/acquis.yaml ]]; then echo Creating /etc/crowdsec/acquis.yaml @@ -82,12 +74,6 @@ if [ "$1" = configure ]; then set -e fi - - if [[ -f /var/lib/crowdsec/data/crowdsec.db.backup ]]; then - cp /var/lib/crowdsec/data/crowdsec.db.backup /var/lib/crowdsec/data/crowdsec.db - rm -f /var/lib/crowdsec/data/crowdsec.db.backup - fi - systemctl --quiet is-enabled crowdsec || systemctl unmask crowdsec && systemctl enable crowdsec API=$(cscli config show --key "Config.API.Server") @@ -107,8 +93,6 @@ if [ "$1" = configure ]; then echo " * Detailed guides are available in our documentation: https://docs.crowdsec.net" echo " * Configuration items created by the community can be found at the Hub: https://hub.crowdsec.net" echo " * Gain insights into your use of CrowdSec with the help of the console https://app.crowdsec.net" - - fi echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'" diff --git a/debian/preinst b/debian/preinst index 217b836caa6..df5b56cef3f 100644 --- a/debian/preinst +++ b/debian/preinst @@ -5,39 +5,4 @@ set -e # Source debconf library. . /usr/share/debconf/confmodule - -OLD_MAJOR_VERSION=$(echo $2 | cut -d'.' -f1) -OLD_MINOR_VERSION=$(echo $2 | cut -d'.' -f2) -OLD_PATCH_VERSION=$(echo $2 | cut -d'.' -f3|cut -d'-' -f1) - -NEW_MAJOR_VERSION=$(echo $3 | cut -d'.' -f1) -NEW_MINOR_VERSION=$(echo $3 | cut -d'.' -f2) -NEW_PATCH_VERSION=$(echo $3 | cut -d'.' -f3|cut -d'-' -f1) - - - -if [ "$1" = upgrade ]; then - - OLD_MAJOR_VERSION=$(echo $2 | cut -d'.' -f1) - OLD_MINOR_VERSION=$(echo $2 | cut -d'.' -f2) - OLD_PATCH_VERSION=$(echo $2 | cut -d'.' -f3|cut -d'-' -f1) - - NEW_MAJOR_VERSION=$(echo $3 | cut -d'.' -f1) - NEW_MINOR_VERSION=$(echo $3 | cut -d'.' -f2) - NEW_PATCH_VERSION=$(echo $3 | cut -d'.' -f3|cut -d'-' -f1) - - - if [[ $OLD_MAJOR_VERSION -eq "1" ]] && [[ $OLD_MINOR_VERSION -eq "0" ]] && [[ $OLD_PATCH_VERSION -lt "9" ]]; then - if [[ -f /var/lib/crowdsec/data/crowdsec.db ]]; then - cp /var/lib/crowdsec/data/crowdsec.db /var/lib/crowdsec/data/crowdsec.db.backup - fi - fi - - if [[ $NEW_MAJOR_VERSION -gt $OLD_MAJOR_VERSION ]]; then - echo "Stopping crowdsec" - systemctl stop crowdsec || true - cscli config backup /var/lib/crowdsec/backup - fi -fi - echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'" diff --git a/debian/prerm b/debian/prerm index a463a6a1c80..10afcf1906d 100644 --- a/debian/prerm +++ b/debian/prerm @@ -1,9 +1,8 @@ if [ "$1" = "remove" ]; then - cscli dashboard remove -f -y --error || echo "Ignore the above error if you never installed the local dashboard." systemctl stop crowdsec systemctl disable crowdsec fi if [ "$1" = "upgrade" ]; then systemctl stop crowdsec -fi \ No newline at end of file +fi diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 06a4918592b..d3928270598 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -365,13 +365,13 @@ func copyEvent(evt types.Event, line string) types.Event { return evtCopy } -func transform(transformChan chan types.Event, output chan types.Event, AcquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) { +func transform(transformChan chan types.Event, output chan types.Event, acquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) { defer trace.CatchPanic("crowdsec/acquis") logger.Infof("transformer started") for { select { - case <-AcquisTomb.Dying(): + case <-acquisTomb.Dying(): logger.Debugf("transformer is dying") return case evt := <-transformChan: @@ -420,7 +420,7 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo } } -func StartAcquisition(ctx context.Context, sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error { +func StartAcquisition(ctx context.Context, sources []DataSource, output chan types.Event, acquisTomb *tomb.Tomb) error { // Don't wait if we have no sources, as it will hang forever if len(sources) == 0 { return nil @@ -430,7 +430,7 @@ func StartAcquisition(ctx context.Context, sources []DataSource, output chan typ subsrc := sources[i] // ensure its a copy log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc) - AcquisTomb.Go(func() error { + acquisTomb.Go(func() error { defer trace.CatchPanic("crowdsec/acquis") var err error @@ -449,21 +449,21 @@ func StartAcquisition(ctx context.Context, sources []DataSource, output chan typ "datasource": subsrc.GetName(), }) - AcquisTomb.Go(func() error { - transform(outChan, output, AcquisTomb, transformRuntime, transformLogger) + acquisTomb.Go(func() error { + transform(outChan, output, acquisTomb, transformRuntime, transformLogger) return nil }) } if subsrc.GetMode() == configuration.TAIL_MODE { - err = subsrc.StreamingAcquisition(ctx, outChan, AcquisTomb) + err = subsrc.StreamingAcquisition(ctx, outChan, acquisTomb) } else { - err = subsrc.OneShotAcquisition(ctx, outChan, AcquisTomb) + err = subsrc.OneShotAcquisition(ctx, outChan, acquisTomb) } if err != nil { // if one of the acqusition returns an error, we kill the others to properly shutdown - AcquisTomb.Kill(err) + acquisTomb.Kill(err) } return nil @@ -471,7 +471,7 @@ func StartAcquisition(ctx context.Context, sources []DataSource, output chan typ } /*return only when acquisition is over (cat) or never (tail)*/ - err := AcquisTomb.Wait() + err := acquisTomb.Wait() return err } diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index fae7ed87f4a..a4c2c5124b3 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -155,14 +155,14 @@ func (w *AppsecSource) GetAggregMetrics() []prometheus.Collector { return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram} } -func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { err := w.UnmarshalConfig(yamlConfig) if err != nil { return fmt.Errorf("unable to parse appsec configuration: %w", err) } w.logger = logger - w.metricsLevel = MetricsLevel + w.metricsLevel = metricsLevel w.logger.Tracef("Appsec configuration: %+v", w.config) if w.config.AuthCacheDuration == nil { @@ -180,7 +180,7 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLe w.InChan = make(chan appsec.ParsedRequest) appsecCfg := appsec.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")} - //we keep the datasource name + // we keep the datasource name appsecCfg.Name = w.config.Name // let's load the associated appsec_config: @@ -275,6 +275,7 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. for _, runner := range w.AppsecRunners { runner.outChan = out + t.Go(func() error { defer trace.CatchPanic("crowdsec/acquis/appsec/live/runner") return runner.Run(t) @@ -285,16 +286,20 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. if w.config.ListenSocket != "" { w.logger.Infof("creating unix socket %s", w.config.ListenSocket) _ = os.RemoveAll(w.config.ListenSocket) + listener, err := net.Listen("unix", w.config.ListenSocket) if err != nil { return fmt.Errorf("appsec server failed: %w", err) } + defer listener.Close() + if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { err = w.server.ServeTLS(listener, w.config.CertFilePath, w.config.KeyFilePath) } else { err = w.server.Serve(listener) } + if err != nil && !errors.Is(err, http.ErrServerClosed) { return fmt.Errorf("appsec server failed: %w", err) } @@ -304,8 +309,10 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. }) t.Go(func() error { var err error + if w.config.ListenAddr != "" { w.logger.Infof("creating TCP server on %s", w.config.ListenAddr) + if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { err = w.server.ListenAndServeTLS(w.config.CertFilePath, w.config.KeyFilePath) } else { @@ -323,9 +330,11 @@ func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types. w.logger.Info("Shutting down Appsec server") // xx let's clean up the appsec runners :) appsec.AppsecRulesDetails = make(map[int]appsec.RulesDetails) + if err := w.server.Shutdown(ctx); err != nil { w.logger.Errorf("Error shutting down Appsec server: %s", err.Error()) } + return nil }) @@ -356,11 +365,13 @@ func (w *AppsecSource) IsAuth(apiKey string) bool { } req.Header.Add("X-Api-Key", apiKey) + resp, err := client.Do(req) if err != nil { log.Errorf("Error performing request: %s", err) return false } + defer resp.Body.Close() return resp.StatusCode == http.StatusOK @@ -373,17 +384,21 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { apiKey := r.Header.Get(appsec.APIKeyHeaderName) clientIP := r.Header.Get(appsec.IPHeaderName) remoteIP := r.RemoteAddr + if apiKey == "" { w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) rw.WriteHeader(http.StatusUnauthorized) + return } + expiration, exists := w.AuthCache.Get(apiKey) // if the apiKey is not in cache or has expired, just recheck the auth if !exists || time.Now().After(expiration) { if !w.IsAuth(apiKey) { rw.WriteHeader(http.StatusUnauthorized) w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) + return } @@ -396,8 +411,10 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { if err != nil { w.logger.Errorf("%s", err) rw.WriteHeader(http.StatusInternalServerError) + return } + parsedRequest.AppsecEngine = w.config.Name logger := w.logger.WithFields(log.Fields{ diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index ba267c9050b..5739ebc3124 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -154,13 +154,13 @@ func (cw *CloudwatchSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { err := cw.UnmarshalConfig(yamlConfig) if err != nil { return err } - cw.metricsLevel = MetricsLevel + cw.metricsLevel = metricsLevel cw.logger = logger.WithField("group", cw.Config.GroupName) @@ -330,9 +330,12 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(ctx context.Context, out cha LastIngestionTime := time.Unix(0, *event.LastIngestionTime*int64(time.Millisecond)) if LastIngestionTime.Before(oldest) { cw.logger.Tracef("stop iteration, %s reached oldest age, stop (%s < %s)", *event.LogStreamName, LastIngestionTime, time.Now().UTC().Add(-*cw.Config.MaxStreamAge)) + hasMoreStreams = false + return false } + cw.logger.Tracef("stream %s is elligible for monitoring", *event.LogStreamName) // the stream has been updated recently, check if we should monitor it var expectMode int @@ -341,6 +344,7 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(ctx context.Context, out cha } else { expectMode = types.TIMEMACHINE } + monitorStream := LogStreamTailConfig{ GroupName: cw.Config.GroupName, StreamName: *event.LogStreamName, @@ -354,16 +358,20 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(ctx context.Context, out cha out <- monitorStream } } + if lastPage { cw.logger.Tracef("reached last page") + hasMoreStreams = false } + return true }, ) if err != nil { return fmt.Errorf("while describing group %s: %w", cw.Config.GroupName, err) } + cw.logger.Tracef("after DescribeLogStreamsPagesWithContext") } } @@ -373,12 +381,14 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(ctx context.Context, out cha // LogStreamManager receives the potential streams to monitor, and starts a go routine when needed func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStreamTailConfig, outChan chan types.Event) error { cw.logger.Debugf("starting to monitor streams for %s", cw.Config.GroupName) + pollDeadStreamInterval := time.NewTicker(def_PollDeadStreamInterval) for { select { case newStream := <-in: //nolint:govet // copylocks won't matter if the tomb is not initialized shouldCreate := true + cw.logger.Tracef("received new streams to monitor : %s/%s", newStream.GroupName, newStream.StreamName) if cw.Config.StreamName != nil && newStream.StreamName != *cw.Config.StreamName { @@ -402,12 +412,16 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr if !stream.t.Alive() { cw.logger.Debugf("stream %s already exists, but is dead", newStream.StreamName) cw.monitoredStreams = append(cw.monitoredStreams[:idx], cw.monitoredStreams[idx+1:]...) + if cw.metricsLevel != configuration.METRICS_NONE { openedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Dec() } + break } + shouldCreate = false + break } } @@ -417,19 +431,23 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr if cw.metricsLevel != configuration.METRICS_NONE { openedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Inc() } + newStream.t = tomb.Tomb{} newStream.logger = cw.logger.WithField("stream", newStream.StreamName) cw.logger.Debugf("starting tail of stream %s", newStream.StreamName) newStream.t.Go(func() error { return cw.TailLogStream(ctx, &newStream, outChan) }) + cw.monitoredStreams = append(cw.monitoredStreams, &newStream) } case <-pollDeadStreamInterval.C: newMonitoredStreams := cw.monitoredStreams[:0] + for idx, stream := range cw.monitoredStreams { if !cw.monitoredStreams[idx].t.Alive() { cw.logger.Debugf("remove dead stream %s", stream.StreamName) + if cw.metricsLevel != configuration.METRICS_NONE { openedStreams.With(prometheus.Labels{"group": cw.monitoredStreams[idx].GroupName}).Dec() } @@ -437,20 +455,25 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr newMonitoredStreams = append(newMonitoredStreams, stream) } } + cw.monitoredStreams = newMonitoredStreams case <-cw.t.Dying(): cw.logger.Infof("LogStreamManager for %s is dying, %d alive streams", cw.Config.GroupName, len(cw.monitoredStreams)) + for idx, stream := range cw.monitoredStreams { if cw.monitoredStreams[idx].t.Alive() { cw.logger.Debugf("killing stream %s", stream.StreamName) cw.monitoredStreams[idx].t.Kill(nil) + if err := cw.monitoredStreams[idx].t.Wait(); err != nil { cw.logger.Debugf("error while waiting for death of %s : %s", stream.StreamName, err) } } } + cw.monitoredStreams = nil cw.logger.Debugf("routine cleanup done, return") + return nil } } @@ -458,12 +481,14 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr func (cw *CloudwatchSource) TailLogStream(ctx context.Context, cfg *LogStreamTailConfig, outChan chan types.Event) error { var startFrom *string + lastReadMessage := time.Now().UTC() ticker := time.NewTicker(cfg.PollStreamInterval) // resume at existing index if we already had streamIndexMutex.Lock() v := cw.streamIndexes[cfg.GroupName+"+"+cfg.StreamName] streamIndexMutex.Unlock() + if v != "" { cfg.logger.Debugf("restarting on index %s", v) startFrom = &v @@ -474,7 +499,9 @@ func (cw *CloudwatchSource) TailLogStream(ctx context.Context, cfg *LogStreamTai select { case <-ticker.C: cfg.logger.Tracef("entering loop") + hasMorePages := true + for hasMorePages { /*for the first call, we only consume the last item*/ cfg.logger.Tracef("calling GetLogEventsPagesWithContext") @@ -489,36 +516,44 @@ func (cw *CloudwatchSource) TailLogStream(ctx context.Context, cfg *LogStreamTai func(page *cloudwatchlogs.GetLogEventsOutput, lastPage bool) bool { cfg.logger.Tracef("%d results, last:%t", len(page.Events), lastPage) startFrom = page.NextForwardToken + if page.NextForwardToken != nil { streamIndexMutex.Lock() cw.streamIndexes[cfg.GroupName+"+"+cfg.StreamName] = *page.NextForwardToken streamIndexMutex.Unlock() } + if lastPage { /*wait another ticker to check on new log availability*/ cfg.logger.Tracef("last page") + hasMorePages = false } + if len(page.Events) > 0 { lastReadMessage = time.Now().UTC() } + for _, event := range page.Events { evt, err := cwLogToEvent(event, cfg) if err != nil { cfg.logger.Warningf("cwLogToEvent error, discarded event : %s", err) } else { cfg.logger.Debugf("pushing message : %s", evt.Line.Raw) + if cw.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"group": cfg.GroupName, "stream": cfg.StreamName}).Inc() } outChan <- evt } } + return true }, ) if err != nil { newerr := fmt.Errorf("while reading %s/%s: %w", cfg.GroupName, cfg.StreamName, err) cfg.logger.Warningf("err : %s", newerr) + return newerr } cfg.logger.Tracef("done reading GetLogEventsPagesWithContext") diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index 798eba29440..582da3d53a1 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -136,9 +136,9 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (d *DockerSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (d *DockerSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { d.logger = logger - d.metricsLevel = MetricsLevel + d.metricsLevel = metricsLevel err := d.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 9f439b0c82e..697a3d35dc2 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -102,9 +102,9 @@ func (f *FileSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { f.logger = logger - f.metricsLevel = MetricsLevel + f.metricsLevel = metricsLevel err := f.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 3e4f26915fd..97e220570ff 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -157,9 +157,9 @@ func (hc *HttpConfiguration) Validate() error { return nil } -func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { h.logger = logger - h.metricsLevel = MetricsLevel + h.metricsLevel = metricsLevel err := h.UnmarshalConfig(yamlConfig) if err != nil { @@ -339,12 +339,14 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { if r.Method != http.MethodPost { h.logger.Errorf("method not allowed: %s", r.Method) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) + return } if err := authorizeRequest(r, &h.Config); err != nil { h.logger.Errorf("failed to authorize request from '%s': %s", r.RemoteAddr, err) http.Error(w, "Unauthorized", http.StatusUnauthorized) + return } diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index 47d90e2b3a0..f72878d9b3c 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -210,9 +210,9 @@ func (j *JournalCtlSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { j.logger = logger - j.metricsLevel = MetricsLevel + j.metricsLevel = metricsLevel err := j.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index 77fc44e310d..f213b85814c 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -85,9 +85,9 @@ func (k *KafkaSource) UnmarshalConfig(yamlConfig []byte) error { return err } -func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { k.logger = logger - k.metricsLevel = MetricsLevel + k.metricsLevel = metricsLevel k.logger.Debugf("start configuring %s source", dataSourceName) @@ -160,6 +160,7 @@ func (k *KafkaSource) ReadMessage(ctx context.Context, out chan types.Event) err k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err)) continue } + k.logger.Tracef("got message: %s", string(m.Value)) l := types.Line{ Raw: string(m.Value), @@ -170,9 +171,11 @@ func (k *KafkaSource) ReadMessage(ctx context.Context, out chan types.Event) err Module: k.GetName(), } k.logger.Tracef("line with message read from topic '%s': %+v", k.Config.Topic, l) + if k.metricsLevel != configuration.METRICS_NONE { linesRead.With(prometheus.Labels{"topic": k.Config.Topic}).Inc() } + evt := types.MakeEvent(k.Config.UseTimeMachine, types.LOG, true) evt.Line = l out <- evt diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index b166a706ca9..16c91ad06bc 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -161,9 +161,9 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { k.logger = logger - k.metricsLevel = MetricsLevel + k.metricsLevel = metricsLevel err := k.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index 4558b745e53..b0650d3906e 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -97,9 +97,9 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (ka *KubernetesAuditSource) Configure(config []byte, logger *log.Entry, MetricsLevel int) error { +func (ka *KubernetesAuditSource) Configure(config []byte, logger *log.Entry, metricsLevel int) error { ka.logger = logger - ka.metricsLevel = MetricsLevel + ka.metricsLevel = metricsLevel err := ka.UnmarshalConfig(config) if err != nil { diff --git a/pkg/acquisition/modules/loki/loki.go b/pkg/acquisition/modules/loki/loki.go index c57e6a67c94..47493d8cdfe 100644 --- a/pkg/acquisition/modules/loki/loki.go +++ b/pkg/acquisition/modules/loki/loki.go @@ -120,10 +120,10 @@ func (l *LokiSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (l *LokiSource) Configure(config []byte, logger *log.Entry, MetricsLevel int) error { +func (l *LokiSource) Configure(config []byte, logger *log.Entry, metricsLevel int) error { l.Config = LokiConfiguration{} l.logger = logger - l.metricsLevel = MetricsLevel + l.metricsLevel = metricsLevel err := l.UnmarshalConfig(config) if err != nil { return err diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index fb6a04600c1..df805d08cae 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -124,10 +124,10 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { s.logger = logger s.logger.Infof("Starting syslog datasource configuration") - s.metricsLevel = MetricsLevel + s.metricsLevel = metricsLevel err := s.UnmarshalConfig(yamlConfig) if err != nil { return err diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index 8283bcc21a2..22186ea96cb 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -287,9 +287,9 @@ func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLevel int) error { +func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { w.logger = logger - w.metricsLevel = MetricsLevel + w.metricsLevel = metricsLevel err := w.UnmarshalConfig(yamlConfig) if err != nil { diff --git a/pkg/apiclient/client.go b/pkg/apiclient/client.go index 47d97a28344..ec473beca77 100644 --- a/pkg/apiclient/client.go +++ b/pkg/apiclient/client.go @@ -125,8 +125,8 @@ func NewClient(config *Config) (*ApiClient, error) { return c, nil } -func NewDefaultClient(URL *url.URL, prefix string, userAgent string, client *http.Client) (*ApiClient, error) { - transport, baseURL := createTransport(URL) +func NewDefaultClient(url *url.URL, prefix string, userAgent string, client *http.Client) (*ApiClient, error) { + transport, baseURL := createTransport(url) if client == nil { client = &http.Client{} diff --git a/pkg/csplugin/listfiles_test.go b/pkg/csplugin/listfiles_test.go index c476d7a4e4a..32269f3f5f1 100644 --- a/pkg/csplugin/listfiles_test.go +++ b/pkg/csplugin/listfiles_test.go @@ -12,19 +12,22 @@ import ( ) func TestListFilesAtPath(t *testing.T) { - dir, err := os.MkdirTemp("", "test-listfiles") - require.NoError(t, err) - t.Cleanup(func() { - os.RemoveAll(dir) - }) - _, err = os.Create(filepath.Join(dir, "notification-gitter")) + dir := t.TempDir() + + f, err := os.Create(filepath.Join(dir, "notification-gitter")) require.NoError(t, err) - _, err = os.Create(filepath.Join(dir, "slack")) + require.NoError(t, f.Close()) + + f, err = os.Create(filepath.Join(dir, "slack")) require.NoError(t, err) + require.NoError(t, f.Close()) + err = os.Mkdir(filepath.Join(dir, "somedir"), 0o755) require.NoError(t, err) - _, err = os.Create(filepath.Join(dir, "somedir", "inner")) + + f, err = os.Create(filepath.Join(dir, "somedir", "inner")) require.NoError(t, err) + require.NoError(t, f.Close()) tests := []struct { name string diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index 52cda1ed2e1..c509fb448e3 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -96,17 +96,17 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { return profilesRuntime, nil } -func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*models.Decision, error) { +func (profile *Runtime) GenerateDecisionFromProfile(alert *models.Alert) ([]*models.Decision, error) { var decisions []*models.Decision - for _, refDecision := range Profile.Cfg.Decisions { + for _, refDecision := range profile.Cfg.Decisions { decision := models.Decision{} /*the reference decision from profile is in simulated mode */ if refDecision.Simulated != nil && *refDecision.Simulated { decision.Simulated = new(bool) *decision.Simulated = true /*the event is already in simulation mode */ - } else if Alert.Simulated != nil && *Alert.Simulated { + } else if alert.Simulated != nil && *alert.Simulated { decision.Simulated = new(bool) *decision.Simulated = true } @@ -116,7 +116,7 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod if refDecision.Scope != nil && *refDecision.Scope != "" { *decision.Scope = *refDecision.Scope } else { - *decision.Scope = *Alert.Source.Scope + *decision.Scope = *alert.Source.Scope } /*some fields are populated from the reference object : duration, scope, type*/ @@ -125,19 +125,19 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod *decision.Duration = *refDecision.Duration } - if Profile.Cfg.DurationExpr != "" && Profile.RuntimeDurationExpr != nil { + if profile.Cfg.DurationExpr != "" && profile.RuntimeDurationExpr != nil { profileDebug := false - if Profile.Cfg.Debug != nil && *Profile.Cfg.Debug { + if profile.Cfg.Debug != nil && *profile.Cfg.Debug { profileDebug = true } - duration, err := exprhelpers.Run(Profile.RuntimeDurationExpr, map[string]interface{}{"Alert": Alert}, Profile.Logger, profileDebug) + duration, err := exprhelpers.Run(profile.RuntimeDurationExpr, map[string]interface{}{"Alert": alert}, profile.Logger, profileDebug) if err != nil { - Profile.Logger.Warningf("Failed to run duration_expr : %v", err) + profile.Logger.Warningf("Failed to run duration_expr : %v", err) } else { durationStr := fmt.Sprint(duration) if _, err := time.ParseDuration(durationStr); err != nil { - Profile.Logger.Warningf("Failed to parse expr duration result '%s'", duration) + profile.Logger.Warningf("Failed to parse expr duration result '%s'", duration) } else { *decision.Duration = durationStr } @@ -149,7 +149,7 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod /*for the others, let's populate it from the alert and its source*/ decision.Value = new(string) - *decision.Value = *Alert.Source.Value + *decision.Value = *alert.Source.Value decision.Origin = new(string) *decision.Origin = types.CrowdSecOrigin @@ -158,7 +158,7 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod } decision.Scenario = new(string) - *decision.Scenario = *Alert.Scenario + *decision.Scenario = *alert.Scenario decisions = append(decisions, &decision) } @@ -166,21 +166,21 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod } // EvaluateProfile is going to evaluate an Alert against a profile to generate Decisions -func (Profile *Runtime) EvaluateProfile(Alert *models.Alert) ([]*models.Decision, bool, error) { +func (profile *Runtime) EvaluateProfile(alert *models.Alert) ([]*models.Decision, bool, error) { var decisions []*models.Decision matched := false - for eIdx, expression := range Profile.RuntimeFilters { + for eIdx, expression := range profile.RuntimeFilters { debugProfile := false - if Profile.Cfg.Debug != nil && *Profile.Cfg.Debug { + if profile.Cfg.Debug != nil && *profile.Cfg.Debug { debugProfile = true } - output, err := exprhelpers.Run(expression, map[string]interface{}{"Alert": Alert}, Profile.Logger, debugProfile) + output, err := exprhelpers.Run(expression, map[string]interface{}{"Alert": alert}, profile.Logger, debugProfile) if err != nil { - Profile.Logger.Warningf("failed to run profile expr for %s: %v", Profile.Cfg.Name, err) - return nil, matched, fmt.Errorf("while running expression %s: %w", Profile.Cfg.Filters[eIdx], err) + profile.Logger.Warningf("failed to run profile expr for %s: %v", profile.Cfg.Name, err) + return nil, matched, fmt.Errorf("while running expression %s: %w", profile.Cfg.Filters[eIdx], err) } switch out := output.(type) { @@ -188,22 +188,22 @@ func (Profile *Runtime) EvaluateProfile(Alert *models.Alert) ([]*models.Decision if out { matched = true /*the expression matched, create the associated decision*/ - subdecisions, err := Profile.GenerateDecisionFromProfile(Alert) + subdecisions, err := profile.GenerateDecisionFromProfile(alert) if err != nil { - return nil, matched, fmt.Errorf("while generating decision from profile %s: %w", Profile.Cfg.Name, err) + return nil, matched, fmt.Errorf("while generating decision from profile %s: %w", profile.Cfg.Name, err) } decisions = append(decisions, subdecisions...) } else { - Profile.Logger.Debugf("Profile %s filter is unsuccessful", Profile.Cfg.Name) + profile.Logger.Debugf("Profile %s filter is unsuccessful", profile.Cfg.Name) - if Profile.Cfg.OnFailure == "break" { + if profile.Cfg.OnFailure == "break" { break } } default: - return nil, matched, fmt.Errorf("unexpected type %t (%v) while running '%s'", output, output, Profile.Cfg.Filters[eIdx]) + return nil, matched, fmt.Errorf("unexpected type %t (%v) while running '%s'", output, output, profile.Cfg.Filters[eIdx]) } } diff --git a/pkg/database/database.go b/pkg/database/database.go index bb41dd3b645..80479710751 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -68,7 +68,7 @@ func NewClient(ctx context.Context, config *csconfig.DatabaseCfg) (*Client, erro return nil, err // unsupported database caught here } - if config.Type == "sqlite" { + if config.Type == "sqlite" && config.DbPath != ":memory:" { /*if it's the first startup, we want to touch and chmod file*/ if _, err = os.Stat(config.DbPath); os.IsNotExist(err) { f, err := os.OpenFile(config.DbPath, os.O_CREATE|os.O_RDWR, 0o600) diff --git a/pkg/database/flush.go b/pkg/database/flush.go index 8f646ddc961..4a3a93a406c 100644 --- a/pkg/database/flush.go +++ b/pkg/database/flush.go @@ -222,7 +222,7 @@ func (c *Client) FlushAgentsAndBouncers(ctx context.Context, agentsCfg *csconfig return nil } -func (c *Client) FlushAlerts(ctx context.Context, MaxAge string, MaxItems int) error { +func (c *Client) FlushAlerts(ctx context.Context, maxAge string, maxItems int) error { var ( deletedByAge int deletedByNbItem int @@ -247,22 +247,22 @@ func (c *Client) FlushAlerts(ctx context.Context, MaxAge string, MaxItems int) e c.Log.Debugf("FlushAlerts (Total alerts): %d", totalAlerts) - if MaxAge != "" { + if maxAge != "" { filter := map[string][]string{ - "created_before": {MaxAge}, + "created_before": {maxAge}, } nbDeleted, err := c.DeleteAlertWithFilter(ctx, filter) if err != nil { c.Log.Warningf("FlushAlerts (max age): %s", err) - return fmt.Errorf("unable to flush alerts with filter until=%s: %w", MaxAge, err) + return fmt.Errorf("unable to flush alerts with filter until=%s: %w", maxAge, err) } c.Log.Debugf("FlushAlerts (deleted max age alerts): %d", nbDeleted) deletedByAge = nbDeleted } - if MaxItems > 0 { + if maxItems > 0 { // We get the highest id for the alerts // We subtract MaxItems to avoid deleting alerts that are not old enough // This gives us the oldest alert that we want to keep @@ -282,7 +282,7 @@ func (c *Client) FlushAlerts(ctx context.Context, MaxAge string, MaxItems int) e } if len(lastAlert) != 0 { - maxid := lastAlert[0].ID - MaxItems + maxid := lastAlert[0].ID - maxItems c.Log.Debugf("FlushAlerts (max id): %d", maxid) @@ -299,12 +299,12 @@ func (c *Client) FlushAlerts(ctx context.Context, MaxAge string, MaxItems int) e if deletedByNbItem > 0 { c.Log.Infof("flushed %d/%d alerts because the max number of alerts has been reached (%d max)", - deletedByNbItem, totalAlerts, MaxItems) + deletedByNbItem, totalAlerts, maxItems) } if deletedByAge > 0 { c.Log.Infof("flushed %d/%d alerts because they were created %s ago or more", - deletedByAge, totalAlerts, MaxAge) + deletedByAge, totalAlerts, maxAge) } return nil diff --git a/pkg/exprhelpers/crowdsec_cti.go b/pkg/exprhelpers/crowdsec_cti.go index 9b9eac4b95c..900bd7824a8 100644 --- a/pkg/exprhelpers/crowdsec_cti.go +++ b/pkg/exprhelpers/crowdsec_cti.go @@ -29,29 +29,29 @@ var ( var ctiClient *cticlient.CrowdsecCTIClient -func InitCrowdsecCTI(Key *string, TTL *time.Duration, Size *int, LogLevel *log.Level) error { - if Key == nil || *Key == "" { +func InitCrowdsecCTI(key *string, ttl *time.Duration, size *int, logLevel *log.Level) error { + if key == nil || *key == "" { log.Warningf("CTI API key not set or empty, CTI will not be available") return cticlient.ErrDisabled } - CTIApiKey = *Key - if Size == nil { - Size = new(int) - *Size = 1000 + CTIApiKey = *key + if size == nil { + size = new(int) + *size = 1000 } - if TTL == nil { - TTL = new(time.Duration) - *TTL = 5 * time.Minute + if ttl == nil { + ttl = new(time.Duration) + *ttl = 5 * time.Minute } clog := log.New() if err := types.ConfigureLogger(clog); err != nil { return fmt.Errorf("while configuring datasource logger: %w", err) } - if LogLevel != nil { - clog.SetLevel(*LogLevel) + if logLevel != nil { + clog.SetLevel(*logLevel) } subLogger := clog.WithField("type", "crowdsec-cti") - CrowdsecCTIInitCache(*Size, *TTL) + CrowdsecCTIInitCache(*size, *ttl) ctiClient = cticlient.NewCrowdsecCTIClient(cticlient.WithAPIKey(CTIApiKey), cticlient.WithLogger(subLogger)) CTIApiEnabled = true return nil diff --git a/pkg/exprhelpers/exprlib_test.go b/pkg/exprhelpers/exprlib_test.go index f2eb208ebfa..932db4b7da4 100644 --- a/pkg/exprhelpers/exprlib_test.go +++ b/pkg/exprhelpers/exprlib_test.go @@ -3,7 +3,6 @@ package exprhelpers import ( "context" "errors" - "os" "testing" "time" @@ -26,15 +25,12 @@ const TestFolder = "tests" func getDBClient(t *testing.T) *database.Client { t.Helper() - dbPath, err := os.CreateTemp("", "*sqlite") - require.NoError(t, err) - ctx := context.Background() testDBClient, err := database.NewClient(ctx, &csconfig.DatabaseCfg{ Type: "sqlite", DbName: "crowdsec", - DbPath: dbPath.Name(), + DbPath: ":memory:", }) require.NoError(t, err) diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 2f48da8c644..d0f6f2cfe22 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -127,31 +127,33 @@ func Init(databaseClient *database.Client) error { dataFileRegex = make(map[string][]*regexp.Regexp) dataFileRe2 = make(map[string][]*re2.Regexp) dbClient = databaseClient + XMLCacheInit() + return nil } -func RegexpCacheInit(filename string, CacheCfg types.DataSource) error { +func RegexpCacheInit(filename string, cacheCfg types.DataSource) error { // cache is explicitly disabled - if CacheCfg.Cache != nil && !*CacheCfg.Cache { + if cacheCfg.Cache != nil && !*cacheCfg.Cache { return nil } // cache is implicitly disabled if no cache config is provided - if CacheCfg.Strategy == nil && CacheCfg.TTL == nil && CacheCfg.Size == nil { + if cacheCfg.Strategy == nil && cacheCfg.TTL == nil && cacheCfg.Size == nil { return nil } // cache is enabled size := 50 - if CacheCfg.Size != nil { - size = *CacheCfg.Size + if cacheCfg.Size != nil { + size = *cacheCfg.Size } gc := gcache.New(size) strategy := "LRU" - if CacheCfg.Strategy != nil { - strategy = *CacheCfg.Strategy + if cacheCfg.Strategy != nil { + strategy = *cacheCfg.Strategy } switch strategy { @@ -165,8 +167,8 @@ func RegexpCacheInit(filename string, CacheCfg types.DataSource) error { return fmt.Errorf("unknown cache strategy '%s'", strategy) } - if CacheCfg.TTL != nil { - gc.Expiration(*CacheCfg.TTL) + if cacheCfg.TTL != nil { + gc.Expiration(*cacheCfg.TTL) } cache := gc.Build() @@ -240,6 +242,7 @@ func Distinct(params ...any) (any, error) { if rt := reflect.TypeOf(params[0]).Kind(); rt != reflect.Slice && rt != reflect.Array { return nil, nil } + array := params[0].([]interface{}) if array == nil { return []interface{}{}, nil @@ -254,6 +257,7 @@ func Distinct(params ...any) (any, error) { ret = append(ret, val) } } + return ret, nil } @@ -282,8 +286,10 @@ func flatten(args []interface{}, v reflect.Value) []interface{} { } func existsInFileMaps(filename string, ftype string) (bool, error) { - ok := false var err error + + ok := false + switch ftype { case "regex", "regexp": if fflag.Re2RegexpInfileSupport.IsEnabled() { @@ -296,10 +302,11 @@ func existsInFileMaps(filename string, ftype string) (bool, error) { default: err = fmt.Errorf("unknown data type '%s' for : '%s'", ftype, filename) } + return ok, err } -//Expr helpers +// Expr helpers // func Get(arr []string, index int) string { func Get(params ...any) (any, error) { @@ -315,10 +322,12 @@ func Get(params ...any) (any, error) { func Atof(params ...any) (any, error) { x := params[0].(string) log.Debugf("debug atof %s", x) + ret, err := strconv.ParseFloat(x, 64) if err != nil { log.Warningf("Atof : can't convert float '%s' : %v", x, err) } + return ret, nil } @@ -340,22 +349,28 @@ func Distance(params ...any) (any, error) { long1 := params[1].(string) lat2 := params[2].(string) long2 := params[3].(string) + lat1f, err := strconv.ParseFloat(lat1, 64) if err != nil { log.Warningf("lat1 is not a float : %v", err) + return 0.0, fmt.Errorf("lat1 is not a float : %v", err) } + long1f, err := strconv.ParseFloat(long1, 64) if err != nil { log.Warningf("long1 is not a float : %v", err) + return 0.0, fmt.Errorf("long1 is not a float : %v", err) } + lat2f, err := strconv.ParseFloat(lat2, 64) if err != nil { log.Warningf("lat2 is not a float : %v", err) return 0.0, fmt.Errorf("lat2 is not a float : %v", err) } + long2f, err := strconv.ParseFloat(long2, 64) if err != nil { log.Warningf("long2 is not a float : %v", err) @@ -363,7 +378,7 @@ func Distance(params ...any) (any, error) { return 0.0, fmt.Errorf("long2 is not a float : %v", err) } - //either set of coordinates is 0,0, return 0 to avoid FPs + // either set of coordinates is 0,0, return 0 to avoid FPs if (lat1f == 0.0 && long1f == 0.0) || (lat2f == 0.0 && long2f == 0.0) { log.Warningf("one of the coordinates is 0,0, returning 0") return 0.0, nil @@ -373,6 +388,7 @@ func Distance(params ...any) (any, error) { second := haversine.Coord{Lat: lat2f, Lon: long2f} _, km := haversine.Distance(first, second) + return km, nil } diff --git a/pkg/fflag/features_test.go b/pkg/fflag/features_test.go index 144e7049362..bf8ddeca8fd 100644 --- a/pkg/fflag/features_test.go +++ b/pkg/fflag/features_test.go @@ -351,11 +351,9 @@ func TestSetFromYaml(t *testing.T) { } func TestSetFromYamlFile(t *testing.T) { - tmpfile, err := os.CreateTemp("", "test") + tmpfile, err := os.CreateTemp(t.TempDir(), "test") require.NoError(t, err) - defer os.Remove(tmpfile.Name()) - // write the config file _, err = tmpfile.WriteString("- experimental1") require.NoError(t, err) diff --git a/pkg/leakybucket/bayesian.go b/pkg/leakybucket/bayesian.go index 357d51f597b..30e1b396ef8 100644 --- a/pkg/leakybucket/bayesian.go +++ b/pkg/leakybucket/bayesian.go @@ -31,9 +31,9 @@ type BayesianBucket struct { DumbProcessor } -func updateProbability(prior, probGivenEvil, ProbGivenBenign float32) float32 { +func updateProbability(prior, probGivenEvil, probGivenBenign float32) float32 { numerator := probGivenEvil * prior - denominator := numerator + ProbGivenBenign*(1-prior) + denominator := numerator + probGivenBenign*(1-prior) return numerator / denominator } diff --git a/pkg/leakybucket/overflow_filter.go b/pkg/leakybucket/overflow_filter.go index 01dd491ed41..b37e431fadf 100644 --- a/pkg/leakybucket/overflow_filter.go +++ b/pkg/leakybucket/overflow_filter.go @@ -36,10 +36,10 @@ func NewOverflowFilter(g *BucketFactory) (*OverflowFilter, error) { return &u, nil } -func (u *OverflowFilter) OnBucketOverflow(Bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *types.Queue) (types.RuntimeAlert, *types.Queue) { +func (u *OverflowFilter) OnBucketOverflow(bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *types.Queue) (types.RuntimeAlert, *types.Queue) { return func(l *Leaky, s types.RuntimeAlert, q *types.Queue) (types.RuntimeAlert, *types.Queue) { el, err := exprhelpers.Run(u.FilterRuntime, map[string]interface{}{ - "queue": q, "signal": s, "leaky": l}, l.logger, Bucket.Debug) + "queue": q, "signal": s, "leaky": l}, l.logger, bucket.Debug) if err != nil { l.logger.Errorf("Failed running overflow filter: %s", err) return s, q diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index 475f3af0928..72356bc1924 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -60,9 +60,14 @@ func TestSetupHelperProcess(t *testing.T) { func tempYAML(t *testing.T, content string) os.File { t.Helper() require := require.New(t) - file, err := os.CreateTemp("", "") + file, err := os.CreateTemp(t.TempDir(), "") require.NoError(err) + t.Cleanup(func() { + require.NoError(file.Close()) + require.NoError(os.Remove(file.Name())) + }) + _, err = file.WriteString(dedent.Dedent(content)) require.NoError(err) @@ -249,7 +254,6 @@ func TestListSupported(t *testing.T) { t.Parallel() f := tempYAML(t, tc.yml) - defer os.Remove(f.Name()) supported, err := setup.ListSupported(&f) cstest.RequireErrorContains(t, err, tc.expectedErr) @@ -375,7 +379,6 @@ func TestDetectSimpleRule(t *testing.T) { - false ugly: `) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{}) require.NoError(err) @@ -421,7 +424,6 @@ detect: for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := tempYAML(t, tc.config) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{}) cstest.RequireErrorContains(t, err, tc.expectedErr) @@ -514,7 +516,6 @@ detect: for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := tempYAML(t, tc.config) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{}) cstest.RequireErrorContains(t, err, tc.expectedErr) @@ -542,7 +543,6 @@ func TestDetectForcedUnit(t *testing.T) { journalctl_filter: - _SYSTEMD_UNIT=crowdsec-setup-forced.service `) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{ForcedUnits: []string{"crowdsec-setup-forced.service"}}) require.NoError(err) @@ -580,7 +580,6 @@ func TestDetectForcedProcess(t *testing.T) { when: - ProcessRunning("foobar") `) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}}) require.NoError(err) @@ -610,7 +609,6 @@ func TestDetectSkipService(t *testing.T) { when: - ProcessRunning("foobar") `) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}, SkipServices: []string{"wizard"}}) require.NoError(err) @@ -825,7 +823,6 @@ func TestDetectForcedOS(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := tempYAML(t, tc.config) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{ForcedOS: tc.forced}) cstest.RequireErrorContains(t, err, tc.expectedErr) @@ -1009,7 +1006,6 @@ func TestDetectDatasourceValidation(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := tempYAML(t, tc.config) - defer os.Remove(f.Name()) detected, err := setup.Detect(&f, setup.DetectOptions{}) cstest.RequireErrorContains(t, err, tc.expectedErr) require.Equal(tc.expected, detected) diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index ac438ad0c14..c24b3f2ac0d 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -143,18 +143,15 @@ rm -rf %{buildroot} #systemctl stop crowdsec || true -if [ $1 == 2 ];then - if [[ ! -d /var/lib/crowdsec/backup ]]; then - cscli config backup /var/lib/crowdsec/backup - fi -fi +#if [ $1 == 2 ]; then +# upgrade pre-install here +#fi %post -p /bin/bash #install if [ $1 == 1 ]; then - if [ ! -f "/var/lib/crowdsec/data/crowdsec.db" ] ; then touch /var/lib/crowdsec/data/crowdsec.db fi @@ -185,21 +182,6 @@ if [ $1 == 1 ]; then echo " * Detailed guides are available in our documentation: https://docs.crowdsec.net" echo " * Configuration items created by the community can be found at the Hub: https://hub.crowdsec.net" echo " * Gain insights into your use of CrowdSec with the help of the console https://app.crowdsec.net" - -#upgrade -elif [ $1 == 2 ] && [ -d /var/lib/crowdsec/backup ]; then - cscli config restore /var/lib/crowdsec/backup - if [ $? == 0 ]; then - rm -rf /var/lib/crowdsec/backup - fi - - if [[ -f %{_sysconfdir}/crowdsec/online_api_credentials.yaml ]] ; then - chmod 600 %{_sysconfdir}/crowdsec/online_api_credentials.yaml - fi - - if [[ -f %{_sysconfdir}/crowdsec/local_api_credentials.yaml ]] ; then - chmod 600 %{_sysconfdir}/crowdsec/local_api_credentials.yaml - fi fi %systemd_post %{name}.service diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index 63c204a9e86..9af3c841759 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -172,41 +172,13 @@ teardown() { } @test "cscli config backup / restore" { - # test that we need a valid path - # disabled because in CI, the empty string is not passed as a parameter - #rune -1 cscli config backup "" - #assert_stderr --partial "failed to backup config: directory path can't be empty" + CONFIG_DIR=$(config_get '.config_paths.config_dir') rune -1 cscli config backup "/dev/null/blah" - assert_stderr --partial "failed to backup config: while creating /dev/null/blah: mkdir /dev/null/blah: not a directory" + assert_stderr --partial "'cscli config backup' has been removed, you can manually backup/restore $CONFIG_DIR instead" - # pick a dirpath - backupdir=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp -u) - - # succeed the first time - rune -0 cscli config backup "$backupdir" - assert_stderr --partial "Starting configuration backup" - - # don't overwrite an existing backup - rune -1 cscli config backup "$backupdir" - assert_stderr --partial "failed to backup config" - assert_stderr --partial "file exists" - - SIMULATION_YAML="$(config_get '.config_paths.simulation_path')" - - # restore - rm "$SIMULATION_YAML" - rune -0 cscli config restore "$backupdir" - assert_file_exists "$SIMULATION_YAML" - - # cleanup - rm -rf -- "${backupdir:?}" - - # backup: detect missing files - rm "$SIMULATION_YAML" - rune -1 cscli config backup "$backupdir" - assert_stderr --regexp "failed to backup config: failed copy .* to .*: stat .*: no such file or directory" - rm -rf -- "${backupdir:?}" + rune -1 cscli config restore "/dev/null/blah" + assert_stderr --partial "'cscli config restore' has been removed, you can manually backup/restore $CONFIG_DIR instead" } @test "'cscli completion' with or without configuration file" { diff --git a/test/bats/02_nolapi.bats b/test/bats/02_nolapi.bats index cefa6d798b4..70495a0ed91 100644 --- a/test/bats/02_nolapi.bats +++ b/test/bats/02_nolapi.bats @@ -66,18 +66,6 @@ teardown() { refute_output --partial "Local API Server" } -@test "cscli config backup" { - config_disable_lapi - backupdir=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp -u) - rune -0 cscli config backup "$backupdir" - assert_stderr --partial "Starting configuration backup" - rune -1 cscli config backup "$backupdir" - rm -rf -- "${backupdir:?}" - - assert_stderr --partial "failed to backup config" - assert_stderr --partial "file exists" -} - @test "lapi status shouldn't be ok without api.server" { config_disable_lapi rune -1 cscli machines list diff --git a/test/bats/03_noagent.bats b/test/bats/03_noagent.bats index 6be5101cee2..972b84977ad 100644 --- a/test/bats/03_noagent.bats +++ b/test/bats/03_noagent.bats @@ -60,18 +60,6 @@ teardown() { refute_output --partial "Crowdsec" } -@test "no agent: cscli config backup" { - config_disable_agent - backupdir=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp -u) - rune -0 cscli config backup "$backupdir" - assert_stderr --partial "Starting configuration backup" - rune -1 cscli config backup "$backupdir" - - assert_stderr --partial "failed to backup config" - assert_stderr --partial "file exists" - rm -rf -- "${backupdir:?}" -} - @test "no agent: lapi status should be ok" { config_disable_agent ./instance-crowdsec start diff --git a/test/bats/04_nocapi.bats b/test/bats/04_nocapi.bats index 8d0018a9a4a..eaeb0939112 100644 --- a/test/bats/04_nocapi.bats +++ b/test/bats/04_nocapi.bats @@ -51,17 +51,6 @@ teardown() { assert_output --regexp "Global:.*Crowdsec.*cscli:.*Local API Server:" } -@test "no agent: cscli config backup" { - config_disable_capi - backupdir=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp -u) - rune -0 cscli config backup "$backupdir" - assert_stderr --partial "Starting configuration backup" - rune -1 cscli config backup "$backupdir" - assert_stderr --partial "failed to backup config" - assert_stderr --partial "file exists" - rm -rf -- "${backupdir:?}" -} - @test "without capi: cscli lapi status -> success" { config_disable_capi ./instance-crowdsec start diff --git a/wizard.sh b/wizard.sh index 57311d40cdb..4da970cd695 100755 --- a/wizard.sh +++ b/wizard.sh @@ -21,11 +21,8 @@ DOCKER_MODE="false" CROWDSEC_LIB_DIR="/var/lib/crowdsec" CROWDSEC_USR_DIR="/usr/local/lib/crowdsec" CROWDSEC_DATA_DIR="${CROWDSEC_LIB_DIR}/data" -CROWDSEC_DB_PATH="${CROWDSEC_DATA_DIR}/crowdsec.db" CROWDSEC_PATH="/etc/crowdsec" CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}" -CROWDSEC_LOG_FILE="/var/log/crowdsec.log" -LAPI_LOG_FILE="/var/log/crowdsec_api.log" CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins" CROWDSEC_CONSOLE_DIR="${CROWDSEC_PATH}/console" @@ -35,8 +32,6 @@ CSCLI_BIN="./cmd/crowdsec-cli/cscli" CLIENT_SECRETS="local_api_credentials.yaml" LAPI_SECRETS="online_api_credentials.yaml" -CONSOLE_FILE="console.yaml" - BIN_INSTALL_PATH="/usr/local/bin" CROWDSEC_BIN_INSTALLED="${BIN_INSTALL_PATH}/crowdsec" @@ -91,9 +86,6 @@ SENTINEL_PLUGIN_CONFIG="./cmd/notification-sentinel/sentinel.yaml" FILE_PLUGIN_CONFIG="./cmd/notification-file/file.yaml" -BACKUP_DIR=$(mktemp -d) -rm -rf -- "$BACKUP_DIR" - log_info() { msg=$1 date=$(date "+%Y-%m-%d %H:%M:%S") @@ -426,27 +418,20 @@ install_crowdsec() { mkdir -p "${CROWDSEC_CONFIG_PATH}/contexts" || exit mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit - # tmp - mkdir -p /tmp/data mkdir -p /etc/crowdsec/hub/ - install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - - ## end tmp - install -v -m 600 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/"${CONSOLE_FILE}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit + # Don't overwrite existing files + [[ ! -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" ]] && install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" >/dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/${LAPI_SECRETS}" ]] && install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/config.yaml" ]] && install -v -m 600 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/dev.yaml" ]] && install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/user.yaml" ]] && install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/acquis.yaml" ]] && install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/profiles.yaml" ]] && install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/simulation.yaml" ]] && install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/console.yaml" ]] && install -v -m 644 -D ./config/console.yaml "${CROWDSEC_CONFIG_PATH}" > /dev/null || exit + [[ ! -f "${CROWDSEC_CONFIG_PATH}/context.yaml" ]] && install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" > /dev/null || exit - DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file" - if [[ ${DOCKER_MODE} == "false" ]]; then - CFG=${CROWDSEC_CONFIG_PATH} BIN=${CROWDSEC_BIN_INSTALLED} envsubst '$CFG $BIN' < ./config/crowdsec.service > "${SYSTEMD_PATH_FILE}" || log_fatal "unable to crowdsec systemd file" - fi install_bins if [[ ${DOCKER_MODE} == "false" ]]; then @@ -471,23 +456,12 @@ update_full() { log_err "Cscli binary '$CSCLI_BIN' not found. Please build it with 'make build'" && exit fi - log_info "Backing up existing configuration" - ${CSCLI_BIN_INSTALLED} config backup ${BACKUP_DIR} - log_info "Saving default database content if exist" - if [[ -f "/var/lib/crowdsec/data/crowdsec.db" ]]; then - cp /var/lib/crowdsec/data/crowdsec.db ${BACKUP_DIR}/crowdsec.db - fi - log_info "Cleanup existing crowdsec configuration" + log_info "Removing old binaries" uninstall_crowdsec log_info "Installing crowdsec" install_crowdsec - log_info "Restoring configuration" + log_info "Updating hub" ${CSCLI_BIN_INSTALLED} hub update - ${CSCLI_BIN_INSTALLED} config restore ${BACKUP_DIR} - log_info "Restoring saved database if exist" - if [[ -f "${BACKUP_DIR}/crowdsec.db" ]]; then - cp ${BACKUP_DIR}/crowdsec.db /var/lib/crowdsec/data/crowdsec.db - fi log_info "Finished, restarting" systemctl restart crowdsec || log_fatal "Failed to restart crowdsec" } @@ -565,15 +539,6 @@ uninstall_crowdsec() { ${CSCLI_BIN} dashboard remove -f -y >/dev/null delete_bins - # tmp - rm -rf /tmp/data/ - ## end tmp - - find /etc/crowdsec -maxdepth 1 -mindepth 1 | grep -v "bouncer" | xargs rm -rf || echo "" - rm -f ${CROWDSEC_LOG_FILE} || echo "" - rm -f ${LAPI_LOG_FILE} || echo "" - rm -f ${CROWDSEC_DB_PATH} || echo "" - rm -rf ${CROWDSEC_LIB_DIR} || echo "" rm -rf ${CROWDSEC_USR_DIR} || echo "" rm -f ${SYSTEMD_PATH_FILE} || echo "" log_info "crowdsec successfully uninstalled" @@ -765,12 +730,11 @@ usage() { echo " ./wizard.sh --unattended Install in unattended mode, no question will be asked and defaults will be followed" echo " ./wizard.sh --docker-mode Will install crowdsec without systemd and generate random machine-id" echo " ./wizard.sh -n|--noop Do nothing" - - exit 0 } if [[ $# -eq 0 ]]; then -usage + usage + exit 0 fi while [[ $# -gt 0 ]]