Refact pkg/cwhub: fix some known issues and reorganize files (#2616)

* bump gopkg.in/yaml.v3
* test: cannot remove local items with cscli
* test dangling links
* test: cannot install local item with cscli
* pkg/cwhub: reorg (move) functions in files
* allow hub upgrade with local items
* data download: honor Last-Modified header
* fatal -> warning when attempting to remove a local item (allows remove --all)
* cscli...inspect -o yaml|human: rename remote_path -> path
* Correct count of removed items
Still no separate counter for the --purge option, but should be clear enough

cmd/crowdsec-cli/item_suggest.go

@@ -12,10 +12,10 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-const MaxDistance = 7
+// suggestNearestMessage returns a message with the most similar item name, if one is found
+func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
+	const maxDistance = 7
 
-// SuggestNearestMessage returns a message with the most similar item name, if one is found
-func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
 	score := 100
 	nearest := ""
 
@@ -29,7 +29,7 @@ func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str
 
 	msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
 
-	if score < MaxDistance {
+	if score < maxDistance {
 		msg += fmt.Sprintf(", did you mean '%s'?", nearest)
 	}
 

cmd/crowdsec-cli/itemcommands.go

@@ -214,7 +214,7 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
 		for _, name := range args {
 			item := hub.GetItem(it.name, name)
 			if item == nil {
-				msg := SuggestNearestMessage(hub, it.name, name)
+				msg := suggestNearestMessage(hub, it.name, name)
 				if !ignoreError {
 					return fmt.Errorf(msg)
 				}
@@ -319,6 +319,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
 					return err
 				}
 				if didRemove {
+					log.Infof("Removed %s", item.Name)
 					removed++
 				}
 			}
@@ -361,6 +362,8 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
 				removed++
 			}
 		}
+
+		log.Infof("Removed %d %s", removed, it.name)
 		if removed > 0 {
 			log.Infof(ReloadMessage())
 		}

cmd/crowdsec-cli/utils.go

@@ -13,14 +13,12 @@ import (
 )
 
 func printHelp(cmd *cobra.Command) {
-	err := cmd.Help()
-	if err != nil {
+	if err := cmd.Help(); err != nil {
 		log.Fatalf("unable to print help(): %s", err)
 	}
 }
 
 func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
-
 	/*if a range is provided, change the scope*/
 	if *ipRange != "" {
 		_, _, err := net.ParseCIDR(*ipRange)
@@ -50,7 +48,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 }
 
 func getDBClient() (*database.Client, error) {
-	var err error
 	if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
 		return nil, err
 	}

pkg/cwhub/cwhub.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 )
@@ -30,3 +31,10 @@ func safePath(dir, filePath string) (string, error) {
 
 	return absFilePath, nil
 }
+
+// SortItemSlice sorts a slice of items by name, case insensitive.
+func SortItemSlice(items []*Item) {
+	sort.Slice(items, func(i, j int) bool {
+		return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
+	})
+}

pkg/cwhub/dataset.go

@@ -6,9 +6,10 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"time"
 
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
@@ -51,6 +52,62 @@ func downloadFile(url string, destPath string) error {
 	return nil
 }
 
+// needsUpdate checks if a data file has to be downloaded (or updated).
+// if the local file doesn't exist, update.
+// if the remote is newer than the local file, update.
+// if the remote has no modification date, but local file has been modified > a week ago, update.
+func needsUpdate(destPath string, url string) bool {
+	fileInfo, err := os.Stat(destPath)
+	switch {
+	case os.IsNotExist(err):
+		return true
+	case err != nil:
+		log.Errorf("while getting %s: %s", destPath, err)
+		return true
+	}
+
+	resp, err := hubClient.Head(url)
+	if err != nil {
+		log.Errorf("while getting %s: %s", url, err)
+		// Head failed, Get would likely fail too -> no update
+		return false
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		log.Errorf("bad http code %d for %s", resp.StatusCode, url)
+		return false
+	}
+
+	// update if local file is older than this
+	shelfLife := 7 * 24 * time.Hour
+
+	lastModify := fileInfo.ModTime()
+
+	localIsOld := lastModify.Add(shelfLife).Before(time.Now())
+
+	remoteLastModified := resp.Header.Get("Last-Modified")
+	if remoteLastModified == "" {
+		if localIsOld {
+			log.Infof("no last modified date for %s, but local file is older than %s", url, shelfLife)
+		}
+		return localIsOld
+	}
+
+	lastAvailable, err := time.Parse(time.RFC1123, remoteLastModified)
+	if err != nil {
+		log.Warningf("while parsing last modified date for %s: %s", url, err)
+		return localIsOld
+	}
+
+	if lastModify.Before(lastAvailable) {
+		log.Infof("new version available, updating %s", destPath)
+		return true
+	}
+
+	return false
+}
+
 // downloadDataSet downloads all the data files for an item.
 func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
 	dec := yaml.NewDecoder(reader)
@@ -72,9 +129,7 @@ func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
 				return err
 			}
 
-			if _, err := os.Stat(destPath); os.IsNotExist(err) || force {
-				log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
-
+			if force || needsUpdate(destPath, dataS.SourceURL) {
 				if err := downloadFile(dataS.SourceURL, destPath); err != nil {
 					return fmt.Errorf("while getting data: %w", err)
 				}