pkg/cwhub: improve support for k8s config maps as custom items

 - allow links to links
 - ignore hidden ..data directories, but allow links to their content

pkg/cwhub/sync.go

@@ -20,22 +20,49 @@ func isYAMLFileName(path string) bool {
 	return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
 }
 
-// linkTarget returns the target of a symlink, or empty string if it's dangling.
-func linkTarget(path string, logger *logrus.Logger) (string, error) {
-	hubpath, err := os.Readlink(path)
-	if err != nil {
-		return "", fmt.Errorf("unable to read symlink: %s", path)
+// resolveSymlink returns the ultimate target path of a symlink
+// returns error if the symlink is dangling or too many symlinks are followed
+func resolveSymlink(path string) (string, error) {
+	const maxSymlinks = 10 // Prevent infinite loops
+	for i := 0; i < maxSymlinks; i++ {
+		fi, err := os.Lstat(path)
+		if err != nil {
+			return "", err // dangling link
+		}
+
+		if fi.Mode()&os.ModeSymlink == 0 {
+			// found the target
+			return path, nil
+		}
+
+		path, err = os.Readlink(path)
+		if err != nil {
+			return "", err
+		}
+
+		// relative to the link's directory?
+		if !filepath.IsAbs(path) {
+			path = filepath.Join(filepath.Dir(path), path)
+		}
 	}
 
-	logger.Tracef("symlink %s -> %s", path, hubpath)
+	return "", errors.New("too many levels of symbolic links")
+}
 
-	_, err = os.Lstat(hubpath)
-	if os.IsNotExist(err) {
-		logger.Warningf("link target does not exist: %s -> %s", path, hubpath)
-		return "", nil
+// isPathInside checks if a path is inside the given directory
+// it can return false negatives if the filesystem is case insensitive
+func isPathInside(path, dir string) (bool, error) {
+	absFilePath, err := filepath.Abs(path)
+	if err != nil {
+		return false, err
 	}
 
-	return hubpath, nil
+	absDir, err := filepath.Abs(dir)
+	if err != nil {
+		return false, err
+	}
+
+	return strings.HasPrefix(absFilePath, absDir), nil
 }
 
 // information used to create a new Item, from a file path.
@@ -61,9 +88,9 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo
 	if strings.HasPrefix(path, hubDir) {
 		logger.Tracef("in hub dir")
 
-		// .../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
-		// .../hub/scenarios/crowdsec/ssh_bf.yaml
-		// .../hub/profiles/crowdsec/linux.yaml
+		// .../hub/parsers/s00-raw/crowdsecurity/skip-pretag.yaml
+		// .../hub/scenarios/crowdsecurity/ssh_bf.yaml
+		// .../hub/profiles/crowdsecurity/linux.yaml
 		if len(subs) < 4 {
 			return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
 		}
@@ -176,8 +203,6 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
 }
 
 func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
-	hubpath := ""
-
 	if err != nil {
 		h.logger.Debugf("while syncing hub dir: %s", err)
 		// there is a path error, we ignore the file
@@ -190,8 +215,26 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 		return err
 	}
 
+	// permission errors, files removed while reading, etc.
+	if f == nil {
+		return nil
+	}
+
+	if f.IsDir() {
+		// if a directory starts with a dot, we don't traverse it
+		// - single dot prefix is hidden by unix convention
+		// - double dot prefix is used by k8s to mount config maps
+		if strings.HasPrefix(f.Name(), ".") {
+			h.logger.Tracef("skipping hidden directory %s", path)
+			return filepath.SkipDir
+		}
+
+		// keep traversing
+		return nil
+	}
+
 	// we only care about YAML files
-	if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) {
+	if !isYAMLFileName(f.Name()) {
 		return nil
 	}
 
@@ -201,35 +244,38 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 		return nil
 	}
 
-	// non symlinks are local user files or hub files
-	if f.Type()&os.ModeSymlink == 0 {
-		h.logger.Tracef("%s is not a symlink", path)
-
-		if !info.inhub {
-			h.logger.Tracef("%s is a local file, skip", path)
+	// follow the link to see if it falls in the hub directory
+	// if it's not a link, target == path
+	target, err := resolveSymlink(path)
+	if err != nil {
+		// target does not exist, the user might have removed the file
+		// or switched to a hub branch without it; or symlink loop
+		h.logger.Warningf("Ignoring file %s: %s", path, err)
+		return nil
+	}
 
-			item, err := newLocalItem(h, path, info)
-			if err != nil {
-				return err
-			}
+	targetInHub, err := isPathInside(target, h.local.HubDir)
+	if err != nil {
+		h.logger.Warningf("Ignoring file %s: %s", path, err)
+		return nil
+	}
 
-			h.addItem(item)
+	// local (custom) item if the file or link target is not inside the hub dir
+	if !targetInHub {
+		h.logger.Tracef("%s is a local file, skip", path)
 
-			return nil
-		}
-	} else {
-		hubpath, err = linkTarget(path, h.logger)
+		item, err := newLocalItem(h, path, info)
 		if err != nil {
 			return err
 		}
 
-		if hubpath == "" {
-			// target does not exist, the user might have removed the file
-			// or switched to a hub branch without it
-			return nil
-		}
+		h.addItem(item)
+
+		return nil
 	}
 
+	hubpath := target
+
 	// try to find which configuration item it is
 	h.logger.Tracef("check [%s] of %s", info.fname, info.ftype)
 

test/bats/20_hub_items.bats

@@ -176,7 +176,7 @@ teardown() {
     rune -0 mkdir -p "$CONFIG_DIR/collections"
     rune -0 ln -s /this/does/not/exist.yaml "$CONFIG_DIR/collections/foobar.yaml"
     rune -0 cscli hub list
-    assert_stderr --partial "link target does not exist: $CONFIG_DIR/collections/foobar.yaml -> /this/does/not/exist.yaml"
+    assert_stderr --partial "Ignoring file $CONFIG_DIR/collections/foobar.yaml: lstat /this/does/not/exist.yaml: no such file or directory"
     rune -0 cscli hub list -o json
     rune -0 jq '.collections' <(output)
     assert_json '[]'
@@ -200,3 +200,63 @@ teardown() {
     rune -0 cscli hub list
     assert_stderr --partial "Ignoring file $CONFIG_DIR/scenarios/foo/bar.yaml: unknown configuration type"
 }
+
+@test "don't traverse hidden directories (starting with a dot)" {
+    rune -0 mkdir -p "$CONFIG_DIR/scenarios/.foo"
+    rune -0 touch "$CONFIG_DIR/scenarios/.foo/bar.yaml"
+    rune -0 cscli hub list --trace
+    assert_stderr --partial "skipping hidden directory $CONFIG_DIR/scenarios/.foo"
+}
+
+@test "allow symlink to target inside a hidden directory" {
+    # k8s config maps use hidden directories and links when mounted
+    rune -0 mkdir -p "$CONFIG_DIR/scenarios/.foo"
+
+    # ignored
+    rune -0 touch "$CONFIG_DIR/scenarios/.foo/hidden.yaml"
+    rune -0 cscli scenarios list -o json
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output 0
+
+    # real file
+    rune -0 touch "$CONFIG_DIR/scenarios/myfoo.yaml"
+    rune -0 cscli scenarios list -o json
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output 1
+
+    rune -0 rm "$CONFIG_DIR/scenarios/myfoo.yaml"
+    rune -0 cscli scenarios list -o json
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output 0
+
+    # link to ignored is not ignored
+    rune -0 ln -s "$CONFIG_DIR/scenarios/.foo/hidden.yaml" "$CONFIG_DIR/scenarios/myfoo.yaml"
+    rune -0 cscli scenarios list -o json
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output 1
+}
+
+@test "item files can be links to links" {
+    rune -0 mkdir -p "$CONFIG_DIR"/scenarios/{.foo,.bar}
+
+    rune -0 ln -s "$CONFIG_DIR/scenarios/.foo/hidden.yaml" "$CONFIG_DIR/scenarios/.bar/hidden.yaml"
+
+    # link to a danling link
+    rune -0 ln -s "$CONFIG_DIR/scenarios/.bar/hidden.yaml" "$CONFIG_DIR/scenarios/myfoo.yaml"
+    rune -0 cscli scenarios list
+    assert_stderr --partial "Ignoring file $CONFIG_DIR/scenarios/myfoo.yaml: lstat $CONFIG_DIR/scenarios/.foo/hidden.yaml: no such file or directory"
+    rune -0 cscli scenarios list -o json
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output 0
+
+    # detect link loops
+    rune -0 ln -s "$CONFIG_DIR/scenarios/.bar/hidden.yaml" "$CONFIG_DIR/scenarios/.foo/hidden.yaml"
+    rune -0 cscli scenarios list
+    assert_stderr --partial "Ignoring file $CONFIG_DIR/scenarios/myfoo.yaml: too many levels of symbolic links"
+
+    rune -0 rm "$CONFIG_DIR/scenarios/.foo/hidden.yaml"
+    rune -0 touch "$CONFIG_DIR/scenarios/.foo/hidden.yaml"
+    rune -0 cscli scenarios list -o json
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output 1
+}