Fix error spam on AKS (#33697) (#33703)

This was happening due to the error level logging when the log path matcher detected a `log.file.path` that does not start with a standard Docker container log folder `/var/lib/docker/containers` because AKS dropped support for Docker in September 2022 and switched to containerd. It looks like this message was not supposed to be on the error level in the first place since it just means that the matcher didn't match and it's not an error. But it was mistakenly promoted from the debug level in #16866 most likely because the message started with `Error` and looked confusing. This a partial fix to unblock our customers, but we still need to come up with the full AKS/containerd support in a follow up change. (cherry picked from commit 29f0b4c) Co-authored-by: Denis <denis.rechkunov@elastic.co>
elastic · Nov 17, 2022 · 75e80d7 · 75e80d7
1 parent fc2f198
commit 75e80d7
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 31 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -16,6 +16,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 
 *Filebeat*
 
+- Fixed error spam from `add_kubernetes_metadata` processor when running on AKS. {pull}33697[33697]
 
 *Heartbeat*
 
@@ -218,31 +219,3 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 *Functionbeat*
 
 ==== Known Issue
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/filebeat/processor/add_kubernetes_metadata/matchers.go b/filebeat/processor/add_kubernetes_metadata/matchers.go
@@ -62,7 +62,7 @@ func newLogsPathMatcher(cfg conf.C) (add_kubernetes_metadata.Matcher, error) {
 
 	err := cfg.Unpack(&config)
 	if err != nil || config.LogsPath == "" {
-		return nil, fmt.Errorf("fail to unpack the `logs_path` configuration: %s", err)
+		return nil, fmt.Errorf("fail to unpack the `logs_path` configuration: %w", err)
 	}
 
 	logPath := config.LogsPath
@@ -92,7 +92,7 @@ func (f *LogPathMatcher) MetadataIndex(event mapstr.M) string {
 	f.logger.Debugf("Incoming log.file.path value: %s", source)
 
 	if !strings.Contains(source, f.LogsPath) {
-		f.logger.Errorf("Error extracting container id - source value does not contain matcher's logs_path '%s'.", f.LogsPath)
+		f.logger.Debugf("log.file.path value does not contain matcher's logs_path '%s', skipping...", f.LogsPath)
 		return ""
 	}
 
@@ -102,7 +102,7 @@ func (f *LogPathMatcher) MetadataIndex(event mapstr.M) string {
 	if f.ResourceType == "pod" {
 		// Pod resource type will extract only the pod UID, which offers less granularity of metadata when compared to the container ID
 		if strings.Contains(source, ".log") && !strings.HasSuffix(source, ".gz") {
-			// Specify a pod resource type when writting logs into manually mounted log volume,
+			// Specify a pod resource type when writing logs into manually mounted log volume,
 			// those logs apper under under "/var/lib/kubelet/pods/<pod_id>/volumes/..."
 			if strings.HasPrefix(f.LogsPath, podKubeletLogsPath()) {
 				pathDirs := strings.Split(source, pathSeparator)