Fix diagnostic scrubbing errors (#3165)

* fix diagnostic scrubbing errors

* add changelog

* spelling

* notice

* notice, again

* add tests, cleanup

* change key scrubbing test

NOTICE.txt

@@ -6875,6 +6875,66 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v2@v2.4.0/LICENSE:
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : gopkg.in/yaml.v3
+Version: v3.0.1
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.1/LICENSE:
+
+
+This project is covered by two different licenses: MIT and Apache.
+
+#### MIT License ####
+
+The following files were ported to Go from C files of libyaml, and thus
+are still covered by their original MIT license, with the additional
+copyright staring in 2011 when the project was ported over:
+
+    apic.go emitterc.go parserc.go readerc.go scannerc.go
+    writerc.go yamlh.go yamlprivateh.go
+
+Copyright (c) 2006-2010 Kirill Simonov
+Copyright (c) 2006-2011 Kirill Simonov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+### Apache License ###
+
+All the remaining project files are covered by the Apache license:
+
+Copyright (c) 2011-2019 Canonical Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
 --------------------------------------------------------------------------------
 Dependency : gotest.tools
 Version: v2.2.0+incompatible
@@ -17071,66 +17131,6 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
---------------------------------------------------------------------------------
-Dependency : gopkg.in/yaml.v3
-Version: v3.0.1
-Licence type (autodetected): MIT
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.1/LICENSE:
-
-
-This project is covered by two different licenses: MIT and Apache.
-
-#### MIT License ####
-
-The following files were ported to Go from C files of libyaml, and thus
-are still covered by their original MIT license, with the additional
-copyright staring in 2011 when the project was ported over:
-
-    apic.go emitterc.go parserc.go readerc.go scannerc.go
-    writerc.go yamlh.go yamlprivateh.go
-
-Copyright (c) 2006-2010 Kirill Simonov
-Copyright (c) 2006-2011 Kirill Simonov
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-### Apache License ###
-
-All the remaining project files are covered by the Apache license:
-
-Copyright (c) 2011-2019 Canonical Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
 --------------------------------------------------------------------------------
 Dependency : gotest.tools/v3
 Version: v3.4.0

changelog/fragments/1690916478-fix-cred-scrubbing.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fix credential redaction in diagnostic bundle collection
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: diagnostics
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

go.mod

@@ -62,6 +62,7 @@ require (
 	google.golang.org/grpc v1.53.0
 	google.golang.org/protobuf v1.29.1
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools v2.2.0+incompatible
 	gotest.tools/gotestsum v1.7.0
 	k8s.io/api v0.23.4
@@ -144,7 +145,6 @@ require (
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
 	google.golang.org/grpc/examples v0.0.0-20220304170021-431ea809a767 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	howett.net/plist v1.0.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect

internal/pkg/agent/cmd/diagnostics.go

@@ -65,17 +65,21 @@ func diagnosticCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 
 	agentDiag, err := daemon.DiagnosticAgent(ctx, additionalDiags)
 	if err != nil {
-		return fmt.Errorf("failed to fetch agent diagnostics: %w", err)
+		fmt.Fprintf(streams.Err, "[WARNING]: failed to fetch agent diagnostics: %s", err)
 	}
 
 	unitDiags, err := daemon.DiagnosticUnits(ctx)
 	if err != nil {
-		return fmt.Errorf("failed to fetch unit diagnostics: %w", err)
+		fmt.Fprintf(streams.Err, "[WARNING]: failed to fetch unit diagnostics: %s", err)
 	}
 
 	compDiags, err := daemon.DiagnosticComponents(ctx, additionalDiags)
 	if err != nil {
-		return fmt.Errorf("failed to fetch component diagnostics: %w", err)
+		fmt.Fprintf(streams.Err, "[WARNING]: failed to fetch component diagnostics: %s", err)
+	}
+
+	if len(compDiags) == 0 && len(unitDiags) == 0 && len(agentDiag) == 0 {
+		return fmt.Errorf("no diags could be fetched")
 	}
 
 	f, err := os.Create(fileName)

internal/pkg/diagnostics/diagnostics.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/elastic/elastic-agent/pkg/control/v2/client"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/release"
@@ -303,18 +303,18 @@ func writeErrorResult(zw *zip.Writer, path string, errBody string) error {
 	return nil
 }
 
-func writeRedacted(errOut, w io.Writer, fullFilePath string, fr client.DiagnosticFileResult) error {
-	out := &fr.Content
+func writeRedacted(errOut, resultWriter io.Writer, fullFilePath string, fileResult client.DiagnosticFileResult) error {
+	out := &fileResult.Content
 
 	// Should we support json too?
-	if fr.ContentType == "application/yaml" {
+	if fileResult.ContentType == "application/yaml" {
 		unmarshalled := map[interface{}]interface{}{}
-		err := yaml.Unmarshal(fr.Content, &unmarshalled)
+		err := yaml.Unmarshal(fileResult.Content, &unmarshalled)
 		if err != nil {
 			// Best effort, output a warning but still include the file
 			fmt.Fprintf(errOut, "[WARNING] Could not redact %s due to unmarshalling error: %s\n", fullFilePath, err)
 		} else {
-			redacted, err := yaml.Marshal(redactMap(unmarshalled))
+			redacted, err := yaml.Marshal(redactMap(errOut, unmarshalled))
 			if err != nil {
 				// Best effort, output a warning but still include the file
 				fmt.Fprintf(errOut, "[WARNING] Could not redact %s due to marshalling error: %s\n", fullFilePath, err)
@@ -324,23 +324,46 @@ func writeRedacted(errOut, w io.Writer, fullFilePath string, fr client.Diagnosti
 		}
 	}
 
-	_, err := w.Write(*out)
+	_, err := resultWriter.Write(*out)
 	return err
 }
 
-func redactMap(m map[interface{}]interface{}) map[interface{}]interface{} {
-	for k, v := range m {
-		if v != nil && reflect.TypeOf(v).Kind() == reflect.Map {
-			v = redactMap(v.(map[interface{}]interface{}))
-		}
-		if s, ok := k.(string); ok {
-			if redactKey(s) {
-				v = REDACTED
+// redactMap sensitive values from the underlying map
+// the whole generic function here is out of paranoia. Although extremely unlikely,
+// we have no way of guaranteeing we'll get a "normal" map[string]interface{},
+// since the diagnostic interface is a bit of a free-for-all
+func redactMap[K comparable](errOut io.Writer, inputMap map[K]interface{}) map[K]interface{} {
+	if inputMap == nil {
+		return nil
+	}
+	for rootKey, rootValue := range inputMap {
+		if rootValue != nil {
+			switch cast := rootValue.(type) {
+			case map[string]interface{}:
+				rootValue = redactMap(errOut, cast)
+			case map[interface{}]interface{}:
+				rootValue = redactMap(errOut, cast)
+			case map[int]interface{}:
+				rootValue = redactMap(errOut, cast)
+			case string:
+				if keyString, ok := any(rootKey).(string); ok {
+					if redactKey(keyString) {
+						rootValue = REDACTED
+					}
+				}
+			default:
+				// in cases where we got some weird kind of map we couldn't parse, print a warning
+				if reflect.TypeOf(rootValue).Kind() == reflect.Map {
+					fmt.Fprintf(errOut, "[WARNING]: file may be partly redacted, could not cast value %v of type %T", rootKey, rootValue)
+				}
+
 			}
-			m[k] = v
 		}
+
+		inputMap[rootKey] = rootValue
+
 	}
-	return m
+	return inputMap
 }
 
 func redactKey(k string) bool {