Fix the bug that the default configs of slice/map are not overridden

CHANGELOG.md

@@ -9,7 +9,7 @@
 - Add an option `enable_fetch_replicaset` to control whether to fetch ReplicaSet metadata. The default value is false which aims to release pressure on Kubernetes API server. ([#492](https://github.com/KindlingProject/kindling/pull/492))
 
 ### Bug fixes
-
+- Fix the bug that the default configs of slice/map are not overridden. ([#497](https://github.com/KindlingProject/kindling/pull/497))
 
 ## v0.7.1 - 2023-03-01
 ### New features

collector/internal/application/factory.go

@@ -1,13 +1,15 @@
 package application
 
 import (
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+
 	"github.com/Kindling-project/kindling/collector/pkg/component"
 	"github.com/Kindling-project/kindling/collector/pkg/component/analyzer"
 	"github.com/Kindling-project/kindling/collector/pkg/component/consumer"
 	"github.com/Kindling-project/kindling/collector/pkg/component/consumer/exporter"
 	"github.com/Kindling-project/kindling/collector/pkg/component/consumer/processor"
 	"github.com/Kindling-project/kindling/collector/pkg/component/receiver"
-	"github.com/spf13/viper"
 )
 
 const (
@@ -103,37 +105,44 @@ func (c *ComponentsFactory) RegisterExporter(
 	}
 }
 
+// mapStructureDecoderConfigFunc is a function that is used to configure the mapstructure decoder.
+// ZeroFields option is set to true to allow the map/slice in the configuration file to override
+// the default values in the config struct.
+var mapStructureDecoderConfigFunc = func(dc *mapstructure.DecoderConfig) {
+	dc.ZeroFields = true
+}
+
 func (c *ComponentsFactory) ConstructConfig(viper *viper.Viper) error {
 	for _, componentKind := range ComponentsKeyMap {
 		switch componentKind {
 		case ReceiversKey:
 			for k, factory := range c.Receivers {
 				key := ReceiversKey + "." + k
-				err := viper.UnmarshalKey(key, factory.Config)
+				err := viper.UnmarshalKey(key, factory.Config, mapStructureDecoderConfigFunc)
 				if err != nil {
 					return err
 				}
 			}
 		case AnalyzersKey:
 			for k, factory := range c.Analyzers {
 				key := AnalyzersKey + "." + k
-				err := viper.UnmarshalKey(key, factory.Config)
+				err := viper.UnmarshalKey(key, factory.Config, mapStructureDecoderConfigFunc)
 				if err != nil {
 					return err
 				}
 			}
 		case ProcessorsKey:
 			for k, factory := range c.Processors {
 				key := ProcessorsKey + "." + k
-				err := viper.UnmarshalKey(key, factory.Config)
+				err := viper.UnmarshalKey(key, factory.Config, mapStructureDecoderConfigFunc)
 				if err != nil {
 					return err
 				}
 			}
 		case ExportersKey:
 			for k, factory := range c.Exporters {
 				key := ExportersKey + "." + k
-				err := viper.UnmarshalKey(key, factory.Config)
+				err := viper.UnmarshalKey(key, factory.Config, mapStructureDecoderConfigFunc)
 				if err != nil {
 					return err
 				}

collector/internal/application/factory_test.go

@@ -1,28 +1,66 @@
 package application
 
 import (
-	"reflect"
 	"testing"
 
-	"github.com/Kindling-project/kindling/collector/pkg/component/consumer/processor/k8sprocessor"
 	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/Kindling-project/kindling/collector/pkg/component/analyzer/network"
+	"github.com/Kindling-project/kindling/collector/pkg/component/consumer/processor/k8sprocessor"
 )
 
 func TestConstructConfig(t *testing.T) {
 	factory := NewComponentsFactory()
 	factory.RegisterProcessor(k8sprocessor.K8sMetadata, k8sprocessor.NewKubernetesProcessor, &k8sprocessor.DefaultConfig)
+	factory.RegisterAnalyzer(network.Network.String(), network.NewNetworkAnalyzer, network.NewDefaultConfig())
+
+	// Construct the config from the yaml file
 	v := viper.New()
-	v.SetConfigFile("testdata/kindling-collector-config.yaml")
-	v.ReadInConfig()
-	factory.ConstructConfig(v)
-	k8sprocessorFactory := factory.Processors[k8sprocessor.K8sMetadata]
-	cfg := k8sprocessorFactory.Config.(*k8sprocessor.Config)
+	v.SetConfigFile("./testdata/kindling-collector-config.yaml")
+	err := v.ReadInConfig()
+	assert.NoError(t, err)
+
+	err = factory.ConstructConfig(v)
+	assert.NoError(t, err)
+
+	//// Assert the config is as expected
+	k8sProcessorFactory := factory.Processors[k8sprocessor.K8sMetadata]
+	k8sCfg := k8sProcessorFactory.Config.(*k8sprocessor.Config)
+	// The expected config is exactly the opposite of the default config
 	expectedCfg := &k8sprocessor.Config{
-		KubeAuthType:      "kubeConfig",
-		KubeConfigDir:     "~/.kube/config",
-		GraceDeletePeriod: 60,
+		Enable:                false,
+		KubeAuthType:          "kubeConfig",
+		KubeConfigDir:         "/opt/.kube/config",
+		GraceDeletePeriod:     30,
+		EnableFetchReplicaSet: true,
 	}
-	if !reflect.DeepEqual(cfg, expectedCfg) {
-		t.Errorf("Expected %v, but get %v", expectedCfg, cfg)
+	assert.Equal(t, expectedCfg, k8sCfg)
+
+	networkAnalyzerFactory := factory.Analyzers[network.Network.String()]
+	networkConfig := networkAnalyzerFactory.Config
+	expectedNetworkConfig := &network.Config{
+		EnableTimeoutCheck:    true,
+		ConnectTimeout:        100,
+		FdReuseTimeout:        15,
+		NoResponseThreshold:   120,
+		ResponseSlowThreshold: 500,
+		EnableConntrack:       true,
+		ConntrackMaxStateSize: 131072,
+		ConntrackRateLimit:    500,
+		ProcRoot:              "/proc",
+		// Case: This slice is from the default config. The config file doesn't have this field.
+		ProtocolParser: []string{"http", "mysql", "dns", "redis", "kafka", "dubbo"},
+		// Case: This slice is overridden by the config file. The default config is different.
+		ProtocolConfigs: []network.ProtocolConfig{
+			{
+				Key:           "http",
+				Ports:         []uint32{80, 8080},
+				PayloadLength: 100,
+				Threshold:     200,
+			},
+		},
+		UrlClusteringMethod: "blank",
 	}
+	assert.Equal(t, expectedNetworkConfig, networkConfig)
 }

collector/internal/application/testdata/kindling-collector-config.yaml

@@ -1,114 +1,31 @@
-receivers:
-  cgoreceiver:
-    subscribe:
-      - name: syscall_exit-writev
-        category: net
-      - name: syscall_exit-readv
-        category: net
-      - name: syscall_exit-write
-        category: net
-      - name: syscall_exit-read
-        category: net
-      - name: syscall_exit-sendto
-        category: net
-      - name: syscall_exit-recvfrom
-        category: net
-      - name: syscall_exit-sendmsg
-        category: net
-      - name: syscall_exit-recvmsg
-        category: net
-      - name: kprobe-tcp_close
-      - name: kprobe-tcp_rcv_established
-      - name: kprobe-tcp_drop
-      - name: kprobe-tcp_retransmit_skb
-      - name: syscall_exit-connect
-      - name: kretprobe-tcp_connect
-      - name: kprobe-tcp_set_state
 analyzers:
   mockanalyzer:
     num: 10
   networkanalyzer:
-    connect_timeout: 100
-    fd_reuse_timeout: 15
-    response_slow_threshold: 500
-    enable_conntrack: true
-    conntrack_max_state_size: 131072
-    conntrack_rate_limit: 500
-    proc_root: /proc
-    protocol_parser: [ http, mysql, dns, redis, kafka, dubbo ]
+    # If the destination port of data is one of the followings, the protocol of such network request
+    # is set to the corresponding one. Note the program will try to identify the protocol automatically
+    # for the ports that are not in the lists, in which case the cpu usage will be increased much inevitably.
     protocol_config:
       - key: "http"
-        payload_length: 200
-      - key: "dubbo"
-        payload_length: 200
-      - key: "mysql"
-        slow_threshold: 100
-        disable_discern: false
-      - key: "kafka"
-        slow_threshold: 100
-      - key: "cassandra"
-        ports: [ 9042 ]
-        slow_threshold: 100
-      - key: "s3"
-        ports: [ 9190 ]
-        slow_threshold: 100
-      - key: "dns"
-        ports: [ 53 ]
-        slow_threshold: 100
-
+        ports: [ 80, 8080 ]
+        # payload_length indicates the maximum size that payload can be fetched for target protocol
+        # The trace data sent may contain such payload, so the higher this value, the larger network traffic.
+        payload_length: 100
+        slow_threshold: 200
+    url_clustering_method: blank
 processors:
   k8smetadataprocessor:
+    # Set "enable" false if you want to run the agent in the non-Kubernetes environment.
+    # Otherwise, the agent will panic if it can't connect to the API-server.
+    enable: false
     kube_auth_type: kubeConfig
-  nodemetricprocessor:
-
-exporters:
-  otelexporter:
-    metric_aggregation_map:
-      kindling_entity_request_total: counter
-      kindling_entity_request_duration_nanoseconds_total: counter
-      kindling_entity_request_average_duration_nanoseconds: histogram
-      kindling_entity_request_send_bytes_total: counter
-      kindling_entity_request_receive_bytes_total: counter
-      kindling_topology_request_total: counter
-      kindling_topology_request_duration_nanoseconds_total: counter
-      kindling_topology_request_average_duration_nanoseconds: histogram
-      kindling_topology_request_request_bytes_total: counter
-      kindling_topology_request_response_bytes_total: counter
-      kindling_trace_request_duration_nanoseconds: gauge
-      kindling_tcp_srtt_microseconds: gauge
-      kindling_tcp_retransmit_total: counter
-      kindling_tcp_packet_loss_total: counter
-    export_kind: prometheus
-    custom_labels:
-      job: test-hcmine
-    prometheus:
-      port: :8080
-    otlp:
-      collect_period: 15s
-      endpoint: 10.10.10.10:8080
-    stdout:
-      collect_period: 15s
-
-observability:
-  logger:
-    console_level: debug # debug,info,warn,error,none
-    file_level: debug
-    file_rotation:
-      filename: agent.log
-      maxsize: 512 #MB
-      maxage: 30 #day
-      maxbackups: 5
-      localtime: true
-      compress: false
-  opentelemetry:
-    # Export data in the following ways: ["prometheus", "otlp", "stdout"]
-    # Note: configure the corresponding section to make everything ok
-    export_kind: stdout
-    prometheus:
-      port: :9501
-    otlp:
-      collect_period: 15s
-      # Note: DO NOT add the prefix "http://"
-      endpoint: 10.10.10.10:8080
-    stdout:
-      collect_period: 15s
+    kube_config_dir: /opt/.kube/config
+    # GraceDeletePeriod controls the delay interval after receiving delete event.
+    # The unit is seconds, and the default value is 60 seconds.
+    # Should not be lower than 30 seconds.
+    grace_delete_period: 30
+    # enable_fetch_replicaset controls whether to fetch ReplicaSet information.
+    # The default value is false. It should be enabled if the ReplicaSet
+    # is used to control pods in the third-party CRD except for Deployment.
+    enable_fetch_replicaset: true
+  nodemetricprocessor:

collector/pkg/component/controller/controller.go

@@ -4,8 +4,10 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/Kindling-project/kindling/collector/pkg/component"
+	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
+
+	"github.com/Kindling-project/kindling/collector/pkg/component"
 )
 
 type ControllerAPI interface {
@@ -50,7 +52,9 @@ type ControllerConfig struct {
 func (cf *ControllerFactory) ConstructConfig(viper *viper.Viper, tools *component.TelemetryTools) error {
 	var controllerConfig ControllerConfig
 	key := ControllerComponent
-	err := viper.UnmarshalKey(key, &controllerConfig)
+	err := viper.UnmarshalKey(key, &controllerConfig, func(config *mapstructure.DecoderConfig) {
+		config.ZeroFields = true
+	})
 	if err != nil {
 		tools.Logger.Errorf("Error happened when reading controller config, will disable all controller: %v", err)
 	}

collector/pkg/component/telemetry.go

@@ -3,13 +3,15 @@ package component
 import (
 	"log"
 
-	"github.com/Kindling-project/kindling/collector/pkg/observability"
-	"github.com/Kindling-project/kindling/collector/pkg/observability/logger"
+	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/metric/global"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
+
+	"github.com/Kindling-project/kindling/collector/pkg/observability"
+	"github.com/Kindling-project/kindling/collector/pkg/observability/logger"
 )
 
 const (
@@ -40,7 +42,9 @@ func (t *TelemetryManager) ConstructConfig(viper *viper.Viper) {
 func (t *TelemetryManager) initLogger(viper *viper.Viper) {
 	var loggerConfig = logger.Config{}
 	key := ObservabilityConfig + "." + LogKey
-	err := viper.UnmarshalKey(key, &loggerConfig)
+	err := viper.UnmarshalKey(key, &loggerConfig, func(config *mapstructure.DecoderConfig) {
+		config.ZeroFields = true
+	})
 	if err != nil {
 		log.Printf("Error happened when reading logger config, and default config will be used: %v", err)
 	}
@@ -51,7 +55,9 @@ func (t *TelemetryManager) initLogger(viper *viper.Viper) {
 func (t *TelemetryManager) initProvider(viper *viper.Viper) {
 	var config = &observability.DefaultConfig
 	key := ObservabilityConfig + "." + MetricKey
-	err := viper.UnmarshalKey(key, config)
+	err := viper.UnmarshalKey(key, config, func(config *mapstructure.DecoderConfig) {
+		config.ZeroFields = true
+	})
 	if err != nil {
 		log.Printf("Error happened when reading observability config, and default config will be used: %v", err)
 	}