fix: move filestats receiver into host_monitoring connection (#61)

### Description

CS-917 fix: move filestats receiver into host_monitoring connection

### Checklist
- [ ] Created tests which fail without the change (if possible)
- [ ] Extended the README / documentation, if necessary

.goreleaser.yaml

@@ -73,6 +73,8 @@ archives:
         dst: "observe-agent.yaml"
       - src: "packaging/windows/connections/host_monitoring/*"
         dst: "connections/host_monitoring"
+      - src: "packaging/windows/connections/self_monitoring/*"
+        dst: "connections/self_monitoring"
 
 changelog:
   use: github

cmd/commands/initconfig/configschema.go

@@ -14,8 +14,13 @@ type HostMonitoringConfig struct {
 	Metrics HostMonitoringMetricsConfig
 }
 
+type SelfMonitoringConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
 type AgentConfig struct {
 	Token          string               `yaml:"token"`
 	ObserveURL     string               `yaml:"observe_url"`
+	SelfMonitoring SelfMonitoringConfig `yaml:"self_monitoring"`
 	HostMonitoring HostMonitoringConfig `yaml:"host_monitoring"`
 }

cmd/commands/initconfig/initconfig.go

@@ -18,6 +18,7 @@ var (
 	config_path                     string
 	token                           string
 	observe_url                     string
+	self_monitoring_enabled         bool
 	host_monitoring_enabled         bool
 	host_monitoring_logs_enabled    bool
 	host_monitoring_metrics_enabled bool
@@ -30,6 +31,7 @@ const configTemplate = "observe-agent.tmpl"
 type FlatAgentConfig struct {
 	Token                         string
 	ObserveURL                    string
+	SelfMonitoring_Enabled        bool
 	HostMonitoring_Enabled        bool
 	HostMonitoring_LogsEnabled    bool
 	HostMonitoring_MetricsEnabled bool
@@ -44,6 +46,7 @@ func NewConfigureCmd() *cobra.Command {
 			configValues := FlatAgentConfig{
 				Token:                         viper.GetString("token"),
 				ObserveURL:                    viper.GetString("observe_url"),
+				SelfMonitoring_Enabled:        viper.GetBool("self_monitoring.enabled"),
 				HostMonitoring_Enabled:        viper.GetBool("host_monitoring.enabled"),
 				HostMonitoring_LogsEnabled:    viper.GetBool("host_monitoring.logs.enabled"),
 				HostMonitoring_MetricsEnabled: viper.GetBool("host_monitoring.metrics.enabled"),
@@ -79,11 +82,13 @@ func RegisterConfigFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&config_path, "config_path", "", "", "Path to write config output file to")
 	cmd.PersistentFlags().StringVar(&token, "token", "", "Observe token")
 	cmd.PersistentFlags().StringVar(&observe_url, "observe_url", "", "Observe data collection url")
+	cmd.PersistentFlags().BoolVar(&self_monitoring_enabled, "self_monitoring.enabled", true, "Enable self monitoring")
 	cmd.PersistentFlags().BoolVar(&host_monitoring_enabled, "host_monitoring.enabled", true, "Enable host monitoring")
 	cmd.PersistentFlags().BoolVar(&host_monitoring_logs_enabled, "host_monitoring.logs.enabled", true, "Enable host monitoring logs")
 	cmd.PersistentFlags().BoolVar(&host_monitoring_metrics_enabled, "host_monitoring.metrics.enabled", true, "Enable host monitoring metrics")
 	viper.BindPFlag("token", cmd.PersistentFlags().Lookup("token"))
 	viper.BindPFlag("observe_url", cmd.PersistentFlags().Lookup("observe_url"))
+	viper.BindPFlag("self_monitoring.enabled", cmd.PersistentFlags().Lookup("self_monitoring.enabled"))
 	viper.BindPFlag("host_monitoring.enabled", cmd.PersistentFlags().Lookup("host_monitoring.enabled"))
 	viper.BindPFlag("host_monitoring.logs.enabled", cmd.PersistentFlags().Lookup("host_monitoring.logs.enabled"))
 	viper.BindPFlag("host_monitoring.metrics.enabled", cmd.PersistentFlags().Lookup("host_monitoring.metrics.enabled"))

cmd/commands/initconfig/initconfig_test.go

@@ -22,6 +22,9 @@ func Test_InitConfigCommand(t *testing.T) {
 			expectedConfig: AgentConfig{
 				Token:      "test-token",
 				ObserveURL: "test-url",
+				SelfMonitoring: SelfMonitoringConfig{
+					Enabled: true,
+				},
 				HostMonitoring: HostMonitoringConfig{
 					Enabled: true,
 					Logs: HostMonitoringLogsConfig{
@@ -35,7 +38,7 @@ func Test_InitConfigCommand(t *testing.T) {
 			expectErr: "",
 		},
 		{
-			args: []string{"--config_path=./test-config.yaml", "--token=test-token", "--observe_url=test-url", "--host_monitoring.enabled=false", "--host_monitoring.logs.enabled=false", "--host_monitoring.metrics.enabled=false"},
+			args: []string{"--config_path=./test-config.yaml", "--token=test-token", "--observe_url=test-url", "--self_monitoring.enabled=false", "--host_monitoring.enabled=false", "--host_monitoring.logs.enabled=false", "--host_monitoring.metrics.enabled=false"},
 			expectedConfig: AgentConfig{
 				Token:      "test-token",
 				ObserveURL: "test-url",

cmd/commands/initconfig/observe-agent.tmpl

@@ -7,6 +7,9 @@ observe_url: {{ .ObserveURL }}
 # Debug mode - Sets agent log level to debug
 debug: false
 
+self_monitoring:
+  enabled: {{ .SelfMonitoring_Enabled }}
+
 host_monitoring:
   enabled: {{ .HostMonitoring_Enabled }}
   logs: 

cmd/connections/hostmonitoring.go

@@ -13,6 +13,10 @@ type HostMonitoringConfig struct {
 var HostMonitoringConnectionType = ConnectionType{
 	Name: "host_monitoring",
 	ConfigFields: []CollectorConfigFragment{
+		{
+			configYAMLPath:    "enabled",
+			colConfigFilePath: "host.yaml",
+		},
 		{
 			configYAMLPath:    "metrics.enabled",
 			colConfigFilePath: "metrics.yaml",

packaging/docker/observe-agent/connections/host_monitoring/host.yaml

@@ -0,0 +1,12 @@
+receivers:
+  filestats/agent:
+    include: '/etc/observe-agent/otel-collector.yaml'
+    collection_interval: 240m
+    initial_delay: 60s
+
+service:
+  pipelines:
+    metrics/agent-filestats:
+       receivers: [filestats/agent]
+       processors: [resourcedetection, resourcedetection/cloud]
+       exporters: [otlphttp/observe]

packaging/docker/observe-agent/connections/self_monitoring/logs_and_metrics.yaml

@@ -1,9 +1,4 @@
 receivers:
-  filestats/agent:
-    include: '/etc/observe-agent/otel-collector.yaml'
-    collection_interval: 240m
-    initial_delay: 60s
-
   filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields
     include: [/etc/observe-agent/otel-collector.yaml]
     start_at: beginning
@@ -30,11 +25,6 @@ receivers:
 
 service:
   pipelines:
-    metrics/agent-filestats:
-       receivers: [filestats/agent]
-       processors: [resourcedetection, resourcedetection/cloud]
-       exporters: [otlphttp/observe]
-
     metrics/agent-internal:
       receivers: [prometheus/agent, count]
       processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]

packaging/docker/observe-agent/otel-collector.yaml

@@ -16,35 +16,6 @@ receivers:
       http:
         endpoint: localhost:4318
 
-  filestats/agent:
-    include: '/etc/observe-agent/otel-collector.yaml'
-    collection_interval: 240m
-    initial_delay: 60s
-
-  filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields
-    include: [/etc/observe-agent/otel-collector.yaml]
-    start_at: beginning
-    poll_interval: 5m
-    multiline:
-      line_end_pattern: ENDOFLINEPATTERN
-
-  prometheus/agent:
-    config:
-      scrape_configs:
-        - job_name: 'otelcol'
-          scrape_interval: 10s
-          static_configs:
-            - targets: ['0.0.0.0:8888']
-          metric_relabel_configs:
-            - source_labels: [__name__]
-              regex: '.*grpc_io.*'
-              action: drop
-
-  journald/agent:
-    units:
-      - observe-agent
-    priority: info
-
 processors:
   # Snowflake limit for identifiers: Regardless of whether an identifier is unquoted or double-quoted, the maximum number of characters allowed is 255 (including blank spaces).
   # https://docs.snowflake.com/en/sql-reference/identifiers-syntax#identifier-requirements

packaging/linux/etc/observe-agent/connections/host_monitoring/host.yaml

@@ -0,0 +1,12 @@
+receivers:
+  filestats/agent:
+    include: '/etc/observe-agent/otel-collector.yaml'
+    collection_interval: 240m
+    initial_delay: 60s
+
+service:
+  pipelines:
+    metrics/agent-filestats:
+       receivers: [filestats/agent]
+       processors: [resourcedetection, resourcedetection/cloud]
+       exporters: [otlphttp/observe]

packaging/linux/etc/observe-agent/connections/self_monitoring/logs_and_metrics.yaml

@@ -1,9 +1,4 @@
 receivers:
-  filestats/agent:
-    include: '/etc/observe-agent/otel-collector.yaml'
-    collection_interval: 240m
-    initial_delay: 60s
-
   filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields
     include: [/etc/observe-agent/otel-collector.yaml]
     start_at: beginning
@@ -30,11 +25,6 @@ receivers:
 
 service:
   pipelines:
-    metrics/agent-filestats:
-       receivers: [filestats/agent]
-       processors: [resourcedetection, resourcedetection/cloud]
-       exporters: [otlphttp/observe]
-
     metrics/agent-internal:
       receivers: [prometheus/agent, count]
       processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]

packaging/windows/connections/host_monitoring/host.yaml

@@ -0,0 +1,12 @@
+receivers:
+  filestats/agent:
+    include: 'C:\Program Files\Observe\observe-agent\config\otel-collector.yaml'
+    collection_interval: 240m
+    initial_delay: 60s
+
+service:
+  pipelines:
+    metrics/agent-filestats:
+       receivers: [filestats/agent]
+       processors: [resourcedetection, resourcedetection/cloud]
+       exporters: [otlphttp/observe]

packaging/windows/connections/self_monitoring/logs_and_metrics.yaml

@@ -1,9 +1,4 @@
 receivers:
-  filestats/agent:
-    include: 'C:\Program Files\Observe\observe-agent\config\otel-collector.yaml'
-    collection_interval: 240m
-    initial_delay: 60s
-
   filelog/agent-config: # TODO: Add observe-agent.yaml once we can obfuscate sensitive config fields
     include: ['C:\Program Files\Observe\observe-agent\config\otel-collector.yaml']
     start_at: beginning
@@ -25,11 +20,6 @@ receivers:
 
 service:
   pipelines:
-    metrics/agent-filestats:
-       receivers: [filestats/agent]
-       processors: [resourcedetection, resourcedetection/cloud]
-       exporters: [otlphttp/observe]
-
     metrics/agent-internal:
       receivers: [prometheus/agent, count]
       processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, batch]