filters: add extra logging, fix filters config updates

bd-log-filter/Cargo.toml

@@ -11,6 +11,7 @@ bd-client-stats-store.path  = "../bd-client-stats-store"
 bd-log-matcher.path         = "../bd-log-matcher"
 bd-log-primitives.path      = "../bd-log-primitives"
 bd-proto.path               = "../bd-proto"
+itertools.workspace         = true
 regex.workspace             = true
 log.workspace               = true
 time.workspace              = true

bd-log-filter/src/lib.rs

@@ -14,6 +14,7 @@ use bd_log_primitives::{FieldsRef, Log, LogField, LogFields};
 use bd_proto::protos::filter::filter::filter::{self};
 use bd_proto::protos::filter::filter::{Filter as FilterProto, FiltersConfiguration};
 use filter::transform::Transform_type;
+use itertools::Itertools;
 use regex::Regex;
 use std::borrow::Cow;
 
@@ -54,7 +55,13 @@ impl FilterChain {
           })
           .ok()
       })
-      .collect();
+      .collect_vec();
+
+    log::debug!(
+      "{} filters created, {} failed",
+      filters.len(),
+      failures_count
+    );
 
     (Self { filters }, failures_count)
   }
@@ -69,6 +76,12 @@ impl FilterChain {
         continue;
       }
 
+      log::trace!(
+        "filter matched {:?} log, applying {} transforms",
+        log.message,
+        filter.transforms.len()
+      );
+
       for transform in &filter.transforms {
         transform.apply(log);
       }

bd-logger/src/client_config.rs

@@ -250,7 +250,7 @@ impl ApplyConfig for LoggerUpdate {
       workflows,
       insights,
       bdtail,
-      ..
+      filters,
     } = configuration;
 
     let maybe_stream_buffer = self
@@ -272,7 +272,7 @@ impl ApplyConfig for LoggerUpdate {
       WorkflowsConfiguration::default()
     };
 
-    let (filter_chain, filter_config_parse_failure_count) = FilterChain::new(configuration.filters);
+    let (filter_chain, filter_config_parse_failure_count) = FilterChain::new(filters);
     self
       .filter_config_parse_failure
       .inc_by(filter_config_parse_failure_count);

bd-logger/src/log_replay.rs

@@ -164,6 +164,7 @@ impl ProcessingPipeline {
     self.buffer_selector = config.buffer_selector;
     self.buffer_producers = config.buffer_producers;
     self.tail_configs = config.tail_configs;
+    self.filter_chain = config.filter_chain;
 
     if self.workflows_enabled_flag.read_mark_update() {
       let workflows_engine_config = WorkflowsEngineConfig::new(