work in progress for idaholab#395, malcolm reporting capture statisti…

…cs from zeek/suricata

config/logstash.env.example

@@ -18,5 +18,7 @@ LOGSTASH_NETBOX_AUTO_POPULATE=false
 # Caching parameters for NetBox's LogStash lookups
 LOGSTASH_NETBOX_CACHE_SIZE=1000
 LOGSTASH_NETBOX_CACHE_TTL=30
+# Zeek log types that will be ignored (dropped) by LogStash
+LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout
 # Logstash memory allowance and other Java options
 LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true

logstash/pipelines/zeek/10_zeek_prep.conf

@@ -23,24 +23,14 @@ filter {
       end"
   }
 
-  # report types we're going to ignore
-  if (([log_source] == "analyzer") or
-      ([log_source] == "bsap_ip_unknown") or
-      ([log_source] == "bsap_serial_unknown") or
-      ([log_source] == "ecat_arp_info") or
-      ([log_source] == "reporter") or
-      ([log_source] == "broker") or
-      ([log_source] == "cluster") or
-      ([log_source] == "capture_loss") or
-      ([log_source] == "communication") or
-      ([log_source] == "packet_filter") or
-      ([log_source] == "png") or
-      ([log_source] == "stats") or
-      ([log_source] == "stderr") or
-      ([log_source] == "stdout") or
-      ([log_source] == "loaded_scripts")) {
-    drop { id => "drop_zeek_ignored_source" }
+  # Zeek logs we're going to ignore
+  ruby {
+    id => "ruby_zeek_log_type_determine_drop"
+    init => "logtypesStr = ENV['LOGSTASH_ZEEK_IGNORED_LOGS'] || 'analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout' ; $logtypes = logtypesStr.gsub(/\s+/, '').split(',');"
+    code => "event.set('[@metadata][drop_zeek_log]', true) if $logtypes.include?(event.get('[log_source]').to_s)"
   }
+  if [@metadata][drop_zeek_log] { drop { id => "drop_zeek_ignored_source" } }
+
 
   # remove some tags pulled from the filename we might not want
   if ([@metadata][zeek_log_tags]) {

logstash/pipelines/zeek/11_zeek_parse.conf

@@ -5439,6 +5439,266 @@ filter {
 
     } # if / else if for opcua log types
 
+  } else if ([log_source] == "analyzer") {
+    #############################################################################################################################
+    # analyzer.log
+    # Zeek Logging analyzer confirmations and violations into analyzer.log
+    # https://docs.zeek.org/en/master/scripts/base/frameworks/analyzer/logging.zeek.html
+
+    dissect {
+      id => "dissect_zeek_diagnostic_analyzer"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][cause]}	%{[zeek_cols][analyzer_kind]}	%{[zeek_cols][analyzer_name]}	%{[zeek_cols][uid]}	%{[zeek_cols][fuid]}	%{[zeek_cols][orig_h]}	%{[zeek_cols][orig_p]}	%{[zeek_cols][resp_h]}	%{[zeek_cols][resp_p]}	%{[zeek_cols][failure_reason]}	%{[zeek_cols][failure_data]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_analyzer"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_analyzer"
+        init => "$zeek_diagnostic_analyzer_field_names = [ 'ts', 'cause', 'analyzer_kind', 'analyzer_name', 'uid', 'fuid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'failure_reason', 'failure_data' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_analyzer_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_analyzer"
+             add_tag => [ "_zeekdiagnostic" ] }
+
+  } else if ([log_source] == "broker") {
+    #############################################################################################################################
+    # broker.log
+    # https://docs.zeek.org/en/master/scripts/base/frameworks/broker/log.zeek.html
+
+    dissect {
+      id => "dissect_zeek_diagnostic_broker"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][event_type]}	%{[zeek_cols][event_action]}	%{[zeek_cols][peer_ip]}	%{[zeek_cols][peer_port]}	%{[zeek_cols][peer_message]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_broker"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_broker"
+        init => "$zeek_diagnostic_broker_field_names = [ 'ts', 'event_type', 'event_action', 'peer_ip', 'peer_port', 'peer_message' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_broker_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_broker"
+             add_tag => [ "_zeekdiagnostic" ] }
+
+  } else if ([log_source] == "capture_loss") {
+    #############################################################################################################################
+    # capture_loss.log
+    # Reports analysis of missing traffic. Zeek bases its conclusions on analysis of TCP sequence numbers.
+    # https://docs.zeek.org/en/master/logs/capture-loss-and-reporter.html
+
+    dissect {
+      id => "dissect_zeek_diagnostic_capture_loss"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][ts_delta]}	%{[zeek_cols][peer]}	%{[zeek_cols][gaps]}	%{[zeek_cols][acks]}	%{[zeek_cols][percent_lost]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_capture_loss"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_capture_loss"
+        init => "$zeek_diagnostic_capture_loss_field_names = [ 'ts', 'ts_delta', 'peer', 'gaps', 'acks', 'percent_lost' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_capture_loss_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_capture_loss"
+             add_tag => [ "_zeekdiagnostic" ] }
+
+  } else if ([log_source] == "cluster") {
+    #############################################################################################################################
+    # cluster.log
+    # Logging for establishing and controlling a cluster of Zeek instances
+    # https://docs.zeek.org/en/master/scripts/base/frameworks/cluster/main.zeek.html#type-Cluster::Info
+
+    dissect {
+      id => "dissect_zeek_diagnostic_cluster"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][node]}	%{[zeek_cols][node_message]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_cluster"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_cluster"
+        init => "$zeek_diagnostic_cluster_field_names = [ 'ts', 'node', 'node_message' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_cluster_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_cluster"
+             add_tag => [ "_zeekdiagnostic" ] }
+
+  } else if ([log_source] == "config") {
+    #############################################################################################################################
+    # config.log
+    # Logging for Zeek configuration changes
+    # https://docs.zeek.org/en/master/scripts/base/frameworks/config/main.zeek.html#type-Config::Info
+
+    dissect {
+      id => "dissect_zeek_diagnostic_config"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][value_name]}	%{[zeek_cols][value_old]}	%{[zeek_cols][value_new]}	%{[zeek_cols][location]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_config"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_config"
+        init => "$zeek_diagnostic_config_field_names = [ 'ts', 'value_name', 'value_old', 'value_new', 'location' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_config_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_config"
+             add_tag => [ "_zeekdiagnostic" ] }
+
+  } else if ([log_source] == "packet_filter") {
+    #############################################################################################################################
+    # packet_filter.log
+    # https://docs.zeek.org/en/master/scripts/base/frameworks/packet-filter/main.zeek.html#type-PacketFilter::Info
+
+    dissect {
+      id => "dissect_zeek_diagnostic_packet_filter"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][node]}	%{[zeek_cols][filter]}	%{[zeek_cols][init]}	%{[zeek_cols][success]}	%{[zeek_cols][failure_reason]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_packet_filter"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_packet_filter"
+        init => "$zeek_diagnostic_packet_filter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_packet_filter_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_packet_filter"
+             add_tag => [ "_zeekdiagnostic" ] }
+
+  } else if ([log_source] == "print") {
+    #############################################################################################################################
+    # print.log
+    # https://docs.zeek.org/en/master/scripts/base/frameworks/logging/main.zeek.html#type-Log::PrintLogInfo
+
+    dissect {
+      id => "dissect_zeek_diagnostic_print"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][vals]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_print"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_print"
+        init => "$zeek_diagnostic_print_field_names = [ 'ts', 'vals' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_print_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "split_zeek_diagnostic_print_vals"
+             split => { "[zeek_cols][vals]" => "," } }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_print"
+             add_tag => [ "_zeekdiagnostic" ] }
+
+
+  } else if ([log_source] == "reporter") {
+    #############################################################################################################################
+    # reporter.log
+    # https://docs.zeek.org/en/master/scripts/base/frameworks/reporter/main.zeek.html#type-Reporter::Info
+
+    dissect {
+      id => "dissect_zeek_diagnostic_reporter"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][level]}	%{[zeek_cols][msg]}	%{[zeek_cols][location]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_reporter"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_reporter"
+        init => "$zeek_diagnostic_reporter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_reporter_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_reporter"
+             add_tag => [ "_zeekdiagnostic" ] }
+
+  } else if ([log_source] == "stats") {
+    #############################################################################################################################
+    # stats.log
+    # https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info
+
+    dissect {
+      id => "dissect_zeek_diagnostic_stats"
+      # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+      mapping => {
+        "[message]" => "%{[zeek_cols][ts]}	%{[zeek_cols][peer]}	%{[zeek_cols][mem]}	%{[zeek_cols][pkts_proc]}	%{[zeek_cols][bytes_recv]}	%{[zeek_cols][pkts_dropped]}	%{[zeek_cols][pkts_link]}	%{[zeek_cols][pkt_lag]}	%{[zeek_cols][pkts_filtered]}	%{[zeek_cols][events_proc]}	%{[zeek_cols][events_queued]}	%{[zeek_cols][active_tcp_conns]}	%{[zeek_cols][active_udp_conns]}	%{[zeek_cols][active_icmp_conns]}	%{[zeek_cols][tcp_conns]}	%{[zeek_cols][udp_conns]}	%{[zeek_cols][icmp_conns]}	%{[zeek_cols][timers]}	%{[zeek_cols][active_timers]}	%{[zeek_cols][files]}	%{[zeek_cols][active_files]}	%{[zeek_cols][dns_requests]}	%{[zeek_cols][active_dns_requests]}	%{[zeek_cols][reassem_tcp_size]}	%{[zeek_cols][reassem_file_size]}	%{[zeek_cols][reassem_frag_size]}	%{[zeek_cols][reassem_unknown_size]}"
+      }
+    }
+    if ("_dissectfailure" in [tags]) {
+      mutate {
+        id => "mutate_split_zeek_diagnostic_stats"
+        # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP
+        split => { "[message]" => "	" }
+      }
+      ruby {
+        id => "ruby_zip_zeek_diagnostic_stats"
+        init => "$zeek_diagnostic_stats_field_names = [ 'ts', 'peer', 'mem', 'pkts_proc', 'bytes_recv', 'pkts_dropped', 'pkts_link', 'pkt_lag', 'pkts_filtered', 'events_proc', 'events_queued', 'active_tcp_conns', 'active_udp_conns', 'active_icmp_conns', 'tcp_conns', 'udp_conns', 'icmp_conns', 'timers', 'active_timers', 'files', 'active_files', 'dns_requests', 'active_dns_requests', 'reassem_tcp_size', 'reassem_file_size', 'reassem_frag_size', 'reassem_unknown_size' ]"
+        code => "event.set('[zeek_cols]', $zeek_diagnostic_stats_field_names.zip(event.get('[message]')).to_h)"
+      }
+    }
+
+    mutate { id => "mutate_add_tag_zeek_diagnostic_stats"
+             add_tag => [ "_zeekdiagnostic" ] }
+
   } else {
     # some other unknown zeek log file. should start with ts at least!
     csv {

shared/bin/suricata_config_populate.py

@@ -187,6 +187,7 @@ def __call__(self, repr, data):
         'SSH_EVE_ENABLED': False,
         'SSH_HASSH': True,
         'SSH_PORTS': 22,
+        'STATS': False,
         'STREAM_CHECKSUM_VALIDATION': False,
         'STREAM_INLINE': 'auto',
         'STREAM_MEMCAP': '64mb',
@@ -1170,7 +1171,7 @@ def main():
                 logging.error(output)
 
     # final tweaks
-    deep_set(cfg, ['stats', 'enabled'], False)
+    deep_set(cfg, ['stats', 'enabled'], val2bool(DEFAULT_VARS['STATS']))
     cfg.pop('rule-files', None)
     deep_set(cfg, ['rule-files'], GetRuleFiles())