NETOBSERV-1545: Expose a counter of BPF drops

Signed-off-by: Mohamed Mahmoud <mmahmoud@redhat.com>

bpf/flows.c

@@ -89,13 +89,24 @@ static inline int flow_monitor(struct __sk_buff *skb, u8 direction) {
         aggregate_flow->dns_record.latency = pkt.dns_latency;
         aggregate_flow->dns_record.errno = dns_errno;
         long ret = bpf_map_update_elem(&aggregated_flows, &id, aggregate_flow, BPF_ANY);
-        if (trace_messages && ret != 0) {
+        if (ret != 0) {
+            u32 *error_counter_p = NULL;
+            u32 initVal = 1, key = HASHMAP_FLOWS_DROPPED_KEY;
             // usually error -16 (-EBUSY) is printed here.
             // In this case, the flow is dropped, as submitting it to the ringbuffer would cause
             // a duplicated UNION of flows (two different flows with partial aggregation of the same packets),
             // which can't be deduplicated.
             // other possible values https://chromium.googlesource.com/chromiumos/docs/+/master/constants/errnos.md
-            bpf_printk("error updating flow %d\n", ret);
+            if (trace_messages) {
+                bpf_printk("error updating flow %d\n", ret);
+            }
+            // Update global counter for hashmap update errors
+            error_counter_p = bpf_map_lookup_elem(&global_counters, &key);
+            if (!error_counter_p) {
+                bpf_map_update_elem(&global_counters, &key, &initVal, BPF_ANY);
+                return TC_ACT_OK;
+            }
+            __sync_fetch_and_add(error_counter_p, 1);
         }
     } else {
         // Key does not exist in the map, and will need to create a new entry.

bpf/maps_definition.h

@@ -36,4 +36,12 @@ struct {
     __uint(map_flags, BPF_F_NO_PREALLOC);
 } dns_flows SEC(".maps");
 
+// Global counter for hashmap update errors
+struct {
+    __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+    __type(key, u32);
+    __type(value, u32);
+    __uint(max_entries, 1);
+} global_counters SEC(".maps");
+
 #endif //__MAPS_DEFINITION_H__

bpf/types.h

@@ -181,5 +181,13 @@ typedef struct dns_flow_id_t {
     u8 protocol;
 } __attribute__((packed)) dns_flow_id;
 
+// Enum to define global counters keys and share it with userspace
+typedef enum global_counters_key_t {
+    HASHMAP_FLOWS_DROPPED_KEY = 0,
+} global_counters_key;
+
+// Force emitting enum global_counters_key_t into the ELF.
+const enum global_counters_key_t *unused5 __attribute__((unused));
+
 #endif /* __TYPES_H__ */
 

pkg/ebpf/tracer.go

@@ -27,7 +27,7 @@ import (
 )
 
 // $BPF_CLANG and $BPF_CFLAGS are set by the Makefile.
-//go:generate bpf2go -cc $BPF_CLANG -cflags $BPF_CFLAGS -target amd64,arm64,ppc64le,s390x -type flow_metrics_t -type flow_id_t -type flow_record_t -type pkt_drops_t -type dns_record_t Bpf ../../bpf/flows.c -- -I../../bpf/headers
+//go:generate bpf2go -cc $BPF_CLANG -cflags $BPF_CFLAGS -target amd64,arm64,ppc64le,s390x -type flow_metrics_t -type flow_id_t -type flow_record_t -type pkt_drops_t -type dns_record_t -type global_counters_key_t Bpf ../../bpf/flows.c -- -I../../bpf/headers
 
 const (
 	qdiscType = "clsact"
@@ -343,6 +343,9 @@ func (m *FlowFetcher) Close() error {
 		if err := m.objects.DnsFlows.Close(); err != nil {
 			errs = append(errs, err)
 		}
+		if err := m.objects.GlobalCounters.Close(); err != nil {
+			errs = append(errs, err)
+		}
 		if len(errs) == 0 {
 			m.objects = nil
 		}
@@ -448,9 +451,25 @@ func (m *FlowFetcher) LookupAndDeleteMap(met *metrics.Metrics) map[BpfFlowId][]B
 	met.BufferSizeGauge.WithBufferName("hashmap-total").Set(float64(count))
 	met.BufferSizeGauge.WithBufferName("hashmap-unique").Set(float64(len(flows)))
 
+	m.ReadGlobalCounter(met)
 	return flows
 }
 
+// ReadGlobalCounter reads the global counter and updates hashmap update error counter metrics
+func (m *FlowFetcher) ReadGlobalCounter(met *metrics.Metrics) {
+	var allCPUValue []uint32
+	key := BpfGlobalCountersKeyTHASHMAP_FLOWS_DROPPED_KEY
+
+	if err := m.objects.GlobalCounters.Lookup(key, &allCPUValue); err != nil {
+		log.WithError(err).Warnf("couldn't read global counter")
+		return
+	}
+	// aggregate all the counters
+	for _, counter := range allCPUValue {
+		met.DroppedFlowsCounter.WithSourceAndReason("flow-fetcher", "CannotUpdateHashMapCounter").Add(float64(counter))
+	}
+}
+
 // DeleteMapsStaleEntries Look for any stale entries in the features maps and delete them
 func (m *FlowFetcher) DeleteMapsStaleEntries(timeOut time.Duration) {
 	m.lookupAndDeleteDNSMap(timeOut)
@@ -521,6 +540,7 @@ func kernelSpecificLoadAndAssign(oldKernel bool, spec *ebpf.CollectionSpec) (Bpf
 		objects.IngressFlowParse = newObjects.IngressFlowParse
 		objects.TcpRcvFentry = newObjects.TCPRcvFentry
 		objects.TcpRcvKprobe = newObjects.TCPRcvKprobe
+		objects.GlobalCounters = newObjects.GlobalCounters
 		objects.KfreeSkb = nil
 	} else {
 		if err := spec.LoadAndAssign(&objects, nil); err != nil {

pkg/ebpf/tracer_legacy.go

@@ -28,6 +28,7 @@ func (m *FlowFetcher) legacyLookupAndDeleteMap(met *metrics.Metrics) map[BpfFlow
 	met.BufferSizeGauge.WithBufferName("hashmap-legacy-total").Set(float64(count))
 	met.BufferSizeGauge.WithBufferName("hashmap-legacy-unique").Set(float64(len(flows)))
 
+	m.ReadGlobalCounter(met)
 	return flows
 }