NETOBSERV-1063: Add TCP drop & DNS tracking to flowlogs-pipeline

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/netobserv/gopipes v0.3.0
 	github.com/netobserv/loki-client-go v0.0.0-20220927092034-f37122a54500
-	github.com/netobserv/netobserv-ebpf-agent v0.3.1-0.20230320150131-c62173ac9558
+	github.com/netobserv/netobserv-ebpf-agent v0.3.2-0.20230624133346-27baf2917e8d
 	github.com/netsampler/goflow2 v1.1.1-0.20220509155230-5300494e4785
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1

go.sum

@@ -716,8 +716,8 @@ github.com/netobserv/gopipes v0.3.0 h1:IYmPnnAVCdSK7VmHmpFhrVBOEm45qpgbZmJz1sSW+
 github.com/netobserv/gopipes v0.3.0/go.mod h1:N7/Gz05EOF0CQQSKWsv3eof22Cj2PB08Pbttw98YFYU=
 github.com/netobserv/loki-client-go v0.0.0-20220927092034-f37122a54500 h1:RmnoJe/ci5q+QdM7upFdxiU+D8F3L3qTd5wXCwwHefw=
 github.com/netobserv/loki-client-go v0.0.0-20220927092034-f37122a54500/go.mod h1:LHXpc5tjKvsfZn0pwLKrvlgEhZcCaw3Di9mUEZGAI4E=
-github.com/netobserv/netobserv-ebpf-agent v0.3.1-0.20230320150131-c62173ac9558 h1:vUcT7a0zMb7QRXffKQA6r/xX/VfyWpO/ee7mFb4WMJQ=
-github.com/netobserv/netobserv-ebpf-agent v0.3.1-0.20230320150131-c62173ac9558/go.mod h1:Z2t24u5bDmgbk1reEeDxdpNvnFKLvwpMlblWPbE15Gw=
+github.com/netobserv/netobserv-ebpf-agent v0.3.2-0.20230624133346-27baf2917e8d h1:teXZcrHysefK1uJXhbOmzZ0Smci8PMVs9XKseD5Kyfo=
+github.com/netobserv/netobserv-ebpf-agent v0.3.2-0.20230624133346-27baf2917e8d/go.mod h1:B0Qlk+QBT3+rf6vf3cF4GNZSPb0PSwQww8kLGkOPdRg=
 github.com/netobserv/prometheus-common v0.31.2-0.20220720134304-43e74fd22881 h1:hx5bi6xBovRjmwUoVJBzhJ3EDo4K4ZUsqqKrJuQ2vMI=
 github.com/netobserv/prometheus-common v0.31.2-0.20220720134304-43e74fd22881/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/netsampler/goflow2 v1.1.1-0.20220509155230-5300494e4785 h1:qhDrIMXlk8YV7BxwA6UR/dQVdUzohjLlmrUXymsBx6g=

pkg/pipeline/decode/decode_protobuf.go

@@ -37,26 +37,35 @@ func PBFlowToMap(flow *pbflow.Record) config.GenericMap {
 		return config.GenericMap{}
 	}
 	out := config.GenericMap{
-		"FlowDirection":   int(flow.Direction.Number()),
-		"Bytes":           flow.Bytes,
-		"SrcAddr":         ipToStr(flow.Network.GetSrcAddr()),
-		"DstAddr":         ipToStr(flow.Network.GetDstAddr()),
-		"SrcMac":          macToStr(flow.DataLink.GetSrcMac()),
-		"DstMac":          macToStr(flow.DataLink.GetDstMac()),
-		"SrcPort":         flow.Transport.GetSrcPort(),
-		"DstPort":         flow.Transport.GetDstPort(),
-		"Etype":           flow.EthProtocol,
-		"Packets":         flow.Packets,
-		"Duplicate":       flow.Duplicate,
-		"Proto":           flow.Transport.GetProtocol(),
-		"TimeFlowStartMs": flow.TimeFlowStart.AsTime().UnixMilli(),
-		"TimeFlowEndMs":   flow.TimeFlowEnd.AsTime().UnixMilli(),
-		"TimeReceived":    time.Now().Unix(),
-		"Interface":       flow.Interface,
-		"AgentIP":         ipToStr(flow.AgentIp),
-		"Flags":           flow.Flags,
-		"IcmpType":        flow.Icmp.GetIcmpType(),
-		"IcmpCode":        flow.Icmp.GetIcmpCode(),
+		"FlowDirection":          int(flow.Direction.Number()),
+		"Bytes":                  flow.Bytes,
+		"SrcAddr":                ipToStr(flow.Network.GetSrcAddr()),
+		"DstAddr":                ipToStr(flow.Network.GetDstAddr()),
+		"SrcMac":                 macToStr(flow.DataLink.GetSrcMac()),
+		"DstMac":                 macToStr(flow.DataLink.GetDstMac()),
+		"SrcPort":                flow.Transport.GetSrcPort(),
+		"DstPort":                flow.Transport.GetDstPort(),
+		"Etype":                  flow.EthProtocol,
+		"Packets":                flow.Packets,
+		"Duplicate":              flow.Duplicate,
+		"Proto":                  flow.Transport.GetProtocol(),
+		"TimeFlowStartMs":        flow.TimeFlowStart.AsTime().UnixMilli(),
+		"TimeFlowEndMs":          flow.TimeFlowEnd.AsTime().UnixMilli(),
+		"TimeReceived":           time.Now().Unix(),
+		"Interface":              flow.Interface,
+		"AgentIP":                ipToStr(flow.AgentIp),
+		"Flags":                  flow.Flags,
+		"IcmpType":               flow.GetIcmpType(),
+		"IcmpCode":               flow.GetIcmpCode(),
+		"TcpDropBytes":           flow.TcpDropBytes,
+		"TcpDropPackets":         flow.TcpDropPackets,
+		"TcpDropLatestFlags":     flow.GetTcpDropLatestFlags(),
+		"TcpDropLatestState":     tcpStateToStr(flow.GetTcpDropLatestState()),
+		"TcpDropLatestDropCause": tcpDropCauseToStr(flow.GetTcpDropLatestDropCause()),
+		"DnsRequestTimeMs":       flow.TimeDnsReq.AsTime().UnixMilli(),
+		"DnsResponseTimeMs":      flow.TimeDnsRsp.AsTime().UnixMilli(),
+		"DnsId":                  flow.GetDnsId(),
+		"DnsFlags":               flow.GetDnsFlags(),
 	}
 	return out
 }
@@ -80,3 +89,189 @@ func macToStr(mac uint64) string {
 		uint8(mac>>8),
 		uint8(mac))
 }
+
+// tcpStateToStr is based on kernel TCP state definition
+// https://elixir.bootlin.com/linux/v6.3/source/include/net/tcp_states.h#L12
+func tcpStateToStr(state uint32) string {
+	switch state {
+	case 1:
+		return "TCP_ESTABLISHED"
+	case 2:
+		return "TCP_SYN_SENT"
+	case 3:
+		return "TCP_SYN_RECV"
+	case 4:
+		return "TCP_FIN_WAIT1"
+	case 5:
+		return "TCP_FIN_WAIT2"
+	case 6:
+		return "TCP_CLOSE"
+	case 7:
+		return "TCP_CLOSE_WAIT"
+	case 8:
+		return "TCP_LAST_ACK"
+	case 9:
+		return "TCP_LISTEN"
+	case 10:
+		return "TCP_CLOSING"
+	case 11:
+		return "TCP_NEW_SYN_RECV"
+	}
+	return "TCP_INVALID_STATE"
+}
+
+// tcpDropCauseToStr is based on kernel drop cause definition
+// https://elixir.bootlin.com/linux/latest/source/include/net/dropreason.h#L88
+func tcpDropCauseToStr(dropCause uint32) string {
+	switch dropCause {
+	case 2:
+		return "SKB_DROP_REASON_NOT_SPECIFIED"
+	case 3:
+		return "SKB_DROP_REASON_NO_SOCKET"
+	case 4:
+		return "SKB_DROP_REASON_PKT_TOO_SMALL"
+	case 5:
+		return "SKB_DROP_REASON_TCP_CSUM"
+	case 6:
+		return "SKB_DROP_REASON_SOCKET_FILTER"
+	case 7:
+		return "SKB_DROP_REASON_UDP_CSUM"
+	case 8:
+		return "SKB_DROP_REASON_NETFILTER_DROP"
+	case 9:
+		return "SKB_DROP_REASON_OTHERHOST"
+	case 10:
+		return "SKB_DROP_REASON_IP_CSUM"
+	case 11:
+		return "SKB_DROP_REASON_IP_INHDR"
+	case 12:
+		return "SKB_DROP_REASON_IP_RPFILTER"
+	case 13:
+		return "SKB_DROP_REASON_UNICAST_IN_L2_MULTICAST"
+	case 14:
+		return "SKB_DROP_REASON_XFRM_POLICY"
+	case 15:
+		return "SKB_DROP_REASON_IP_NOPROTO"
+	case 16:
+		return "SKB_DROP_REASON_SOCKET_RCVBUFF"
+	case 17:
+		return "SKB_DROP_REASON_PROTO_MEM"
+	case 18:
+		return "SKB_DROP_REASON_TCP_MD5NOTFOUND"
+	case 19:
+		return "SKB_DROP_REASON_TCP_MD5UNEXPECTED"
+	case 20:
+		return "SKB_DROP_REASON_TCP_MD5FAILURE"
+	case 21:
+		return "SKB_DROP_REASON_SOCKET_BACKLOG"
+	case 22:
+		return "SKB_DROP_REASON_TCP_FLAGS"
+	case 23:
+		return "SKB_DROP_REASON_TCP_ZEROWINDOW"
+	case 24:
+		return "SKB_DROP_REASON_TCP_OLD_DATA"
+	case 25:
+		return "SKB_DROP_REASON_TCP_OVERWINDOW"
+	case 26:
+		return "SKB_DROP_REASON_TCP_OFOMERGE"
+	case 27:
+		return "SKB_DROP_REASON_TCP_RFC7323_PAWS"
+	case 28:
+		return "SKB_DROP_REASON_TCP_INVALID_SEQUENCE"
+	case 29:
+		return "SKB_DROP_REASON_TCP_RESET"
+	case 30:
+		return "SKB_DROP_REASON_TCP_INVALID_SYN"
+	case 31:
+		return "SKB_DROP_REASON_TCP_CLOSE"
+	case 32:
+		return "SKB_DROP_REASON_TCP_FASTOPEN"
+	case 33:
+		return "SKB_DROP_REASON_TCP_OLD_ACK"
+	case 34:
+		return "SKB_DROP_REASON_TCP_TOO_OLD_ACK"
+	case 35:
+		return "SKB_DROP_REASON_TCP_ACK_UNSENT_DATA"
+	case 36:
+		return "SKB_DROP_REASON_TCP_OFO_QUEUE_PRUNE"
+	case 37:
+		return "SKB_DROP_REASON_TCP_OFO_DROP"
+	case 38:
+		return "SKB_DROP_REASON_IP_OUTNOROUTES"
+	case 39:
+		return "SKB_DROP_REASON_BPF_CGROUP_EGRESS"
+	case 40:
+		return "SKB_DROP_REASON_IPV6DISABLED"
+	case 41:
+		return "SKB_DROP_REASON_NEIGH_CREATEFAIL"
+	case 42:
+		return "SKB_DROP_REASON_NEIGH_FAILED"
+	case 43:
+		return "SKB_DROP_REASON_NEIGH_QUEUEFULL"
+	case 44:
+		return "SKB_DROP_REASON_NEIGH_DEAD"
+	case 45:
+		return "SKB_DROP_REASON_TC_EGRESS"
+	case 46:
+		return "SKB_DROP_REASON_QDISC_DROP"
+	case 47:
+		return "SKB_DROP_REASON_CPU_BACKLOG"
+	case 48:
+		return "SKB_DROP_REASON_XDP"
+	case 49:
+		return "SKB_DROP_REASON_TC_INGRESS"
+	case 50:
+		return "SKB_DROP_REASON_UNHANDLED_PROTO"
+	case 51:
+		return "SKB_DROP_REASON_SKB_CSUM"
+	case 52:
+		return "SKB_DROP_REASON_SKB_GSO_SEG"
+	case 53:
+		return "SKB_DROP_REASON_SKB_UCOPY_FAULT"
+	case 54:
+		return "SKB_DROP_REASON_DEV_HDR"
+	case 55:
+		return "SKB_DROP_REASON_DEV_READY"
+	case 56:
+		return "SKB_DROP_REASON_FULL_RING"
+	case 57:
+		return "SKB_DROP_REASON_NOMEM"
+	case 58:
+		return "SKB_DROP_REASON_HDR_TRUNC"
+	case 59:
+		return "SKB_DROP_REASON_TAP_FILTER"
+	case 60:
+		return "SKB_DROP_REASON_TAP_TXFILTER"
+	case 61:
+		return "SKB_DROP_REASON_ICMP_CSUM"
+	case 62:
+		return "SKB_DROP_REASON_INVALID_PROTO"
+	case 63:
+		return "SKB_DROP_REASON_IP_INADDRERRORS"
+	case 64:
+		return "SKB_DROP_REASON_IP_INNOROUTES"
+	case 65:
+		return "SKB_DROP_REASON_PKT_TOO_BIG"
+	case 66:
+		return "SKB_DROP_REASON_DUP_FRAG"
+	case 67:
+		return "SKB_DROP_REASON_FRAG_REASM_TIMEOUT"
+	case 68:
+		return "SKB_DROP_REASON_FRAG_TOO_FAR"
+	case 69:
+		return "SKB_DROP_REASON_TCP_MINTTL"
+	case 70:
+		return "SKB_DROP_REASON_IPV6_BAD_EXTHDR"
+	case 71:
+		return "SKB_DROP_REASON_IPV6_NDISC_FRAG"
+	case 72:
+		return "SKB_DROP_REASON_IPV6_NDISC_HOP_LIMIT"
+	case 73:
+		return "SKB_DROP_REASON_IPV6_NDISC_BAD_CODE"
+	case 74:
+		return "SKB_DROP_REASON_IPV6_NDISC_BAD_OPTIONS"
+	case 75:
+		return "SKB_DROP_REASON_IPV6_NDISC_NS_OTHERHOST"
+	}
+	return "SKB_DROP_UNKNOWN_CAUSE"
+}