forked from netobserv/network-observability-operator
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request netobserv#160 from mariomac/ebpf
[NETOBSERV-237] gRPC+PB flow ingest/decoder for NetObserv eBPF Agent
- Loading branch information
Showing
14 changed files
with
383 additions
and
208 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| package api | ||
|
|
||
| type IngestGRPCProto struct { | ||
| Port int `yaml:"port" doc:"the port number to listen on"` | ||
| BufferLen int `yaml:"buffer_length" doc:"the length of the ingest channel buffer, in groups of flows, containing each group hundreds of flows (default: 100)"` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package api | ||
|
|
||
| type WriteStdout struct { | ||
| Format string `yaml:"format" doc:"the format of each line: printf (default) or json"` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| package decode | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "net" | ||
| "time" | ||
|
|
||
| "github.com/netobserv/flowlogs-pipeline/pkg/config" | ||
| "github.com/netobserv/netobserv-agent/pkg/pbflow" | ||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| var pflog = logrus.WithField("component", "Protobuf") | ||
|
|
||
| // Protobuf decodes protobuf flow records definitions, as forwarded by | ||
| // ingest.NetObservAgent, into a Generic Map that follows the same naming conventions | ||
| // as the IPFIX flows from ingest.IngestCollector | ||
| type Protobuf struct { | ||
| } | ||
|
|
||
| func NewProtobuf() (Decoder, error) { | ||
| return &Protobuf{}, nil | ||
| } | ||
|
|
||
| // Decode decodes input strings to a list of flow entries | ||
| func (p *Protobuf) Decode(in []interface{}) []config.GenericMap { | ||
| if len(in) == 0 { | ||
| pflog.Warn("empty input. Skipping") | ||
| return []config.GenericMap{} | ||
| } | ||
| pb, ok := in[0].(*pbflow.Records) | ||
| if !ok { | ||
| pflog.WithField("type", fmt.Sprintf("%T", pb)). | ||
| Warn("expecting input to be *pbflow.Records. Skipping") | ||
| } | ||
| out := make([]config.GenericMap, 0, len(pb.Entries)) | ||
| for _, entry := range pb.Entries { | ||
| out = append(out, pbFlowToMap(entry)) | ||
| } | ||
| return out | ||
| } | ||
|
|
||
| func pbFlowToMap(flow *pbflow.Record) config.GenericMap { | ||
| if flow == nil { | ||
| return config.GenericMap{} | ||
| } | ||
| out := config.GenericMap{ | ||
| "FlowDirection": int(flow.Direction.Number()), | ||
| "Bytes": flow.Bytes, | ||
| "SrcAddr": ipToStr(flow.Network.GetSrcAddr()), | ||
| "DstAddr": ipToStr(flow.Network.GetDstAddr()), | ||
| "SrcMac": macToStr(flow.DataLink.GetSrcMac()), | ||
| "DstMac": macToStr(flow.DataLink.GetDstMac()), | ||
| "SrcPort": flow.Transport.GetSrcPort(), | ||
| "DstPort": flow.Transport.GetDstPort(), | ||
| "Etype": flow.EthProtocol, | ||
| "Packets": flow.Packets, | ||
| "Proto": flow.Transport.GetProtocol(), | ||
| "TimeFlowStart": flow.TimeFlowStart.GetSeconds(), | ||
| "TimeFlowEnd": flow.TimeFlowEnd.GetSeconds(), | ||
| "TimeReceived": time.Now().Unix(), | ||
| "Interface": flow.Interface, | ||
| } | ||
| return out | ||
| } | ||
|
|
||
| func ipToStr(ip *pbflow.IP) string { | ||
| if ip.GetIpv6() != nil { | ||
| return net.IP(ip.GetIpv6()).String() | ||
| } else { | ||
| n := ip.GetIpv4() | ||
| return fmt.Sprintf("%d.%d.%d.%d", | ||
| byte(n>>24), byte(n>>16), byte(n>>8), byte(n)) | ||
| } | ||
| } | ||
|
|
||
| func macToStr(mac uint64) string { | ||
| return fmt.Sprintf("%02X:%02X:%02X:%02X:%02X:%02X", | ||
| uint8(mac>>40), | ||
| uint8(mac>>32), | ||
| uint8(mac>>24), | ||
| uint8(mac>>16), | ||
| uint8(mac>>8), | ||
| uint8(mac)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package decode | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/netobserv/flowlogs-pipeline/pkg/config" | ||
| "github.com/netobserv/netobserv-agent/pkg/pbflow" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| "google.golang.org/protobuf/types/known/timestamppb" | ||
| ) | ||
|
|
||
| func TestDecodePBFlows(t *testing.T) { | ||
| decoder := Protobuf{} | ||
|
|
||
| someTime := time.Now() | ||
| flow := &pbflow.Record{ | ||
| Interface: "eth0", | ||
| EthProtocol: 2048, | ||
| Bytes: 456, | ||
| Packets: 123, | ||
| Direction: pbflow.Direction_EGRESS, | ||
| TimeFlowStart: timestamppb.New(someTime), | ||
| TimeFlowEnd: timestamppb.New(someTime), | ||
| Network: &pbflow.Network{ | ||
| SrcAddr: &pbflow.IP{ | ||
| IpFamily: &pbflow.IP_Ipv4{Ipv4: 0x01020304}, | ||
| }, | ||
| DstAddr: &pbflow.IP{ | ||
| IpFamily: &pbflow.IP_Ipv4{Ipv4: 0x05060708}, | ||
| }, | ||
| }, | ||
| DataLink: &pbflow.DataLink{ | ||
| DstMac: 0x112233445566, | ||
| SrcMac: 0x010203040506, | ||
| }, | ||
| Transport: &pbflow.Transport{ | ||
| Protocol: 1, | ||
| SrcPort: 23000, | ||
| DstPort: 443, | ||
| }, | ||
| } | ||
|
|
||
| out := decoder.Decode([]interface{}{&pbflow.Records{Entries: []*pbflow.Record{flow}}}) | ||
| require.Len(t, out, 1) | ||
| assert.NotZero(t, out[0]["TimeReceived"]) | ||
| delete(out[0], "TimeReceived") | ||
| assert.Equal(t, config.GenericMap{ | ||
| "FlowDirection": 1, | ||
| "Bytes": uint64(456), | ||
| "SrcAddr": "1.2.3.4", | ||
| "DstAddr": "5.6.7.8", | ||
| "DstMac": "11:22:33:44:55:66", | ||
| "SrcMac": "01:02:03:04:05:06", | ||
| "SrcPort": uint32(23000), | ||
| "DstPort": uint32(443), | ||
| "Etype": uint32(2048), | ||
| "Packets": uint64(123), | ||
| "Proto": uint32(1), | ||
| "TimeFlowStart": someTime.Unix(), | ||
| "TimeFlowEnd": someTime.Unix(), | ||
| "Interface": "eth0", | ||
| }, out[0]) | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package ingest | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/netobserv/flowlogs-pipeline/pkg/config" | ||
| "github.com/netobserv/netobserv-agent/pkg/grpc" | ||
| "github.com/netobserv/netobserv-agent/pkg/pbflow" | ||
| ) | ||
|
|
||
| const defaultBufferLen = 100 | ||
|
|
||
| // GRPCProtobuf ingests data from the NetObserv eBPF Agent, using Protocol Buffers over gRPC | ||
| type GRPCProtobuf struct { | ||
| collector *grpc.CollectorServer | ||
| flowPackets chan *pbflow.Records | ||
| } | ||
|
|
||
| func NewGRPCProtobuf(params config.StageParam) (*GRPCProtobuf, error) { | ||
| netObserv := params.Ingest.GRPC | ||
| if netObserv.Port == 0 { | ||
| return nil, fmt.Errorf("ingest port not specified") | ||
| } | ||
| bufLen := netObserv.BufferLen | ||
| if bufLen == 0 { | ||
| bufLen = defaultBufferLen | ||
| } | ||
| flowPackets := make(chan *pbflow.Records, bufLen) | ||
| collector, err := grpc.StartCollector(netObserv.Port, flowPackets) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return &GRPCProtobuf{ | ||
| collector: collector, | ||
| flowPackets: flowPackets, | ||
| }, nil | ||
| } | ||
|
|
||
| func (no *GRPCProtobuf) Ingest(out chan<- []interface{}) { | ||
| for fp := range no.flowPackets { | ||
| out <- []interface{}{fp} | ||
| } | ||
| } | ||
|
|
||
| func (no *GRPCProtobuf) Close() error { | ||
| err := no.collector.Close() | ||
| close(no.flowPackets) | ||
| return err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.