Merge pull request #52 from obsrvbl/v5-fields

Add support for the version 5 fields
obsrvbl-oss · Mar 4, 2021 · 336001e · 336001e
2 parents 456822e + e376964
commit 336001e
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 6 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -22,6 +22,9 @@ jobs:
         pip install .
         pip install -U coverage flake8
     - name: Run tests
+      env:
+        AWS_ACCESS_KEY_ID: fake-access-key-id
+        AWS_SECRET_ACCESS_KEY: fake-secret-access-key
       run: |
         coverage run --include "flowlogs_reader/*.py" -m unittest
     - name: Check coverage

diff --git a/flowlogs_reader/flowlogs_reader.py b/flowlogs_reader/flowlogs_reader.py
@@ -74,6 +74,10 @@ class FlowRecord:
         'az_id',
         'sublocation_type',
         'sublocation_id',
+        'pkt_src_aws_service',
+        'pkt_dst_aws_service',
+        'flow_direction',
+        'traffic_path',
     ]
 
     def __init__(self, event_data, EPOCH_32_MAX=2147483647):
@@ -120,6 +124,10 @@ def __init__(self, event_data, EPOCH_32_MAX=2147483647):
             ('az_id', str),
             ('sublocation_type', str),
             ('sublocation_id', str),
+            ('pkt_src_aws_service', str),
+            ('pkt_dst_aws_service', str),
+            ('flow_direction', str),
+            ('traffic_path', int),
         ):
             value = event_data.get(key, '-')
             value = None if (value == '-') else func(value)

diff --git a/setup.cfg b/setup.cfg
@@ -14,7 +14,7 @@
 
 [metadata]
 name = flowlogs_reader
-version = 2.3.0
+version = 2.4.0
 license = Apache
 url = https://github.com/obsrvbl/flowlogs-reader
 description = Reader for AWS VPC Flow Logs

diff --git a/tests/test_flowlogs_reader.py b/tests/test_flowlogs_reader.py
@@ -84,6 +84,22 @@
     '3 IPv4 4 vpc-0461a061 us-east-1 use1-az4 outpost outpostid04\n'
 )
 
+V5_FILE = (
+    'account-id action az-id bytes dstaddr dstport end flow-direction '
+    'instance-id interface-id log-status packets pkt-dst-aws-service '
+    'pkt-dstaddr pkt-src-aws-service pkt-srcaddr protocol region srcaddr '
+    'srcport start sublocation-id sublocation-type subnet-id tcp-flags '
+    'traffic-path type version vpc-id\n'
+    '999999999999 ACCEPT use2-az2 4895 192.0.2.156 50318 1614866511 '
+    'ingress i-00123456789abcdef eni-00123456789abcdef OK 15 - 192.0.2.156 '
+    'S3 198.51.100.6 6 us-east-2 198.51.100.7 443 1614866493 - - '
+    'subnet-0123456789abcdef 19 - IPv4 5 vpc-04456ab739938ee3f\n'
+    '999999999999 ACCEPT use2-az2 3015 198.51.100.6 443 1614866511 '
+    'egress i-00123456789abcdef eni-00123456789abcdef OK 16 S3 198.51.100.7 '
+    '- 192.0.2.156 6 us-east-2 192.0.2.156 50318 1614866493 - - '
+    'subnet-0123456789abcdef 7 7 IPv4 5 vpc-04456ab739938ee3f\n'
+)
+
 
 class FlowRecordTestCase(TestCase):
     def test_parse(self):
@@ -653,6 +669,66 @@ def test_serial_v4(self):
         ]
         self._test_iteration(V4_FILE, expected)
 
+    def test_serial_v5(self):
+        expected = [
+            {
+                'account_id': '999999999999',
+                'action': 'ACCEPT',
+                'az_id': 'use2-az2',
+                'bytes': 4895,
+                'dstaddr': '192.0.2.156',
+                'dstport': 50318,
+                'end': datetime(2021, 3, 4, 14, 1, 51),
+                'flow_direction': 'ingress',
+                'instance_id': 'i-00123456789abcdef',
+                'interface_id': 'eni-00123456789abcdef',
+                'log_status': 'OK',
+                'packets': 15,
+                'pkt_dstaddr': '192.0.2.156',
+                'pkt_src_aws_service': 'S3',
+                'pkt_srcaddr': '198.51.100.6',
+                'protocol': 6,
+                'region': 'us-east-2',
+                'srcaddr': '198.51.100.7',
+                'srcport': 443,
+                'start': datetime(2021, 3, 4, 14, 1, 33),
+                'subnet_id': 'subnet-0123456789abcdef',
+                'tcp_flags': 19,
+                'type': 'IPv4',
+                'version': 5,
+                'vpc_id': 'vpc-04456ab739938ee3f',
+            },
+            {
+                'account_id': '999999999999',
+                'action': 'ACCEPT',
+                'az_id': 'use2-az2',
+                'bytes': 3015,
+                'dstaddr': '198.51.100.6',
+                'dstport': 443,
+                'end': datetime(2021, 3, 4, 14, 1, 51),
+                'flow_direction': 'egress',
+                'instance_id': 'i-00123456789abcdef',
+                'interface_id': 'eni-00123456789abcdef',
+                'log_status': 'OK',
+                'packets': 16,
+                'pkt_dst_aws_service': 'S3',
+                'pkt_dstaddr': '198.51.100.7',
+                'pkt_srcaddr': '192.0.2.156',
+                'protocol': 6,
+                'region': 'us-east-2',
+                'srcaddr': '192.0.2.156',
+                'srcport': 50318,
+                'start': datetime(2021, 3, 4, 14, 1, 33),
+                'subnet_id': 'subnet-0123456789abcdef',
+                'tcp_flags': 7,
+                'traffic_path': 7,
+                'type': 'IPv4',
+                'version': 5,
+                'vpc_id': 'vpc-04456ab739938ee3f',
+            },
+        ]
+        self._test_iteration(V5_FILE, expected)
+
     def test_threads(self):
         expected = [
             {

diff --git a/tests/test_main.py b/tests/test_main.py
@@ -27,27 +27,27 @@
     (
         '2 123456789010 eni-102010ab 198.51.100.1 192.0.2.1 '
         '443 49152 6 10 840 1439387263 1439387264 ACCEPT OK '
-        '- - - - - - - - - - -'
+        '- - - - - - - - - - - - - - -'
     ),
     (
         '2 123456789010 eni-102010ab 192.0.2.1 198.51.100.1 '
         '49152 443 6 20 1680 1439387264 1439387265 ACCEPT OK '
-        '- - - - - - - - - - -'
+        '- - - - - - - - - - - - - - -'
     ),
     (
         '2 123456789010 eni-102010ab 192.0.2.1 198.51.100.2 '
         '49152 443 6 20 1680 1439387265 1439387266 REJECT OK '
-        '- - - - - - - - - - -'
+        '- - - - - - - - - - - - - - -'
     ),
     (
         '2 123456789010 eni-1a2b3c4d - - - - - - - '
         '1431280876 1431280934 - NODATA '
-        '- - - - - - - - - - -'
+        '- - - - - - - - - - - - - - -'
     ),
     (
         '2 123456789010 eni-4b118871 - - - - - - - '
         '1431280876 1431280934 - SKIPDATA '
-        '- - - - - - - - - - -'
+        '- - - - - - - - - - - - - - -'
     ),
 ]
 SAMPLE_RECORDS = [