Merge pull request #45 from obsrvbl/version-2.2.1

Version 2.2.1
obsrvbl-oss · Apr 13, 2020 · a87f21e · a87f21e
2 parents 8b9943c + 19ce8a5
commit a87f21e
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 3 deletions.
diff --git a/flowlogs_reader/flowlogs_reader.py b/flowlogs_reader/flowlogs_reader.py
@@ -31,6 +31,10 @@
 DEFAULT_REGION_NAME = 'us-east-1'
 DUPLICATE_NEXT_TOKEN_MESSAGE = 'The same next token was received twice'
 
+# The lastEventTimestamp may be delayed by up to an hour:
+# https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_LogStream.html  # noqa
+LAST_EVENT_DELAY_MSEC = 3600000
+
 ACCEPT = 'ACCEPT'
 REJECT = 'REJECT'
 SKIPDATA = 'SKIPDATA'
@@ -273,7 +277,10 @@ def _get_log_streams(self):
                 # Since we're ordering by last event timestamp, we're finished
                 # when we encounter a stream that ends before the time we
                 # care about.
-                if log_stream['lastEventTimestamp'] < self.start_ms:
+                last_event_ms = (
+                    log_stream['lastEventTimestamp'] + LAST_EVENT_DELAY_MSEC
+                )
+                if last_event_ms < self.start_ms:
                     break
 
                 yield log_stream['logStreamName']

diff --git a/setup.cfg b/setup.cfg
@@ -14,7 +14,7 @@
 
 [metadata]
 name = flowlogs_reader
-version = 2.2.0
+version = 2.2.1
 license = Apache
 url = https://github.com/obsrvbl/flowlogs-reader
 description = Reader for AWS VPC Flow Logs

diff --git a/tests/test_flowlogs_reader.py b/tests/test_flowlogs_reader.py
@@ -32,6 +32,7 @@
 from flowlogs_reader.flowlogs_reader import (
     DEFAULT_REGION_NAME,
     DUPLICATE_NEXT_TOKEN_MESSAGE,
+    LAST_EVENT_DELAY_MSEC,
 )
 
 
@@ -331,7 +332,9 @@ def _get_paginator(operation):
                             {
                                 'logStreamName': 'too_late',
                                 'firstEventTimestamp': inst.end_ms - 1,
-                                'lastEventTimestamp': inst.start_ms - 1,
+                                'lastEventTimestamp': (
+                                    inst.start_ms - LAST_EVENT_DELAY_MSEC - 1
+                                ),
                             },
                         ],
                     },