use logging v3

gcp_flowlogs_reader/gcp_flowlogs_reader.py

@@ -1,16 +1,17 @@
 from datetime import datetime, timedelta
 from ipaddress import ip_address, IPv4Address, IPv6Address
-from time import sleep
 from typing import NamedTuple, Optional, Union
 
 from google.api_core.exceptions import (
     Forbidden,
     GoogleAPIError,
-    NotFound,
-    TooManyRequests,
+    NotFound
+)
+
+from google.cloud.logging import (
+    Client as LoggingClient,
+    StructEntry
 )
-from google.cloud.logging import Client as LoggingClient
-from google.cloud.logging.entries import StructEntry
 
 try:
     from google.cloud.resource_manager import Client as ResourceManagerClient
@@ -21,20 +22,6 @@
 BASE_LOG_NAME = 'projects/{}/logs/compute.googleapis.com%2Fvpc_flows'
 
 
-def page_helper(logging_client, wait_time=1.0, **kwargs):
-    kwargs['page_token'] = None
-    while True:
-        try:
-            iterator = logging_client.list_entries(**kwargs)
-            for page in iterator.pages:
-                kwargs['page_token'] = iterator.next_page_token
-                yield from page
-            break
-        except TooManyRequests:
-            sleep(wait_time)
-            pass
-
-
 class InstanceDetails(NamedTuple):
     project_id: str
     vm_name: str
@@ -278,12 +265,10 @@ def _reader(self):
 
         for project in self.project_list:
             try:
-                for flow_entry in page_helper(
-                    self.logging_client,
-                    wait_time=self.wait_time,
+                for flow_entry in self.logging_client.list_entries(
                     filter_=' AND '.join(filters),
                     page_size=self.page_size,
-                    projects=[project],
+                    resource_names=[f'projects/{project}'],
                 ):
                     self.bytes_processed += flow_entry.__sizeof__()
                     yield FlowRecord(flow_entry)

setup.cfg

@@ -25,7 +25,7 @@ classifiers =
 packages = find:
 python_requires = >=3.6
 install_requires =
-    google-cloud-logging < 2.0
+    google-cloud-logging >= 3.0
     google-cloud-resource-manager
     six