[Athena] Fix: missing databases (apply pagination) (#2503)

* adding paginator on databases in glue

* reduce number of nested loops

* fixing indentation

* double for-loop instead of generator

* removing unused generator

redash/query_runner/athena.py

@@ -121,18 +121,20 @@ def __get_schema_from_glue(self):
                 region_name=self.configuration['region']
                 )
         schema = {}
-        paginator = client.get_paginator('get_tables')
-
-        for database in client.get_databases()['DatabaseList']:
-            iterator = paginator.paginate(DatabaseName=database['Name'])
-            for table in iterator.search('TableList[]'):
-                table_name = '%s.%s' % (database['Name'], table['Name'])
-                if table_name not in schema:
-                    column = [columns['Name'] for columns in table['StorageDescriptor']['Columns']]
-                    schema[table_name] = {'name': table_name, 'columns': column}
-                    for partition in table.get('PartitionKeys', []):
-                        schema[table_name]['columns'].append(partition['Name'])
 
+        database_paginator = client.get_paginator('get_databases')
+        table_paginator = client.get_paginator('get_tables')
+
+        for databases in database_paginator.paginate():
+            for database in databases['DatabaseList']:
+                iterator = table_paginator.paginate(DatabaseName=database['Name'])
+                for table in iterator.search('TableList[]'):
+                    table_name = '%s.%s' % (database['Name'], table['Name'])
+                    if table_name not in schema:
+                        column = [columns['Name'] for columns in table['StorageDescriptor']['Columns']]
+                        schema[table_name] = {'name': table_name, 'columns': column}
+                        for partition in table.get('PartitionKeys', []):
+                            schema[table_name]['columns'].append(partition['Name'])
         return schema.values()
 
     def get_schema(self, get_stats=False):