Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

🐛 Source Facebook: Improve rate limit management #4820

Merged
merged 3 commits into from
Jul 19, 2021
Merged

🐛 Source Facebook: Improve rate limit management #4820

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b55ad0c
Improve rate limit management
vladimir-remar Jul 19, 2021
00e19fb
bump version
vladimir-remar Jul 19, 2021
a1f6ffd
facebook-marketing.md update the changelog
vladimir-remar Jul 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 35 additions & 17 deletions airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,40 +39,58 @@ class MyFacebookAdsApi(FacebookAdsApi):
"""Custom Facebook API class to intercept all API calls and handle call rate limits"""

call_rate_threshold = 90 # maximum percentage of call limit utilization
pause_interval = pendulum.duration(minutes=1) # default pause interval if reached or close to call rate limit
pause_interval_minimum = pendulum.duration(minutes=1) # default pause interval if reached or close to call rate limit


@staticmethod
def parse_call_rate_header(headers):
call_count = 0
usage = 0
pause_interval = pendulum.duration()

usage_header = headers.get("x-business-use-case-usage") or headers.get("x-app-usage") or headers.get("x-ad-account-usage")
if usage_header:
usage_header = json.loads(usage_header)
call_count = usage_header.get("call_count") or usage_header.get("acc_id_util_pct") or 0
pause_interval = pendulum.duration(minutes=usage_header.get("estimated_time_to_regain_access", 0))
usage_header_business = headers.get("x-business-use-case-usage")
usage_header_app = headers.get("x-app-usage")
usage_header_ad_account = headers.get("x-ad-account-usage")

if usage_header_ad_account:
usage_header_ad_account_loaded = json.loads(usage_header_ad_account)
usage = max(usage, usage_header_ad_account_loaded.get("acc_id_util_pct") )

if usage_header_app:
usage_header_app_loaded = json.loads()
usage = max(usage, usage_header_app_loaded.get("call_count"), usage_header_app_loaded.get("total_time"), usage_header_app_loaded.get("total_cputime") )

return call_count, pause_interval
if usage_header_business:

usage_header_business_loaded = json.loads(usage_header_business)
for business_object_id in usage_header_business_loaded:
usage_limits = usage_header_business_loaded.get(business_object_id)[0]
usage = max(usage, usage_limits.get('call_count'), usage_limits.get('total_cputime'), usage_limits.get('total_time'))
pause_interval = max(pause_interval, pendulum.duration(minutes=usage_limits.get("estimated_time_to_regain_access", 0)))

return usage, pause_interval

def handle_call_rate_limit(self, response, params):
if "batch" in params:
max_call_count = 0
max_pause_interval = self.pause_interval
max_usage = 0
max_pause_interval = self.pause_interval_minimum

for record in response.json():
headers = {header["name"].lower(): header["value"] for header in record["headers"]}
call_count, pause_interval = self.parse_call_rate_header(headers)
max_call_count = max(max_call_count, call_count)
usage, pause_interval = self.parse_call_rate_header(headers)
max_usage = max(max_usage, usage)
max_pause_interval = max(max_pause_interval, pause_interval)

if max_call_count > self.call_rate_threshold:
logger.warn(f"Utilization is too high ({max_call_count})%, pausing for {max_pause_interval}")

if max_usage > self.call_rate_threshold:
max_pause_interval = max(max_pause_interval, self.pause_interval_minimum)
logger.warn(f"Utilization is too high ({max_usage})%, pausing for {max_pause_interval}")
sleep(max_pause_interval.total_seconds())
else:
headers = response.headers()
call_count, pause_interval = self.parse_call_rate_header(headers)
if call_count > self.call_rate_threshold or pause_interval:
logger.warn(f"Utilization is too high ({call_count})%, pausing for {pause_interval}")
usage, pause_interval = self.parse_call_rate_header(headers)
if usage > self.call_rate_threshold or pause_interval:
pause_interval = max(pause_interval, self.pause_interval_minimum)
logger.warn(f"Utilization is too high ({usage})%, pausing for {pause_interval}")
sleep(pause_interval.total_seconds())

def call(
Expand Down