From ccbc822e19e81e658f25ea0030d569223a1ce2cb Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Mon, 22 Jul 2024 18:02:37 -0500 Subject: [PATCH] Address release bugs Signed-off-by: Andrew Brain --- augur/tasks/github/messages.py | 8 ++++++-- augur/tasks/github/pull_requests/tasks.py | 11 +++++++---- augur/tasks/github/util/github_graphql_data_access.py | 4 ++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/augur/tasks/github/messages.py b/augur/tasks/github/messages.py index d47107c163..86fbe054fa 100644 --- a/augur/tasks/github/messages.py +++ b/augur/tasks/github/messages.py @@ -4,7 +4,7 @@ from augur.tasks.init.celery_app import celery_app as celery from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask from augur.application.db.data_parse import * -from augur.tasks.github.util.github_data_access import GithubDataAccess +from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException from augur.tasks.github.util.github_task_session import GithubTaskManifest from augur.tasks.util.worker_util import remove_duplicate_dicts from augur.tasks.github.util.util import get_owner_repo @@ -97,7 +97,11 @@ def process_large_issue_and_pr_message_collection(repo_id, repo_git: str, logger all_data = [] for comment_url in comment_urls: - messages = list(github_data_access.paginate_resource(comment_url)) + try: + messages = list(github_data_access.paginate_resource(comment_url)) + except UrlNotFoundException as e: + logger.warning(e) + continue all_data += messages diff --git a/augur/tasks/github/pull_requests/tasks.py b/augur/tasks/github/pull_requests/tasks.py index 017cf36385..c581ceb357 100644 --- a/augur/tasks/github/pull_requests/tasks.py +++ b/augur/tasks/github/pull_requests/tasks.py @@ -5,8 +5,7 @@ from augur.tasks.init.celery_app import celery_app as celery from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask, AugurSecondaryRepoCollectionTask from augur.application.db.data_parse import * -from augur.tasks.github.util.github_data_access import GithubDataAccess -from augur.tasks.github.util.github_paginator import GithubPaginator +from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException from augur.tasks.util.worker_util import remove_duplicate_dicts from augur.tasks.github.util.util import add_key_value_pair_to_dicts, get_owner_repo from augur.application.db.models import PullRequest, Message, PullRequestReview, PullRequestLabel, PullRequestReviewer, PullRequestMeta, PullRequestAssignee, PullRequestReviewMessageRef, Contributor, Repo @@ -359,8 +358,12 @@ def collect_pull_request_reviews(repo_git: str, full_collection: bool) -> None: pr_review_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/reviews" - pr_reviews = list(github_data_access.paginate_resource(pr_review_url)) - + try: + pr_reviews = list(github_data_access.paginate_resource(pr_review_url)) + except UrlNotFoundException as e: + logger.warning(e) + continue + if pr_reviews: all_pr_reviews[pull_request_id] = pr_reviews diff --git a/augur/tasks/github/util/github_graphql_data_access.py b/augur/tasks/github/util/github_graphql_data_access.py index 7604f3d593..aca316361d 100644 --- a/augur/tasks/github/util/github_graphql_data_access.py +++ b/augur/tasks/github/util/github_graphql_data_access.py @@ -145,6 +145,10 @@ def __extract_data_section(self, keys, json_response): # iterate deeper into the json_response object until we get to the desired data for value in keys: + + if core is None: + raise Exception(f"Error: 'core' is None when trying to index by {value}. Response: {json_response}") + core = core[value] return core