From 5085aa47f3d68960d83358afb4128f5351895265 Mon Sep 17 00:00:00 2001 From: Karina5005 Date: Sun, 5 Mar 2023 21:44:31 +0300 Subject: [PATCH 1/4] fix pagination problem --- backup_github/github.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/backup_github/github.py b/backup_github/github.py index 466ae66..0230acc 100644 --- a/backup_github/github.py +++ b/backup_github/github.py @@ -90,12 +90,19 @@ def __init__( self.retry_seconds = retry_seconds @retry - def make_request(self, url, params=None): - resp = requests.get(url, headers=self.headers, params=params) - logging.info(f"Make request to {url}") - self.raise_by_status(resp) - logging.info("OK") - return resp.json() + def make_request(self, url, params={}): + res = [] + params["page"] = 1 + while True: + resp = requests.get(url, headers=self.headers, params=params) + logging.info(f"Make request to {url}") + self.raise_by_status(resp) + logging.info("OK") + if not isinstance(resp.json(), list) or len(resp.json()) == 0: + break + res += resp.json() + params["page"] = params["page"] + 1 + return res def get_organization(self): return self.make_request("https://api.github.com/orgs/" + self.organization) From 718017ae159fcd25c96ca13c423dade2ec643b4e Mon Sep 17 00:00:00 2001 From: Karina5005 Date: Sun, 5 Mar 2023 22:23:39 +0300 Subject: [PATCH 2/4] fix tests --- backup_github/github.py | 7 +- tests/backup_test.py | 166 +++++++++++++++++++++++++++++++++++---- tests/github_api_test.py | 23 ++++-- 3 files changed, 175 insertions(+), 21 deletions(-) diff --git a/backup_github/github.py b/backup_github/github.py index 0230acc..c89a67c 100644 --- a/backup_github/github.py +++ b/backup_github/github.py @@ -98,9 +98,12 @@ def make_request(self, url, params={}): logging.info(f"Make request to {url}") self.raise_by_status(resp) logging.info("OK") - if not isinstance(resp.json(), list) or len(resp.json()) == 0: + if isinstance(resp.json(), list): + res += resp.json() + else: + return resp.json() + if len(resp.json()) == 0: break - res += resp.json() params["page"] = params["page"] + 1 return res diff --git a/tests/backup_test.py b/tests/backup_test.py index e4845fa..ea28077 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -55,7 +55,7 @@ def check_json(self, expected, path): def test_backup_members(self): with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/orgs/org/members", + url="https://api.github.com/orgs/org/members?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -63,7 +63,15 @@ def test_backup_members(self): response_list=[{"json": self.users, "status_code": 200}], ) m.get( - url="https://api.github.com/orgs/org/memberships/test1", + url="https://api.github.com/orgs/org/members?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[{"json": [], "status_code": 200}], + ) + m.get( + url="https://api.github.com/orgs/org/memberships/test1?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -73,7 +81,7 @@ def test_backup_members(self): ], ) m.get( - url="https://api.github.com/orgs/org/memberships/test2", + url="https://api.github.com/orgs/org/memberships/test2?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -104,7 +112,7 @@ def test_backup_issues(self): os.makedirs(self.backup.output_dir + "/repos/test") with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/issues", + url="https://api.github.com/repos/org/test/issues?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -138,7 +146,20 @@ def test_backup_issues(self): ], ) m.get( - url="https://api.github.com/repos/org/test/issues/1/comments", + url="https://api.github.com/repos/org/test/issues?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -171,7 +192,7 @@ def test_backup_issues_comments(self): os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True) with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/issues", + url="https://api.github.com/repos/org/test/issues?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -205,7 +226,20 @@ def test_backup_issues_comments(self): ], ) m.get( - url="https://api.github.com/repos/org/test/issues/1/comments", + url="https://api.github.com/repos/org/test/issues?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -230,6 +264,19 @@ def test_backup_issues_comments(self): } ], ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) self.backup.backup_issues() self.check_json( {"id": 1, "body": "comment1", "created_at": "2022-10-24T10:05:33Z"}, @@ -252,7 +299,7 @@ def test_backup_pull(self): os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True) with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/pulls", + url="https://api.github.com/repos/org/test/pulls?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -303,6 +350,19 @@ def test_backup_pull(self): } ], ) + m.get( + url="https://api.github.com/repos/org/test/pulls?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) m.get( url="https://api.github.com/repos/org/test/issues/1/comments", request_headers={ @@ -351,7 +411,7 @@ def test_backup_pull_comments(self): os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True) with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/pulls", + url="https://api.github.com/repos/org/test/pulls?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -384,7 +444,20 @@ def test_backup_pull_comments(self): ], ) m.get( - url="https://api.github.com/repos/org/test/issues/1/comments", + url="https://api.github.com/repos/org/test/pulls?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -403,6 +476,19 @@ def test_backup_pull_comments(self): } ], ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) m.get( url="https://api.github.com/repos/org/test/pulls/1/reviews", request_headers={ @@ -425,7 +511,7 @@ def test_backup_pull_review(self): os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True) with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/pulls", + url="https://api.github.com/repos/org/test/pulls?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -458,7 +544,20 @@ def test_backup_pull_review(self): ], ) m.get( - url="https://api.github.com/repos/org/test/issues/1/comments", + url="https://api.github.com/repos/org/test/pulls?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -478,7 +577,20 @@ def test_backup_pull_review(self): ], ) m.get( - url="https://api.github.com/repos/org/test/pulls/1/reviews", + url="https://api.github.com/repos/org/test/issues/1/comments?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/pulls/1/reviews?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -500,7 +612,20 @@ def test_backup_pull_review(self): ], ) m.get( - url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments", + url="https://api.github.com/repos/org/test/pulls/1/reviews?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -526,6 +651,19 @@ def test_backup_pull_review(self): } ], ) + m.get( + url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) m.get( url="https://api.github.com/repos/org/test/pulls/1/reviews/2/comments", request_headers={ diff --git a/tests/github_api_test.py b/tests/github_api_test.py index dc239ee..9b0e456 100644 --- a/tests/github_api_test.py +++ b/tests/github_api_test.py @@ -13,7 +13,7 @@ class TestGithubApi: def test_make_request(self): with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/orgs/test/members", + url="https://api.github.com/orgs/test/members?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer test_token", @@ -31,6 +31,19 @@ def test_make_request(self): } ], ) + m.get( + url="https://api.github.com/orgs/test/members?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer test_token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) resp = self.gh.make_request("https://api.github.com/orgs/test/members") assert resp == [ { @@ -72,7 +85,7 @@ def test_make_request_retry(self): ], ) assert ( - self.gh.make_request("https://api.github.com/orgs/test/members") == {} + self.gh.make_request("https://api.github.com/orgs/test/members") == {} ) def test_make_request_rate_limit_exceeded_ok(self): @@ -115,7 +128,7 @@ def test_make_request_rate_limit_exceeded_ok(self): ], ) assert ( - self.gh.make_request("https://api.github.com/orgs/test/members") == {} + self.gh.make_request("https://api.github.com/orgs/test/members") == {} ) def test_make_request_rate_limit_exceeded_fail(self): @@ -158,8 +171,8 @@ def test_make_request_rate_limit_exceeded_fail(self): ) with pytest.raises(Exception): assert ( - self.gh.make_request("https://api.github.com/orgs/test/members") - == {} + self.gh.make_request("https://api.github.com/orgs/test/members") + == {} ) def test_make_request_timeout(self): From ba8e1e0f6671d8d3d57d89f7e477cd2fca8eae64 Mon Sep 17 00:00:00 2001 From: Karina5005 Date: Sun, 5 Mar 2023 22:26:01 +0300 Subject: [PATCH 3/4] fix format --- tests/github_api_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/github_api_test.py b/tests/github_api_test.py index 9b0e456..d67f123 100644 --- a/tests/github_api_test.py +++ b/tests/github_api_test.py @@ -85,7 +85,7 @@ def test_make_request_retry(self): ], ) assert ( - self.gh.make_request("https://api.github.com/orgs/test/members") == {} + self.gh.make_request("https://api.github.com/orgs/test/members") == {} ) def test_make_request_rate_limit_exceeded_ok(self): @@ -128,7 +128,7 @@ def test_make_request_rate_limit_exceeded_ok(self): ], ) assert ( - self.gh.make_request("https://api.github.com/orgs/test/members") == {} + self.gh.make_request("https://api.github.com/orgs/test/members") == {} ) def test_make_request_rate_limit_exceeded_fail(self): @@ -171,8 +171,8 @@ def test_make_request_rate_limit_exceeded_fail(self): ) with pytest.raises(Exception): assert ( - self.gh.make_request("https://api.github.com/orgs/test/members") - == {} + self.gh.make_request("https://api.github.com/orgs/test/members") + == {} ) def test_make_request_timeout(self): From 574668afbc994b9ff411f2db4093e9a5ac6ec01e Mon Sep 17 00:00:00 2001 From: Karina5005 Date: Mon, 6 Mar 2023 11:31:36 +0300 Subject: [PATCH 4/4] add per_page --- backup_github/github.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backup_github/github.py b/backup_github/github.py index c89a67c..bc3379c 100644 --- a/backup_github/github.py +++ b/backup_github/github.py @@ -93,6 +93,7 @@ def __init__( def make_request(self, url, params={}): res = [] params["page"] = 1 + params["per_page"] = 100 while True: resp = requests.get(url, headers=self.headers, params=params) logging.info(f"Make request to {url}")