diff --git a/backup_github/github.py b/backup_github/github.py index 466ae66..bc3379c 100644 --- a/backup_github/github.py +++ b/backup_github/github.py @@ -90,12 +90,23 @@ def __init__( self.retry_seconds = retry_seconds @retry - def make_request(self, url, params=None): - resp = requests.get(url, headers=self.headers, params=params) - logging.info(f"Make request to {url}") - self.raise_by_status(resp) - logging.info("OK") - return resp.json() + def make_request(self, url, params={}): + res = [] + params["page"] = 1 + params["per_page"] = 100 + while True: + resp = requests.get(url, headers=self.headers, params=params) + logging.info(f"Make request to {url}") + self.raise_by_status(resp) + logging.info("OK") + if isinstance(resp.json(), list): + res += resp.json() + else: + return resp.json() + if len(resp.json()) == 0: + break + params["page"] = params["page"] + 1 + return res def get_organization(self): return self.make_request("https://api.github.com/orgs/" + self.organization) diff --git a/tests/backup_test.py b/tests/backup_test.py index e4845fa..ea28077 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -55,7 +55,7 @@ def check_json(self, expected, path): def test_backup_members(self): with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/orgs/org/members", + url="https://api.github.com/orgs/org/members?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -63,7 +63,15 @@ def test_backup_members(self): response_list=[{"json": self.users, "status_code": 200}], ) m.get( - url="https://api.github.com/orgs/org/memberships/test1", + url="https://api.github.com/orgs/org/members?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[{"json": [], "status_code": 200}], + ) + m.get( + url="https://api.github.com/orgs/org/memberships/test1?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -73,7 +81,7 @@ def test_backup_members(self): ], ) m.get( - url="https://api.github.com/orgs/org/memberships/test2", + url="https://api.github.com/orgs/org/memberships/test2?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -104,7 +112,7 @@ def test_backup_issues(self): os.makedirs(self.backup.output_dir + "/repos/test") with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/issues", + url="https://api.github.com/repos/org/test/issues?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -138,7 +146,20 @@ def test_backup_issues(self): ], ) m.get( - url="https://api.github.com/repos/org/test/issues/1/comments", + url="https://api.github.com/repos/org/test/issues?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -171,7 +192,7 @@ def test_backup_issues_comments(self): os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True) with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/issues", + url="https://api.github.com/repos/org/test/issues?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -205,7 +226,20 @@ def test_backup_issues_comments(self): ], ) m.get( - url="https://api.github.com/repos/org/test/issues/1/comments", + url="https://api.github.com/repos/org/test/issues?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -230,6 +264,19 @@ def test_backup_issues_comments(self): } ], ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) self.backup.backup_issues() self.check_json( {"id": 1, "body": "comment1", "created_at": "2022-10-24T10:05:33Z"}, @@ -252,7 +299,7 @@ def test_backup_pull(self): os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True) with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/pulls", + url="https://api.github.com/repos/org/test/pulls?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -303,6 +350,19 @@ def test_backup_pull(self): } ], ) + m.get( + url="https://api.github.com/repos/org/test/pulls?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) m.get( url="https://api.github.com/repos/org/test/issues/1/comments", request_headers={ @@ -351,7 +411,7 @@ def test_backup_pull_comments(self): os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True) with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/pulls", + url="https://api.github.com/repos/org/test/pulls?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -384,7 +444,20 @@ def test_backup_pull_comments(self): ], ) m.get( - url="https://api.github.com/repos/org/test/issues/1/comments", + url="https://api.github.com/repos/org/test/pulls?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -403,6 +476,19 @@ def test_backup_pull_comments(self): } ], ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) m.get( url="https://api.github.com/repos/org/test/pulls/1/reviews", request_headers={ @@ -425,7 +511,7 @@ def test_backup_pull_review(self): os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True) with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/repos/org/test/pulls", + url="https://api.github.com/repos/org/test/pulls?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -458,7 +544,20 @@ def test_backup_pull_review(self): ], ) m.get( - url="https://api.github.com/repos/org/test/issues/1/comments", + url="https://api.github.com/repos/org/test/pulls?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/issues/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -478,7 +577,20 @@ def test_backup_pull_review(self): ], ) m.get( - url="https://api.github.com/repos/org/test/pulls/1/reviews", + url="https://api.github.com/repos/org/test/issues/1/comments?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/pulls/1/reviews?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -500,7 +612,20 @@ def test_backup_pull_review(self): ], ) m.get( - url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments", + url="https://api.github.com/repos/org/test/pulls/1/reviews?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) + m.get( + url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer token", @@ -526,6 +651,19 @@ def test_backup_pull_review(self): } ], ) + m.get( + url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) m.get( url="https://api.github.com/repos/org/test/pulls/1/reviews/2/comments", request_headers={ diff --git a/tests/github_api_test.py b/tests/github_api_test.py index dc239ee..d67f123 100644 --- a/tests/github_api_test.py +++ b/tests/github_api_test.py @@ -13,7 +13,7 @@ class TestGithubApi: def test_make_request(self): with requests_mock.Mocker() as m: m.get( - url="https://api.github.com/orgs/test/members", + url="https://api.github.com/orgs/test/members?page=1", request_headers={ "Accept": "application/vnd.github+json", "Authorization": "Bearer test_token", @@ -31,6 +31,19 @@ def test_make_request(self): } ], ) + m.get( + url="https://api.github.com/orgs/test/members?page=2", + request_headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer test_token", + }, + response_list=[ + { + "json": [], + "status_code": 200, + } + ], + ) resp = self.gh.make_request("https://api.github.com/orgs/test/members") assert resp == [ {