Skip to content

Commit

Permalink
#39: Pagination (#40)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Karina5005
Karina5005 authored Apr 10, 2023
1 parent abce3c6 commit 4149754
Show file tree
Hide file tree
Showing 3 changed files with 183 additions and 21 deletions.
23 changes: 17 additions & 6 deletions backup_github/github.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,23 @@ def __init__(
self.retry_seconds = retry_seconds

@retry
def make_request(self, url, params=None):
resp = requests.get(url, headers=self.headers, params=params)
logging.info(f"Make request to {url}")
self.raise_by_status(resp)
logging.info("OK")
return resp.json()
def make_request(self, url, params={}):
res = []
params["page"] = 1
params["per_page"] = 100
while True:
resp = requests.get(url, headers=self.headers, params=params)
logging.info(f"Make request to {url}")
self.raise_by_status(resp)
logging.info("OK")
if isinstance(resp.json(), list):
res += resp.json()
else:
return resp.json()
if len(resp.json()) == 0:
break
params["page"] = params["page"] + 1
return res

def get_organization(self):
return self.make_request("https://api.github.com/orgs/" + self.organization)
Expand Down
166 changes: 152 additions & 14 deletions tests/backup_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,23 @@ def check_json(self, expected, path):
def test_backup_members(self):
with requests_mock.Mocker() as m:
m.get(
url="https://api.github.com/orgs/org/members",
url="https://api.github.com/orgs/org/members?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[{"json": self.users, "status_code": 200}],
)
m.get(
url="https://api.github.com/orgs/org/memberships/test1",
url="https://api.github.com/orgs/org/members?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[{"json": [], "status_code": 200}],
)
m.get(
url="https://api.github.com/orgs/org/memberships/test1?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand All @@ -73,7 +81,7 @@ def test_backup_members(self):
],
)
m.get(
url="https://api.github.com/orgs/org/memberships/test2",
url="https://api.github.com/orgs/org/memberships/test2?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand Down Expand Up @@ -104,7 +112,7 @@ def test_backup_issues(self):
os.makedirs(self.backup.output_dir + "/repos/test")
with requests_mock.Mocker() as m:
m.get(
url="https://api.github.com/repos/org/test/issues",
url="https://api.github.com/repos/org/test/issues?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand Down Expand Up @@ -138,7 +146,20 @@ def test_backup_issues(self):
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments",
url="https://api.github.com/repos/org/test/issues?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand Down Expand Up @@ -171,7 +192,7 @@ def test_backup_issues_comments(self):
os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True)
with requests_mock.Mocker() as m:
m.get(
url="https://api.github.com/repos/org/test/issues",
url="https://api.github.com/repos/org/test/issues?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand Down Expand Up @@ -205,7 +226,20 @@ def test_backup_issues_comments(self):
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments",
url="https://api.github.com/repos/org/test/issues?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand All @@ -230,6 +264,19 @@ def test_backup_issues_comments(self):
}
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
self.backup.backup_issues()
self.check_json(
{"id": 1, "body": "comment1", "created_at": "2022-10-24T10:05:33Z"},
Expand All @@ -252,7 +299,7 @@ def test_backup_pull(self):
os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True)
with requests_mock.Mocker() as m:
m.get(
url="https://api.github.com/repos/org/test/pulls",
url="https://api.github.com/repos/org/test/pulls?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand Down Expand Up @@ -303,6 +350,19 @@ def test_backup_pull(self):
}
],
)
m.get(
url="https://api.github.com/repos/org/test/pulls?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments",
request_headers={
Expand Down Expand Up @@ -351,7 +411,7 @@ def test_backup_pull_comments(self):
os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True)
with requests_mock.Mocker() as m:
m.get(
url="https://api.github.com/repos/org/test/pulls",
url="https://api.github.com/repos/org/test/pulls?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand Down Expand Up @@ -384,7 +444,20 @@ def test_backup_pull_comments(self):
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments",
url="https://api.github.com/repos/org/test/pulls?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand All @@ -403,6 +476,19 @@ def test_backup_pull_comments(self):
}
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/pulls/1/reviews",
request_headers={
Expand All @@ -425,7 +511,7 @@ def test_backup_pull_review(self):
os.makedirs(self.backup.output_dir + "/repos/test", exist_ok=True)
with requests_mock.Mocker() as m:
m.get(
url="https://api.github.com/repos/org/test/pulls",
url="https://api.github.com/repos/org/test/pulls?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand Down Expand Up @@ -458,7 +544,20 @@ def test_backup_pull_review(self):
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments",
url="https://api.github.com/repos/org/test/pulls?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/issues/1/comments?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand All @@ -478,7 +577,20 @@ def test_backup_pull_review(self):
],
)
m.get(
url="https://api.github.com/repos/org/test/pulls/1/reviews",
url="https://api.github.com/repos/org/test/issues/1/comments?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/pulls/1/reviews?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand All @@ -500,7 +612,20 @@ def test_backup_pull_review(self):
],
)
m.get(
url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments",
url="https://api.github.com/repos/org/test/pulls/1/reviews?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
Expand All @@ -526,6 +651,19 @@ def test_backup_pull_review(self):
}
],
)
m.get(
url="https://api.github.com/repos/org/test/pulls/1/reviews/1/comments?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
m.get(
url="https://api.github.com/repos/org/test/pulls/1/reviews/2/comments",
request_headers={
Expand Down
15 changes: 14 additions & 1 deletion tests/github_api_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class TestGithubApi:
def test_make_request(self):
with requests_mock.Mocker() as m:
m.get(
url="https://api.github.com/orgs/test/members",
url="https://api.github.com/orgs/test/members?page=1",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer test_token",
Expand All @@ -31,6 +31,19 @@ def test_make_request(self):
}
],
)
m.get(
url="https://api.github.com/orgs/test/members?page=2",
request_headers={
"Accept": "application/vnd.github+json",
"Authorization": "Bearer test_token",
},
response_list=[
{
"json": [],
"status_code": 200,
}
],
)
resp = self.gh.make_request("https://api.github.com/orgs/test/members")
assert resp == [
{
Expand Down

0 comments on commit 4149754

Please sign in to comment.