From 2ec1ecb77e28a935a3986bab7abec14d81613cf7 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 23 Dec 2022 12:25:08 +0000 Subject: [PATCH 01/10] skip paging for `IssueRemoteLinks` stream Signed-off-by: Sergey Chvalyuk --- airbyte-integrations/connectors/source-jira/Dockerfile | 2 +- .../connectors/source-jira/source_jira/streams.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-jira/Dockerfile b/airbyte-integrations/connectors/source-jira/Dockerfile index b74fa3edf0c4..0df724eff5a0 100644 --- a/airbyte-integrations/connectors/source-jira/Dockerfile +++ b/airbyte-integrations/connectors/source-jira/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=0.3.2 LABEL io.airbyte.name=airbyte/source-jira diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 45fde4ece011..945075f80145 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -553,6 +553,9 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg for issue in read_full_refresh(self.issues_stream): yield from super().read_records(stream_slice={"key": issue["key"]}, **kwargs) + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + class IssueResolutions(JiraStream): """ From 36840d0c7d939790e36a76262b5aa72f5604eda9 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 23 Dec 2022 17:09:06 +0000 Subject: [PATCH 02/10] next_page_token improved Signed-off-by: Sergey Chvalyuk --- .../connectors/source-jira/source_jira/streams.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 945075f80145..32080af0e92b 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -48,7 +48,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, startAt = response_json.get("startAt") if startAt is not None: startAt += response_json["maxResults"] - if startAt < response_json["total"]: + if "total" in response_json and startAt < response_json["total"]: + return {"startAt": startAt} + if "values" in response_json and len(response_json["values"]) == self.page_size: return {"startAt": startAt} elif isinstance(response_json, list): if len(response_json) == self.page_size: From a58cf6c1aaa258d93b46b800a25687f0c3a3dde4 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 23 Dec 2022 17:13:10 +0000 Subject: [PATCH 03/10] jira.md updated Signed-off-by: Sergey Chvalyuk --- docs/integrations/sources/jira.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index 423d4cef658c..bd00db295051 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -126,6 +126,7 @@ The Jira connector should not run into Jira API limitations under normal usage. | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| +| 0.3.2 | 2022-12-23 | [\#20859](https://github.com/airbytehq/airbyte/pull/20859) | Fixed pagination for streams `issue_remote_links`, `sprints`, `sprint_issues` | | 0.3.1 | 2022-12-14 | [\#20128](https://github.com/airbytehq/airbyte/pull/20128) | Improved code to become beta | | 0.3.0 | 2022-11-03 | [\#18901](https://github.com/airbytehq/airbyte/pull/18901) | Adds UserGroupsDetailed schema, fix Incremental normalization, add Incremental support for IssueComments, IssueWorklogs | | 0.2.23 | 2022-10-28 | [\#18505](https://github.com/airbytehq/airbyte/pull/18505) | Correcting `max_results` bug introduced in connector stream | From e3a003795ce0434272c78c6c5cf1eae97aab0c05 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 23 Dec 2022 17:35:30 +0000 Subject: [PATCH 04/10] fix next_page_token Signed-off-by: Sergey Chvalyuk --- .../connectors/source-jira/source_jira/streams.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 32080af0e92b..e4efdf99e7bf 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -48,9 +48,10 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, startAt = response_json.get("startAt") if startAt is not None: startAt += response_json["maxResults"] - if "total" in response_json and startAt < response_json["total"]: - return {"startAt": startAt} - if "values" in response_json and len(response_json["values"]) == self.page_size: + if "total" in response_json: + if startAt < response_json["total"]: + return {"startAt": startAt} + elif len(response_json["values"]) == self.page_size: return {"startAt": startAt} elif isinstance(response_json, list): if len(response_json) == self.page_size: From 175550a13077d94c2ecaa39c02967ec84a8a277e Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Sat, 24 Dec 2022 07:00:35 +0000 Subject: [PATCH 05/10] test_pagination_projects added Signed-off-by: Sergey Chvalyuk --- .../source-jira/unit_tests/test_pagination.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py new file mode 100644 index 000000000000..9519d2e08e7e --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import json +from http import HTTPStatus + +import responses +from source_jira.streams import Projects +from source_jira.utils import read_full_refresh + + +@responses.activate +def test_pagination_projects(): + domain = "domain.com" + responses_json = [ + (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "isLast": False, "values": [{"self": "p1"}, {"self": "p2"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "isLast": False, "values": [{"self": "p3"}, {"self": "p4"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "isLast": True, "values": [{"self": "p5"}, {"self": "p6"}]})), + ] + + responses.add_callback( + responses.GET, + f"https://{domain}/rest/api/3/project/search", + callback=lambda request: responses_json.pop(0), + content_type="application/json", + ) + + stream = Projects(authenticator=None, domain=domain, projects=[]) + records = list(read_full_refresh(stream)) + assert records == [{"self": "p1"}, {"self": "p2"}, {"self": "p3"}, {"self": "p4"}, {"self": "p5"}, {"self": "p6"}] From 85bac3ded2b01b9edc848a28dce2bd773b609f97 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Sat, 24 Dec 2022 08:59:29 +0000 Subject: [PATCH 06/10] test_pagination_issues added Signed-off-by: Sergey Chvalyuk --- .../source-jira/unit_tests/test_pagination.py | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py index 9519d2e08e7e..fc63998dfa7d 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py @@ -7,7 +7,7 @@ from http import HTTPStatus import responses -from source_jira.streams import Projects +from source_jira.streams import Issues, Projects from source_jira.utils import read_full_refresh @@ -15,9 +15,9 @@ def test_pagination_projects(): domain = "domain.com" responses_json = [ - (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "isLast": False, "values": [{"self": "p1"}, {"self": "p2"}]})), - (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "isLast": False, "values": [{"self": "p3"}, {"self": "p4"}]})), - (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "isLast": True, "values": [{"self": "p5"}, {"self": "p6"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "isLast": False, "values": [{"id": "1"}, {"id": "2"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "isLast": False, "values": [{"id": "3"}, {"id": "4"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "isLast": True, "values": [{"id": "5"}, {"id": "6"}]})), ] responses.add_callback( @@ -29,4 +29,33 @@ def test_pagination_projects(): stream = Projects(authenticator=None, domain=domain, projects=[]) records = list(read_full_refresh(stream)) - assert records == [{"self": "p1"}, {"self": "p2"}, {"self": "p3"}, {"self": "p4"}, {"self": "p5"}, {"self": "p6"}] + assert records == [{"id": "1"}, {"id": "2"}, {"id": "3"}, {"id": "4"}, {"id": "5"}, {"id": "6"}] + + +@responses.activate +def test_pagination_issues(): + domain = "domain.com" + responses_json = [ + (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "issues": [{"id": "1", "updated": "2022-01-01"}, {"id": "2", "updated": "2022-01-01"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "issues": [{"id": "3", "updated": "2022-01-01"}, {"id": "4", "updated": "2022-01-01"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "issues": [{"id": "5", "updated": "2022-01-01"}, {"id": "6", "updated": "2022-01-01"}]})), + ] + + responses.add_callback( + responses.GET, + f"https://{domain}/rest/api/3/search", + callback=lambda request: responses_json.pop(0), + content_type="application/json", + ) + + stream = Issues(authenticator=None, domain=domain, projects=[]) + stream.transform = lambda record, **kwargs: record + records = list(read_full_refresh(stream)) + assert records == [ + {"id": "1", "updated": "2022-01-01"}, + {"id": "2", "updated": "2022-01-01"}, + {"id": "3", "updated": "2022-01-01"}, + {"id": "4", "updated": "2022-01-01"}, + {"id": "5", "updated": "2022-01-01"}, + {"id": "6", "updated": "2022-01-01"} + ] From 1defafe583e831492134e99ac8f1d50f9601db79 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Mon, 26 Dec 2022 12:36:59 +0000 Subject: [PATCH 07/10] next_page_token re-implemented Signed-off-by: Sergey Chvalyuk --- .../connectors/source-jira/source_jira/streams.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index e4efdf99e7bf..ac8e08330cab 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -43,16 +43,16 @@ def url_base(self) -> str: def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: response_json = response.json() if isinstance(response_json, dict): - if response_json.get("isLast"): - return startAt = response_json.get("startAt") if startAt is not None: startAt += response_json["maxResults"] - if "total" in response_json: - if startAt < response_json["total"]: - return {"startAt": startAt} - elif len(response_json["values"]) == self.page_size: - return {"startAt": startAt} + if "isLast" in response_json: + if response_json["isLast"]: + return + elif "total" in response_json: + if startAt >= response_json["total"]: + return + return {"startAt": startAt} elif isinstance(response_json, list): if len(response_json) == self.page_size: query_params = dict(parse_qsl(urlparse.urlparse(response.url).query)) From 4cea4c334535e938005a6ef064ac652e82a025bf Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 27 Dec 2022 07:06:49 +0000 Subject: [PATCH 08/10] test_pagination_users added Signed-off-by: Sergey Chvalyuk --- .../source-jira/unit_tests/test_pagination.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py index fc63998dfa7d..a370342663d2 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py @@ -7,7 +7,7 @@ from http import HTTPStatus import responses -from source_jira.streams import Issues, Projects +from source_jira.streams import Issues, Projects, Users from source_jira.utils import read_full_refresh @@ -59,3 +59,31 @@ def test_pagination_issues(): {"id": "5", "updated": "2022-01-01"}, {"id": "6", "updated": "2022-01-01"} ] + + +@responses.activate +def test_pagination_users(): + domain = "domain.com" + responses_json = [ + (HTTPStatus.OK, {}, json.dumps([{"self": "user1"}, {"self": "user2"}])), + (HTTPStatus.OK, {}, json.dumps([{"self": "user3"}, {"self": "user4"}])), + (HTTPStatus.OK, {}, json.dumps([{"self": "user5"}])), + ] + + responses.add_callback( + responses.GET, + f"https://{domain}/rest/api/3/users/search", + callback=lambda request: responses_json.pop(0), + content_type="application/json", + ) + + stream = Users(authenticator=None, domain=domain, projects=[]) + stream.page_size = 2 + records = list(read_full_refresh(stream)) + assert records == [ + {"self": "user1"}, + {"self": "user2"}, + {"self": "user3"}, + {"self": "user4"}, + {"self": "user5"}, + ] From 16063dcafd266c4e8e5beec7b5cb59bf20676932 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Tue, 27 Dec 2022 07:07:54 +0000 Subject: [PATCH 09/10] jira.md updated Signed-off-by: Sergey Chvalyuk --- docs/integrations/sources/jira.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index bd00db295051..046f002ac134 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -126,7 +126,7 @@ The Jira connector should not run into Jira API limitations under normal usage. | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| -| 0.3.2 | 2022-12-23 | [\#20859](https://github.com/airbytehq/airbyte/pull/20859) | Fixed pagination for streams `issue_remote_links`, `sprints`, `sprint_issues` | +| 0.3.2 | 2022-12-23 | [\#20859](https://github.com/airbytehq/airbyte/pull/20859) | Fixed pagination for streams `issue_remote_links`, `sprints` | | 0.3.1 | 2022-12-14 | [\#20128](https://github.com/airbytehq/airbyte/pull/20128) | Improved code to become beta | | 0.3.0 | 2022-11-03 | [\#18901](https://github.com/airbytehq/airbyte/pull/18901) | Adds UserGroupsDetailed schema, fix Incremental normalization, add Incremental support for IssueComments, IssueWorklogs | | 0.2.23 | 2022-10-28 | [\#18505](https://github.com/airbytehq/airbyte/pull/18505) | Correcting `max_results` bug introduced in connector stream | From 1278459e7776eaa950b1295df65a2ecf319cd36e Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 4 Jan 2023 07:26:57 +0000 Subject: [PATCH 10/10] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 276afe14ffcc..5a7b94dacbf0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -785,7 +785,7 @@ - name: Jira sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 dockerRepository: airbyte/source-jira - dockerImageTag: 0.3.1 + dockerImageTag: 0.3.2 documentationUrl: https://docs.airbyte.com/integrations/sources/jira icon: jira.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index b9c3688da03f..2b40465fb7c3 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6677,7 +6677,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-jira:0.3.1" +- dockerImage: "airbyte/source-jira:0.3.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/jira" connectionSpecification: