diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml index 2d8cde5bbb86..e55baac2c90d 100644 --- a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml @@ -28,6 +28,7 @@ tests: issue_events: ["airbytehq/integration-test", "created_at"] issue_milestones: ["airbytehq/integration-test", "updated_at"] issues: ["airbytehq/integration-test", "updated_at"] + project_columns: ["airbytehq/integration-test", "13167124", "updated_at"] projects: ["airbytehq/integration-test", "updated_at"] pull_request_stats: ["airbytehq/integration-test", "updated_at"] pull_requests: ["airbytehq/integration-test", "updated_at"] diff --git a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json index 70e53839d655..1cdcf20864fe 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json @@ -39,6 +39,16 @@ "updated_at": "2121-06-30T06:44:42Z" } }, + "project_columns": { + "airbytehq/integration-test": { + "13167122": { + "updated_at": "2121-06-29T02:04:57Z" + }, + "13167124": { + "updated_at": "2121-06-29T02:04:57Z" + } + } + }, "projects": { "airbytehq/integration-test": { "updated_at": "2121-06-28T17:24:51Z" diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index 97b5ce92902c..79607d963984 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -151,15 +151,15 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> except Exception as e: message = repr(e) - if '404 Client Error: Not Found for url: https://api.github.com/repos/' in message: + if "404 Client Error: Not Found for url: https://api.github.com/repos/" in message: # HTTPError('404 Client Error: Not Found for url: https://api.github.com/repos/airbytehq/airbyte3?per_page=100')" - full_repo_name = message.split('https://api.github.com/repos/')[1] - full_repo_name = full_repo_name.split('?')[0] + full_repo_name = message.split("https://api.github.com/repos/")[1] + full_repo_name = full_repo_name.split("?")[0] message = f'Unknown repo name: "{full_repo_name}", use existing full repo name /' - elif '404 Client Error: Not Found for url: https://api.github.com/orgs/' in message: + elif "404 Client Error: Not Found for url: https://api.github.com/orgs/" in message: # HTTPError('404 Client Error: Not Found for url: https://api.github.com/orgs/airbytehqBLA/repos?per_page=100')" - org_name = message.split('https://api.github.com/orgs/')[1] - org_name = org_name.split('/')[0] + org_name = message.split("https://api.github.com/orgs/")[1] + org_name = org_name.split("/")[0] message = f'Unknown organization name: "{org_name}"' return False, message @@ -178,6 +178,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: default_branches, branches_to_pull = self._get_branches_data(config.get("branch", ""), repository_args) pull_requests_stream = PullRequests(**repository_args_with_start_date) + projects_stream = Projects(**repository_args_with_start_date) return [ Assignees(**repository_args), @@ -196,8 +197,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: IssueReactions(**repository_args_with_start_date), Issues(**repository_args_with_start_date), Organizations(**organization_args), - ProjectColumns(Projects(**repository_args_with_start_date), **repository_args_with_start_date), - Projects(**repository_args_with_start_date), + ProjectColumns(projects_stream, **repository_args_with_start_date), + projects_stream, PullRequestCommentReactions(**repository_args_with_start_date), PullRequestStats(parent=pull_requests_stream, **repository_args_with_start_date), PullRequests(**repository_args_with_start_date), diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index e0045bf538d0..38745e11dfe1 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -881,11 +881,12 @@ def read_records( yield record def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> str: - repository = stream_slice["repository"] - project_id = stream_slice["project_id"] - stream_state_value = stream_state.get(repository, {}).get(project_id, {}).get(self.cursor_field) - if stream_state_value: - return max(self._start_date, stream_state_value) + if stream_state: + repository = stream_slice["repository"] + project_id = stream_slice["project_id"] + stream_state_value = stream_state.get(repository, {}).get(project_id, {}).get(self.cursor_field) + if stream_state_value: + return max(self._start_date, stream_state_value) return self._start_date def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]):