Source Jira: Add Board, Epic, and Sprint-related streams and improve …

…request caching (#6621) * Create separate cache file per stream and add projects and start_date config * Add project id to issue record * Add projectId to issue schema * Add BoardIssues stream * Add SprintIssues stream * Add Epics stream and deduplicate state code * Add EpicIssues stream and additional fields for Issues stream * Add story points to sprint issues * Add new streams to test catalog * Update gitignore * Rename cache boolean and fix test catalog * Fix streams that depend on Issues stream * Fix sprint_issues stream * Add more fields to issues stream and format * Add option to expand issue changelogs * Remove epic_issues stream * Expand project descriptions * Show rendered fields for epics * Include project key * Include project key in issues stream * Address comments * Use CDK caching * Remove extra changes * Fix sprints stream reading from non-scrum boards * Format
airbytehq · Oct 20, 2021 · 1fcd6c4 · 1fcd6c4
1 parent c2eabc1
commit 1fcd6c4
Show file tree

Hide file tree

Showing 10 changed files with 430 additions and 123 deletions.
diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json
@@ -30,6 +30,16 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "board_issues",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "dashboards",
@@ -40,6 +50,16 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "epics",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "filters",
@@ -160,6 +180,16 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "sprint_issues",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "time_tracking",

diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config.json
@@ -1,5 +1,7 @@
 {
   "api_token": "invalid_token",
   "domain": "invaliddomain.atlassian.net",
-  "email": "test@test.com"
+  "email": "test@test.com",
+  "projects": ["invalidproject"],
+  "start_date": "2021-09-25T00:00:00Z"
 }
diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/board_issues.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/board_issues.json
@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "expand": {
+      "type": "string"
+    },
+    "id": {
+      "type": "string"
+    },
+    "self": {
+      "type": "string"
+    },
+    "key": {
+      "type": "string"
+    },
+    "fields": {
+      "type": "object"
+    },
+    "boardId": {
+      "type": "string"
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json
@@ -16,6 +16,9 @@
     },
     "projectId": {
       "type": "string"
+    },
+    "projectKey": {
+      "type": "string"
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/epics.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/epics.json
@@ -0,0 +1,64 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "expand": {
+      "type": "string",
+      "description": "Expand options that include additional issue details in the response.",
+      "readOnly": true,
+      "xml": {
+        "attribute": true
+      }
+    },
+    "id": {
+      "type": "string",
+      "description": "The ID of the epic.",
+      "readOnly": true
+    },
+    "self": {
+      "type": "string",
+      "description": "The URL of the epic details.",
+      "format": "uri",
+      "readOnly": true
+    },
+    "key": {
+      "type": "string",
+      "description": "The key of the epic.",
+      "readOnly": true
+    },
+    "fields": {
+      "type": "object",
+      "properties": {
+        "summary": {
+          "type": ["string", "null"],
+          "description": "Epic summary"
+        },
+        "description": {
+          "type": ["string", "null"],
+          "description": "Epic description"
+        },
+        "status": {
+          "type": ["string", "object"],
+          "description": "Epic status"
+        },
+        "updated": {
+          "type": ["string", "null"],
+          "format": "date-time",
+          "description": "This field is not shown in schema / swagger, but exists in records and we use it as cursor fiekd."
+        }
+      },
+      "additionalProperties": {}
+    },
+    "projectId": {
+      "type": "string",
+      "description": "The ID of the project containing the epic.",
+      "readOnly": true
+    },
+    "projectKey": {
+      "type": "string",
+      "description": "The key of the project containing the epic.",
+      "readOnly": true
+    }
+  },
+  "additionalProperties": false
+}
diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json
@@ -103,6 +103,16 @@
         }
       },
       "additionalProperties": {}
+    },
+    "projectId": {
+      "type": "string",
+      "description": "The ID of the project containing the issue.",
+      "readOnly": true
+    },
+    "projectKey": {
+      "type": "string",
+      "description": "The key of the project containing the issue.",
+      "readOnly": true
     }
   },
   "additionalProperties": false

diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprint_issues.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprint_issues.json
@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "expand": {
+      "type": "string"
+    },
+    "id": {
+      "type": "string"
+    },
+    "self": {
+      "type": "string"
+    },
+    "key": {
+      "type": "string"
+    },
+    "fields": {
+      "type": "object"
+    },
+    "sprintId": {
+      "type": "number"
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py
@@ -15,8 +15,10 @@
 from .streams import (
     ApplicationRoles,
     Avatars,
+    BoardIssues,
     Boards,
     Dashboards,
+    Epics,
     Filters,
     FilterSharing,
     Groups,
@@ -54,6 +56,7 @@
     ScreenSchemes,
     ScreenTabFields,
     ScreenTabs,
+    SprintIssues,
     Sprints,
     TimeTracking,
     Users,
@@ -96,33 +99,40 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         authenticator = self.get_authenticator(config)
-        args = {"authenticator": authenticator, "domain": config["domain"]}
+        args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]}
+        incremental_args = {**args, "start_date": config["start_date"]}
         return [
             ApplicationRoles(**args),
             Avatars(**args),
             Boards(**args),
+            BoardIssues(**incremental_args),
             Dashboards(**args),
+            Epics(**incremental_args),
             Filters(**args),
             FilterSharing(**args),
             Groups(**args),
-            Issues(**args),
-            IssueComments(**args),
+            Issues(
+                **incremental_args,
+                additional_fields=config.get("additional_fields", []),
+                expand_changelog=config.get("expand_issue_changelog", False)
+            ),
+            IssueComments(**incremental_args),
             IssueFields(**args),
             IssueFieldConfigurations(**args),
             IssueCustomFieldContexts(**args),
             IssueLinkTypes(**args),
             IssueNavigatorSettings(**args),
             IssueNotificationSchemes(**args),
             IssuePriorities(**args),
-            IssueProperties(**args),
-            IssueRemoteLinks(**args),
+            IssueProperties(**incremental_args),
+            IssueRemoteLinks(**incremental_args),
             IssueResolutions(**args),
             IssueSecuritySchemes(**args),
             IssueTypeSchemes(**args),
             IssueTypeScreenSchemes(**args),
-            IssueVotes(**args),
-            IssueWatchers(**args),
-            IssueWorklogs(**args),
+            IssueVotes(**incremental_args),
+            IssueWatchers(**incremental_args),
+            IssueWorklogs(**incremental_args),
             JiraSettings(**args),
             Labels(**args),
             Permissions(**args),
@@ -140,6 +150,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             ScreenTabFields(**args),
             ScreenSchemes(**args),
             Sprints(**args),
+            SprintIssues(**incremental_args),
             TimeTracking(**args),
             Users(**args),
             Workflows(**args),

diff --git a/airbyte-integrations/connectors/source-jira/source_jira/spec.json b/airbyte-integrations/connectors/source-jira/source_jira/spec.json
@@ -4,8 +4,8 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Jira Spec",
     "type": "object",
-    "required": ["api_token", "domain", "email"],
-    "additionalProperties": false,
+    "required": ["api_token", "domain", "email", "projects", "start_date"],
+    "additionalProperties": true,
     "properties": {
       "api_token": {
         "type": "string",
@@ -21,6 +21,37 @@
       "email": {
         "type": "string",
         "description": "The user email for your Jira account"
+      },
+      "projects": {
+        "type": "array",
+        "title": "Projects",
+        "items": {
+          "type": "string"
+        },
+        "examples": ["PROJ1", "PROJ2"],
+        "description": "Comma-separated list of Jira project keys to replicate data for"
+      },
+      "start_date": {
+        "type": "string",
+        "title": "Start Date",
+        "description": "The date from which you'd like to replicate data for Jira in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. Note that it will be used only in the following incremental streams: issues.",
+        "examples": ["2021-03-01T00:00:00Z"],
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+      },
+      "additional_fields": {
+        "type": "array",
+        "title": "Additional Fields",
+        "items": {
+          "type": "string"
+        },
+        "description": "Comma-separated list of additional fields to include in replicating issues",
+        "examples": ["Field A", "Field B"]
+      },
+      "expand_issue_changelog": {
+        "type": "boolean",
+        "title": "Expand Issue Changelog",
+        "description": "Expand the changelog when replicating issues",
+        "default": false
       }
     }
   }