Merge branch 'master' into grubberr/10384-source-github

airbytehq · Feb 23, 2022 · f7468c7 · f7468c7 · github-actions · Feb 23, 2022
2 parents ca2d242 + 342840d
commit f7468c7
Show file tree

Hide file tree

Showing 211 changed files with 34,457 additions and 39,388 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.32-alpha
+current_version = 0.35.35-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

diff --git a/.env b/.env
@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.35.32-alpha
+VERSION=0.35.35-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -83,3 +83,23 @@ If this is a community PR, the Airbyte engineer reviewing this PR is responsible
 - [ ] Documentation which references the generator is updated as needed
 
 </details>
+
+## Tests
+
+<details><summary><strong>Unit</strong></summary>
+
+*Put your unit tests output here.*
+
+</details>
+
+<details><summary><strong>Integration</strong></summary>
+
+*Put your integration tests output here.*
+
+</details>
+
+<details><summary><strong>Acceptance</strong></summary>
+
+*Put your acceptance tests output here.*
+
+</details>
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -305,8 +305,8 @@ jobs:
         with:
           java-version: "17"
 
-      # octavia-cli install and testing requires Python. 
-      # We use 3.8 in this project because 3.7 is not supported on Apple M1. 
+      # octavia-cli install and testing requires Python.
+      # We use 3.8 in this project because 3.7 is not supported on Apple M1.
       - uses: actions/setup-python@v2
         with:
           python-version: "3.8"
@@ -482,8 +482,8 @@ jobs:
         id: start-ec2-runner
         uses: ./.github/actions/start-aws-runner
         with:
-          # github-self-hosted-runner-ubuntu-20-with-120gdisk-docker-20.10.7
-          ec2-image-id: ami-0d4083c04fde515c4
+          # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
+          ec2-image-id: ami-0c1a9bc22624339d8
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
@@ -523,16 +523,6 @@ jobs:
           org.gradle.vfs.watch=false
           EOF
 
-      - name: Install socat (required for port forwarding)
-        run: |
-          # wait to receive lock (see https://askubuntu.com/questions/132059/how-to-make-a-package-manager-wait-if-another-instance-of-apt-is-running)
-          while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
-             sleep 1
-          done
-
-          sudo apt-get update
-          sudo apt-get install socat
-
       - name: Create cluster config file
         run: |
           cat > /tmp/kind-config.yaml <<EOF
@@ -644,8 +634,8 @@ jobs:
         id: start-ec2-runner
         uses: ./.github/actions/start-aws-runner
         with:
-          # github-self-hosted-runner-ubuntu-20-with-120gdisk-docker-20.10.7
-          ec2-image-id: ami-0d4083c04fde515c4
+          # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
+          ec2-image-id: ami-0c1a9bc22624339d8
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
@@ -685,16 +675,6 @@ jobs:
           org.gradle.vfs.watch=false
           EOF
 
-      - name: Install socat (required for port forwarding)
-        run: |
-          # wait to receive lock (see https://askubuntu.com/questions/132059/how-to-make-a-package-manager-wait-if-another-instance-of-apt-is-running)
-          while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
-             sleep 1
-          done
-
-          sudo apt-get update
-          sudo apt-get install socat
-
       - name: Create cluster config file
         run: |
           cat > /tmp/kind-config.yaml <<EOF

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,17 +8,18 @@ repos:
       - id: licenseheaders
         args: ["--tmpl=LICENSE_SHORT", "--ext=py", "-f"]
   - repo: https://github.com/ambv/black
-    rev: 21.11b1
+    rev: 22.1.0
     hooks:
       - id: black
-        args: ["--line-length=140"]
+        args: ["--config", "pyproject.toml"]
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
+    rev: 5.6.4
     hooks:
       - id: isort
         args:
           [
-            "--settings-path=tools/python/.isort.cfg",
+            "--settings-file",
+            "pyproject.toml",
             "--dont-follow-links",
             "--jobs=-1",
           ]
@@ -37,17 +38,17 @@ repos:
           ).?$
 
   - repo: https://github.com/csachs/pyproject-flake8
-    rev: v0.0.1a2.post1
+    rev: 0.0.1a2
     hooks:
       - id: pyproject-flake8
-        args: ["--config=tools/python/.flake8"]
+        args: ["--config", "pyproject.toml"]
         additional_dependencies: ["mccabe"]
         alias: flake8
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
+    rev: 0.930
     hooks:
       - id: mypy
-        args: ["--config-file=tools/python/.mypy.ini"]
+        args: ["--config-file", "pyproject.toml"]
         exclude: |
           (?x)^.*(
               octavia-cli/unit_tests/|

diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile
@@ -5,6 +5,6 @@ ENV APPLICATION airbyte-bootloader
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.32-alpha.tar /app
+ADD bin/${APPLICATION}-0.35.35-alpha.tar /app
 
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.32-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.35-alpha/bin/${APPLICATION}"]
diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java
@@ -80,7 +80,7 @@ void testBootloaderAppBlankDb() throws Exception {
         mockedConfigs.getConfigDatabaseUrl())
             .getAndInitialize();
     val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, this.getClass().getName());
-    assertEquals("0.35.28.001", configsMigrator.getLatestMigration().getVersion().getVersion());
+    assertEquals("0.35.32.001", configsMigrator.getLatestMigration().getVersion().getVersion());
 
     val jobsPersistence = new DefaultJobPersistence(jobDatabase);
     assertEquals(version, jobsPersistence.getVersion().get());

diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -13,7 +13,7 @@
 - name: BigQuery
   destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
   dockerRepository: airbyte/destination-bigquery
-  dockerImageTag: 0.6.8
+  dockerImageTag: 0.6.9
   documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
   icon: bigquery.svg
 - name: BigQuery (denormalized typed struct)
@@ -60,7 +60,7 @@
 - name: Google Cloud Storage (GCS)
   destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a
   dockerRepository: airbyte/destination-gcs
-  dockerImageTag: 0.1.22
+  dockerImageTag: 0.1.23
   documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs
   icon: googlecloudstorage.svg
 - name: Google Firestore
@@ -173,7 +173,7 @@
 - name: S3
   destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
   dockerRepository: airbyte/destination-s3
-  dockerImageTag: 0.2.7
+  dockerImageTag: 0.2.8
   documentationUrl: https://docs.airbyte.io/integrations/destinations/s3
   icon: s3.svg
 - name: SFTP-JSON
@@ -185,7 +185,7 @@
 - name: Snowflake
   destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
   dockerRepository: airbyte/destination-snowflake
-  dockerImageTag: 0.4.14
+  dockerImageTag: 0.4.15
   documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
   icon: snowflake.svg
 - name: MariaDB ColumnStore

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -188,7 +188,7 @@
     supportsDBT: false
     supported_destination_sync_modes:
     - "append"
-- dockerImage: "airbyte/destination-bigquery:0.6.8"
+- dockerImage: "airbyte/destination-bigquery:0.6.9"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery"
     connectionSpecification:
@@ -1162,7 +1162,7 @@
     - "overwrite"
     - "append"
     supportsNamespaces: true
-- dockerImage: "airbyte/destination-gcs:0.1.22"
+- dockerImage: "airbyte/destination-gcs:0.1.23"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/gcs"
     connectionSpecification:
@@ -3454,7 +3454,7 @@
     supported_destination_sync_modes:
     - "append"
     - "overwrite"
-- dockerImage: "airbyte/destination-s3:0.2.7"
+- dockerImage: "airbyte/destination-s3:0.2.8"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3"
     connectionSpecification:
@@ -3825,7 +3825,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-snowflake:0.4.14"
+- dockerImage: "airbyte/destination-snowflake:0.4.15"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake"
     connectionSpecification:

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -22,7 +22,7 @@
 - name: Amazon Seller Partner
   sourceDefinitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460
   dockerRepository: airbyte/source-amazon-seller-partner
-  dockerImageTag: 0.2.14
+  dockerImageTag: 0.2.15
   sourceType: api
   documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-seller-partner
   icon: amazonsellerpartner.svg
@@ -140,7 +140,7 @@
 - name: Cockroachdb
   sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
   dockerRepository: airbyte/source-cockroachdb
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.1.9
   documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb
   icon: cockroachdb.svg
   sourceType: database
@@ -203,7 +203,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.34
+  dockerImageTag: 0.2.35
   documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api
@@ -328,14 +328,14 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.41
+  dockerImageTag: 0.1.42
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api
 - name: IBM Db2
   sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2
   dockerRepository: airbyte/source-db2
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.1.9
   documentationUrl: https://docs.airbyte.io/integrations/sources/db2
   icon: db2.svg
   sourceType: database
@@ -446,7 +446,7 @@
 - name: Microsoft SQL Server (MSSQL)
   sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
   dockerRepository: airbyte/source-mssql
-  dockerImageTag: 0.3.16
+  dockerImageTag: 0.3.17
   documentationUrl: https://docs.airbyte.io/integrations/sources/mssql
   icon: mssql.svg
   sourceType: database
@@ -474,7 +474,7 @@
 - name: MongoDb
   sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e
   dockerRepository: airbyte/source-mongodb-v2
-  dockerImageTag: 0.1.11
+  dockerImageTag: 0.1.13
   documentationUrl: https://docs.airbyte.io/integrations/sources/mongodb-v2
   icon: mongodb.svg
   sourceType: database
@@ -488,7 +488,7 @@
 - name: MySQL
   sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
   dockerRepository: airbyte/source-mysql
-  dockerImageTag: 0.5.5
+  dockerImageTag: 0.5.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/mysql
   icon: mysql.svg
   sourceType: database
@@ -522,7 +522,7 @@
 - name: Oracle DB
   sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83
   dockerRepository: airbyte/source-oracle
-  dockerImageTag: 0.3.13
+  dockerImageTag: 0.3.14
   documentationUrl: https://docs.airbyte.io/integrations/sources/oracle
   icon: oracle.svg
   sourceType: database
@@ -599,7 +599,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 0.4.7
+  dockerImageTag: 0.4.8
   documentationUrl: https://docs.airbyte.io/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database
@@ -725,7 +725,7 @@
 - name: Snowflake
   sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2
   dockerRepository: airbyte/source-snowflake
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.1.9
   documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake
   icon: snowflake.svg
   sourceType: database
Name	Value	Name	Value	Name	Value
Duplicated Lines (%)	0.0	Quality Gate Status	ERROR	Lines of Code	800
Lines to Cover	74	Coverage	28.4	Bugs	0
Duplicated Blocks	0	Reliability Rating	A	Security Rating	A
Code Smells	4	Vulnerabilities	0	Blocker Issues	0
Critical Issues	6	Major Issues	6	Minor Issues	6
Rule	File	Description	Message
python:S2638 (CRITICAL)	source_github/streams.py	Method overrides should not change contracts	Make parameter stream_state keyword-or-positional.
python:S2638 (CRITICAL)	source_github/streams.py	Method overrides should not change contracts	Make parameter stream_slice keyword-or-positional.
python:S2638 (CRITICAL)	source_github/streams.py	Method overrides should not change contracts	Make parameter next_page_token keyword-or-positional.
python:black_need_format (MINOR)	source_github/source.py	Please run one of the commands: "black --config ./pyproject.toml <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
flake8:E501 (MAJOR)	source_github/streams.py	line too long (82 > 79 characters)	line too long (143 > 140 characters)
python:isort_need_format (MINOR)	unit_tests/test_stream.py	Please run one of the commands: "isort <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
python:S3776 (CRITICAL)	source_github/streams.py:91	Cognitive Complexity of functions should not be too high	Refactor this function to reduce its Cognitive Complexity from 25 to the 15 allowed.
flake8:E501 (MAJOR)	source_github/streams.py	line too long (82 > 79 characters)	line too long (150 > 140 characters)
python:S5886 (MAJOR)	source_github/streams.py:84	Function return types should be consistent with their type hint	Return a value of type "Union[int, float]" instead of "NoneType" or update function "backoff_time" type hint.
python:S2638 (CRITICAL)	source_github/streams.py	Method overrides should not change contracts	Add missing parameters organization project_id.
python:isort_need_format (MINOR)	unit_tests/test_source.py	Please run one of the commands: "isort <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
python:isort_need_format (MINOR)	unit_tests/unit_test.py	Please run one of the commands: "isort <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
python:isort_need_format (MINOR)	main.py	Please run one of the commands: "isort <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
python:mypy_unknown (MINOR)	main.py	Unknown error	Duplicate module named "main" (also at "./fixtures/main.py")
flake8:E501 (MAJOR)	fixtures/github.py	line too long (82 > 79 characters)	line too long (151 > 140 characters)
python:S5797 (CRITICAL)	fixtures/github.py:79	Constants should not be used as conditions	Replace this expression; used as a condition it will always be constant.
python:S112 (MAJOR)	source_github/source.py:71	"Exception" and "BaseException" should not be raised	Replace this generic exception class with a more specific one.
flake8:C901 (MAJOR)	source_github/streams.py	flake8:C901	'GithubStream.read_records' is too complex (14)
File	Coverage	File	Coverage
fixtures/github.py	0.0	fixtures/main.py	0.0
source_github/init.py	100.0	source_github/source.py	48.9
source_github/streams.py	53.0