Merge branch 'master' into grubberr/10384-source-github

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.31-alpha
+current_version = 0.35.32-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.35.31-alpha
+VERSION=0.35.32-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/actions/ci-java-tests/action.yml

@@ -9,11 +9,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install Java
-      uses: actions/setup-java@v1
-      with:
-        java-version: '17'
-
     - name: "Build"
       shell: bash
       run: |

.github/actions/ci-py-tests/action.yml

@@ -27,57 +27,26 @@ runs:
     - name: Build Coverage Reports
       id: build-coverage-reports
       shell: bash
-      working-directory: ${{ inputs.module-folder }}
       run: |
-        virtualenv .venv
-        source .venv/bin/activate
-        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
-        pip install coverage[toml]~=6.2
-        mkdir -p .venv/source-acceptance-test
-        mkdir -p reports
-        SAT_DIR=$(git rev-parse --show-toplevel)/airbyte-integrations/bases/source-acceptance-test
-        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
-        git ls-tree -r HEAD --name-only $SAT_DIR | while read src; do cp -f $src .venv/source-acceptance-test; done
-        pip install build
-        python -m build .venv/source-acceptance-test
-        pip install .venv/source-acceptance-test/dist/source_acceptance_test-*.whl
-        [ -f requirements.txt ] && pip install --quiet -r requirements.txt
-        pip install .[tests]
-        coverage run --rcfile=${PYPROJECT_CONFIG} -m pytest ./unit_tests || true
-        coverage xml --rcfile=${PYPROJECT_CONFIG} -o reports/coverage.xml || true
+        GRADLE_JOB=$(source ./tools/lib/lib.sh; full_path_to_gradle_path ${{ inputs.module-folder }} "unitTest")
+        REPORT_FOLDER="${{ inputs.module-folder }}/coverage/"
+        ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB}
 
-        rm -rf .venv
-        echo "::set-output name=coverage-paths::reports/coverage.xml"
+        echo "::set-output name=coverage-paths::coverage/coverage.xml"
 
     - name: Upload coverage to Codecov
-      if: ${{ always() }}
       uses: codecov/codecov-action@v2
+      with:
+        file: ${{ steps.build-coverage-reports.outputs.coverage-paths }}
+        name: "UnitTests of ${{ inputs.module-name }}"
 
     - name: Build Linter Reports
       id: build-linter-reports
       shell: bash
-      working-directory: ${{ inputs.module-folder }}
       run: |
-        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
-        REPORT_FOLDER=reports
-        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
-
-        # run mypy
-        pip install lxml~=4.7 mypy~=0.910 .
-        mypy . --config-file=${PYPROJECT_CONFIG} | tee reports/mypy.log || true
-
-        # run black
-        pip install black~=21.12b0
-        XDG_CACHE_HOME=/dev/null black --config ${PYPROJECT_CONFIG} --diff . | tee reports/black.diff
-
-        # run isort
-        pip install isort~=5.10.1
-        cp ${PYPROJECT_CONFIG} ./pyproject.toml
-        isort --diff . | tee reports/isort.diff
-
-        # run flake8
-        pip install mccabe~=0.6.1 pyproject-flake8~=0.0.1a2
-        pflake8 --exit-zero . | grep ^. | tee reports/flake.txt
+        GRADLE_JOB=$(source ./tools/lib/lib.sh; full_path_to_gradle_path ${{ inputs.module-folder }} "airbytePythonReport")
+        REPORT_FOLDER="${{ inputs.module-folder }}/reports/"
+        ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB}
 
         echo "::set-output name=mypy-logs::reports/mypy.log"
         echo "::set-output name=black-diff::reports/black.diff"

.github/actions/ci-tests-runner/action.yml

@@ -32,13 +32,18 @@ runs:
       with:
         python-version: 3.7
 
+    - name: Install Java
+      uses: actions/setup-java@v1
+      with:
+        java-version: '17'
+
     - name: Tests of CI
       shell: bash
       run: |
         # all CI python packages have the prefix "ci_"
         pip install --quiet tox==3.24.4
-        tox -r -c ./tools/tox_ci.ini
         pip install --quiet -e ./tools/ci_*
+        tox -r -c ./tools/tox_ci.ini
         echo "::echo::off"
 
     - name: Auth with gcloud CLI
@@ -109,6 +114,7 @@ runs:
           [ -f ${{ steps.ci-py-tests.outputs.coverage-paths }} ] && OPTIONS+=("-Dsonar.python.coverage.reportPaths=${{ steps.ci-py-tests.outputs.coverage-paths }}")
           [ -f ${{ steps.ci-py-tests.outputs.flake8-logs }} ] && OPTIONS+=("-Dsonar.python.flake8.reportPaths=${{ steps.ci-py-tests.outputs.flake8-logs }}")
         fi
+        cat ${REPORT_FOLDER}/*
         if [ ${{ inputs.module-lang }} == 'java' ]; then
              [ -d "./src/main/java" ] && OPTIONS+=("-Dsonar.sources=./src/main/java")
              [ -d "./src/test/java" ] && OPTIONS+=("-Dsonar.tests=./src/test/java")
@@ -133,6 +139,7 @@ runs:
         MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))")
         echo "::set-output name=module_dir::${MODULE_DIR}"
 
+
     - name: SonarQube Scan
 
       uses: sonarsource/sonarqube-scan-action@master
@@ -148,7 +155,7 @@ runs:
             -Dsonar.language=${{ inputs.module-lang }}
             -Dsonar.sourceEncoding=UTF-8
             -Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }}
-            -Dsonar.exclusions=reports/**,*.toml
+            -Dsonar.exclusions=reports/**,*.toml,*_tests/**,setup.py,main.py
             -Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }}
             ${{ steps.sq-options.outputs.options }}
 

.github/workflows/sonar-scan.yml

@@ -72,7 +72,7 @@ jobs:
         sonar-token: ${{ secrets.SONAR_TOKEN }}
         sonar-gcp-access-key: ${{ secrets.GCP_SONAR_SA_KEY }}
         pull-request-id: "${{ github.repository }}/${{ github.event.pull_request.number }}"
-        remove-sonar-project: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }}
+        remove-sonar-project: ${{  github.event.action == 'closed' }}
 
 
 

airbyte-api/src/main/openapi/config.yaml

@@ -494,6 +494,29 @@ paths:
                 $ref: "#/components/schemas/SourceReadList"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/sources/clone:
+    post:
+      tags:
+        - source
+      summary: Clone source
+      operationId: cloneSource
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SourceIdRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SourceRead"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "422":
+          $ref: "#/components/responses/InvalidInputResponse"
   /v1/sources/delete:
     post:
       tags:
@@ -892,6 +915,29 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/destinations/clone:
+    post:
+      tags:
+        - destination
+      summary: Clone destination
+      operationId: cloneDestination
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/DestinationIdRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DestinationRead"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "422":
+          $ref: "#/components/responses/InvalidInputResponse"
   /v1/connections/create:
     post:
       tags:

airbyte-bootloader/Dockerfile

@@ -5,6 +5,6 @@ ENV APPLICATION airbyte-bootloader
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.31-alpha.tar /app
+ADD bin/${APPLICATION}-0.35.32-alpha.tar /app
 
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.31-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.32-alpha/bin/${APPLICATION}"]

airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java

@@ -114,7 +114,7 @@ public void load() throws Exception {
 
       final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configDatabase);
       final ConfigRepository configRepository =
-          new ConfigRepository(configPersistence, null, Optional.empty(), Optional.empty());
+          new ConfigRepository(configPersistence, null, Optional.empty(), Optional.empty(), configDatabase);
 
       createWorkspaceIfNoneExists(configRepository);
       LOGGER.info("Default workspace created..");

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json

@@ -0,0 +1,8 @@
+{
+  "sourceDefinitionId": "2817b3f0-04e4-4c7a-9f32-7a5e8a83db95",
+  "name": "PagerDuty",
+  "dockerRepository": "farosai/airbyte-pagerduty-source",
+  "dockerImageTag": "0.1.23",
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/pagerduty",
+  "icon": "pagerduty.svg"
+}

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json

@@ -0,0 +1,8 @@
+{
+  "sourceDefinitionId": "6fe89830-d04d-401b-aad6-6552ffa5c4af",
+  "name": "Harness",
+  "dockerRepository": "farosai/airbyte-harness-source",
+  "dockerImageTag": "0.1.23",
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/harness",
+  "icon": "harness.svg"
+}

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json

@@ -0,0 +1,8 @@
+{
+  "sourceDefinitionId": "7e20ce3e-d820-4327-ad7a-88f3927fd97a",
+  "name": "VictorOps",
+  "dockerRepository": "farosai/airbyte-victorops-source",
+  "dockerImageTag": "0.1.23",
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/victorops",
+  "icon": "victorops.svg"
+}

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json

@@ -0,0 +1,8 @@
+{
+  "sourceDefinitionId": "c47d6804-8b98-449f-970a-5ddb5cb5d7aa",
+  "name": "Customer.io",
+  "dockerRepository": "farosai/airbyte-customer-io-source",
+  "dockerImageTag": "0.1.23",
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/customer-io",
+  "icon": "customer-io.svg"
+}