Merge branch 'main' into st_transform

Signed-off-by: Fan Yang <yangfanlinux@gmail.com>
databendlabs · Jul 10, 2024 · 2244da3 · 2244da3
2 parents dc8993a + de689a3
commit 2244da3
Show file tree

Hide file tree

Showing 239 changed files with 4,696 additions and 4,205 deletions.
diff --git a/.cargo/audit.toml b/.cargo/audit.toml
@@ -36,5 +36,13 @@ ignore = [
     # Degradation of service in h2 servers with CONTINUATION Flood(Wait for tonic to fix)
     "RUSTSEC-2024-0332",
     # `rustls::ConnectionCommon::complete_io` could fall into an infinite loop based on network input
-    "RUSTSEC-2024-0336"
+    "RUSTSEC-2024-0336",
+    # gix-fs: Traversal outside working tree enables arbitrary code execution
+    "RUSTSEC-2024-0350",
+    # gix-index: Traversal outside working tree enables arbitrary code execution
+    "RUSTSEC-2024-0348",
+    # gix-index: Refs and paths with reserved Windows device names access the devices
+    "RUSTSEC-2024-0352",
+    # gix-ref: Refs and paths with reserved Windows device names access the devices
+    "RUSTSEC-2024-0351"
 ]
diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml
@@ -20,7 +20,7 @@ env:
 
 jobs:
   check:
-    runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -32,7 +32,7 @@ jobs:
           github_token: ${{ github.token }}
 
   license:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     outputs:
       license: ${{ steps.license.outputs.license }}
     steps:
@@ -128,7 +128,7 @@ jobs:
           artifacts: query
 
   build_musl:
-    runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
     strategy:
       fail-fast: false
       matrix:
@@ -148,7 +148,7 @@ jobs:
           artifacts: query
 
   test_unit:
-    runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -158,72 +158,78 @@ jobs:
         timeout-minutes: 60
 
   test_metactl:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_metactl
         timeout-minutes: 10
 
   test_compat_meta_query:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_meta_query
         timeout-minutes: 10
 
   test_compat_fuse:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_fuse
         timeout-minutes: 20
 
   test_compat_meta_meta:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_meta_meta
         timeout-minutes: 20
 
   test_logs:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_logs
         timeout-minutes: 20
 
   test_meta_cluster:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_meta_cluster
         timeout-minutes: 10
 
   test_stateless_standalone:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateless_standalone_linux
         timeout-minutes: 15
 
   test_stateless_cluster:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check, license]
     steps:
       - uses: actions/checkout@v4
+      - name: Decode license
+        id: license
+        run: |
+          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
       - uses: ./.github/actions/test_stateless_cluster_linux
+        env:
+          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
         timeout-minutes: 15
 
   test_stateful_standalone:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_standalone_linux
@@ -235,11 +241,17 @@ jobs:
           name: test-stateful-standalone-linux
 
   test_stateful_cluster:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check, license]
     steps:
       - uses: actions/checkout@v4
+      - name: Decode license
+        id: license
+        run: |
+          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
       - uses: ./.github/actions/test_stateful_cluster_linux
+        env:
+          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
         timeout-minutes: 15
       - name: Upload failure
         if: failure()
@@ -249,16 +261,16 @@ jobs:
 
   test_stateful_large_data:
     if: contains(github.event.pull_request.labels.*.name, 'ci-largedata')
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_large_data
         timeout-minutes: 60
 
   test_stateful_iceberg_rest:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_iceberg_rest_standalone
@@ -279,8 +291,8 @@ jobs:
   #       continue-on-error: true
 
   test_ee_standalone:
-    needs: [ build, check, license ]
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    needs: [build, check, license]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     if: needs.license.outputs.license
     steps:
       - uses: actions/checkout@v4
@@ -299,8 +311,8 @@ jobs:
           name: test-stateful-standalone-linux
 
   test_ee_standalone_background:
-    needs: [ build, check, license ]
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    needs: [build, check, license]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     if: needs.license.outputs.license
     steps:
       - uses: actions/checkout@v4
@@ -320,8 +332,8 @@ jobs:
           name: test-stateful-standalone-linux
 
   test_ee_management_mode:
-    needs: [ build, check, license ]
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    needs: [build, check, license]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     if: needs.license.outputs.license
     steps:
       - uses: actions/checkout@v4
@@ -340,38 +352,8 @@ jobs:
         with:
           name: test-ee-management-mode-linux
 
-  sqllogic_ee:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    if: needs.license.outputs.license
-    needs: [ build, check, license ]
-    strategy:
-      fail-fast: false
-      matrix:
-        format:
-          - "parquet"
-          - "native"
-    steps:
-      - uses: actions/checkout@v4
-      - name: Decode license
-        id: license
-        run: |
-          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
-      - uses: ./.github/actions/test_ee_sqllogic_standalone_linux
-        timeout-minutes: 15
-        env:
-          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
-        with:
-          dirs: ee
-          handlers: mysql,http
-          storage-format: ${{ matrix.format }}
-      - name: Upload failure
-        if: failure()
-        uses: ./.github/actions/artifact_failure
-        with:
-          name: test-ee-sqllogic-standalone-ee-${{ matrix.format }}
-
   sqllogic:
-    needs: [ build, check ]
+    needs: [build, check]
     uses: ./.github/workflows/reuse.sqllogic.yml
     secrets: inherit
     with:

diff --git a/.github/workflows/reuse.sqllogic.yml b/.github/workflows/reuse.sqllogic.yml
@@ -19,6 +19,28 @@ env:
   RUNNER_PROVIDER: ${{ inputs.runner_provider }}
 
 jobs:
+  license:
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    outputs:
+      license: ${{ steps.license.outputs.license }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get License from S3
+        if: inputs.runner_provider == 'aws'
+        run: |
+          aws s3 cp s3://databend-ci/misc/license.key license.key
+          aws s3 cp s3://databend-ci/misc/license.json license.json
+      - name: Get License from GCS
+        if: inputs.runner_provider == 'gcp'
+        run: |
+          gcloud storage cp gs://databend-ci/misc/license.key license.key
+          gcloud storage cp gs://databend-ci/misc/license.json license.json
+      - name: Output License
+        id: license
+        run: |
+          echo "license=$(cat license.key | base64 -w 0)" >> $GITHUB_OUTPUT
+          cat license.json
+
   management_mode:
     runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     steps:
@@ -128,6 +150,7 @@ jobs:
 
   cluster:
     runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: license
     strategy:
       fail-fast: false
       matrix:
@@ -145,7 +168,13 @@ jobs:
           - "http"
     steps:
       - uses: actions/checkout@v4
+      - name: Decode license
+        id: license
+        run: |
+          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
       - uses: ./.github/actions/test_sqllogic_cluster_linux
+        env:
+          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
         timeout-minutes: 15
         with:
           dirs: ${{ matrix.dirs }}
@@ -201,3 +230,32 @@ jobs:
         uses: ./.github/actions/artifact_failure
         with:
           name: test-sqllogic-standalone-no-table-meta-cache-${{ matrix.dirs }}-${{ matrix.handler }}
+
+  ee:
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: license
+    strategy:
+      fail-fast: false
+      matrix:
+        format:
+          - "parquet"
+          - "native"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Decode license
+        id: license
+        run: |
+          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
+      - uses: ./.github/actions/test_ee_sqllogic_standalone_linux
+        timeout-minutes: 15
+        env:
+          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
+        with:
+          dirs: ee
+          handlers: mysql,http
+          storage-format: ${{ matrix.format }}
+      - name: Upload failure
+        if: failure()
+        uses: ./.github/actions/artifact_failure
+        with:
+          name: test-ee-sqllogic-standalone-ee-${{ matrix.format }}