Merge branch 'main' into issue-4429

.github/actions/build_debug/action.yml

@@ -1,37 +1,48 @@
-name: 'Build Debug'
-description: 'Build with debug profile'
+name: "Build Debug"
+description: "Build with debug profile"
 inputs:
   github_token:
     description: "Github Token"
     required: true
+  target:
+    description: ""
+    required: true
 runs:
   using: "composite"
   steps:
     - name: Maximize build space
       uses: ./.github/actions/cleanup
 
     - uses: arduino/setup-protoc@v1
+      if: "contains(inputs.target, 'apple')"
       with:
-        version: '3.x'
+        version: "3.x"
         repo-token: ${{ inputs.github_token }}
 
-    - name: Add rust target
-      shell: bash
-      run: rustup target add ${{ matrix.config.target }}
+    - shell: bash
+      if: "contains(inputs.target, 'apple')"
+      run: rustup target add ${{ inputs.target }}
+
+    - name: Setup Build Tool
+      if: "contains(inputs.target, 'linux')"
+      uses: ./.github/actions/setup_build_tool
+      with:
+        image: datafuselabs/build-tool:multiarch
 
     # If you need to reset the cache version, increment the number after `v`
     - uses: Swatinem/rust-cache@v1
       with:
-        sharedKey: debug-v1
+        sharedKey: debug-${{ inputs.target }}-v2
 
-    - uses: actions-rs/cargo@v1
-      with:
-        command: build
-        use-cross: ${{ matrix.config.cross }}
-        args: --target ${{ matrix.config.target }}
+    - shell: bash
+      run: cargo build --target ${{ inputs.target }}
 
     - name: Upload artifact
       uses: actions/upload-artifact@v2
       with:
-        name: debug-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
-        path: ./target/${{ matrix.config.target }}/debug/databend-*
+        name: debug-${{ github.sha }}-${{ inputs.target }}
+        path: ./target/${{ inputs.target }}/debug/databend-*
+
+    # for cache
+    - shell: bash
+      run: cargo metadata --all-features --format-version 1 > ./target/metadata.json

.github/actions/build_release/action.yml

@@ -4,35 +4,45 @@ inputs:
   github_token:
     description: "Github Token"
     required: true
+  target:
+    description: ""
+    required: true
 runs:
   using: "composite"
   steps:
     - name: Maximize build space
       uses: ./.github/actions/cleanup
 
     - uses: arduino/setup-protoc@v1
+      if: "contains(inputs.target, 'apple')"
       with:
         version: '3.x'
         repo-token: ${{ inputs.github_token }}
 
-    - name: Add rust target
-      shell: bash
-      run: rustup target add ${{ matrix.config.target }}
+    - shell: bash
+      if: "contains(inputs.target, 'apple')"
+      run: rustup target add ${{ inputs.target }}
+
+    - name: Setup Build Tool
+      if: "contains(inputs.target, 'linux')"
+      uses: ./.github/actions/setup_build_tool
+      with:
+        image: datafuselabs/build-tool:multiarch
 
     # If you need to reset the cache version, increment the number after `v`
     - uses: Swatinem/rust-cache@v1
       with:
-        sharedKey: release-v1
+        sharedKey: release-${{ inputs.target }}-v2
 
-    - name: Build Databend
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        use-cross: ${{ matrix.config.cross }}
-        args: --release --target ${{ matrix.config.target }}
+    - shell: bash
+      run: cargo build --release --target ${{ inputs.target }}
 
     - name: Upload artifact
       uses: actions/upload-artifact@v2
       with:
-        name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
-        path: ./target/${{ matrix.config.target }}/release/databend-*
+        name: release-${{ github.sha }}-${{ inputs.target }}
+        path: ./target/${{ inputs.target }}/release/databend-*
+
+    # for cache
+    - shell: bash
+      run: cargo metadata --all-features --format-version 1 > ./target/metadata.json

.github/actions/check/action.yml

@@ -1,36 +1,19 @@
-name: 'Check'
-description: 'Check will do all essential checks'
+name: "Check"
+description: "Check will do all essential checks"
 inputs:
   github_token:
     description: "Github Token"
     required: true
 runs:
   using: "composite"
   steps:
-    - name: Maximize build space
-      uses: ./.github/actions/cleanup
-
-    - uses: arduino/setup-protoc@v1
-      with:
-        version: '3.x'
-        repo-token: ${{ inputs.github_token }}
+    - name: Setup Build Tool
+      uses: ./.github/actions/setup_build_tool
 
     # If you need to reset the cache version, increment the number after `v`
     - uses: Swatinem/rust-cache@v1
       with:
-        sharedKey: base-v1
-
-    - name: Format
-      uses: actions-rs/cargo@v1
-      with:
-        command: fmt
-        args: --all -- --check
-
-    - name: Clippy
-      uses: actions-rs/cargo@v1
-      with:
-        command: clippy
-        args: --tests -- -D warnings
+        sharedKey: base-v2
 
     - name: Check License Header
       uses: apache/skywalking-eyes@v0.2.0
@@ -45,29 +28,28 @@ runs:
         file_or_dir: ./
         config_file: .yamllint.yml
 
-    - name: Install cargo-audit
-      uses: actions-rs/cargo@v1
-      if: "!contains(github.event.head_commit.message, 'skip audit')"
-      with:
-        command: install
-        args: cargo-audit
+    - name: Format
+      shell: bash
+      run: cargo fmt --all -- --check
+
+    - name: Clippy
+      shell: bash
+      run: cargo clippy --tests -- -D warnings
 
     - name: Audit dependencies
-      uses: actions-rs/cargo@v1
+      shell: bash
       if: "!contains(github.event.head_commit.message, 'skip audit')"
-      with:
-        command: audit
-
-    - name: Install cargo-udeps
-      uses: actions-rs/cargo@v1
-      if: "!contains(github.event.head_commit.message, 'skip udeps')"
-      with:
-        command: install
-        args: cargo-udeps --locked
+      run: cargo audit --db ./target/advisory-db
 
     - name: Check udeps
-      uses: actions-rs/cargo@v1
+      shell: bash
       if: "!contains(github.event.head_commit.message, 'skip udeps')"
-      with:
-        command: udeps
-        args: --workspace
+      run: cargo udeps --workspace
+
+    - name: Check toml format
+      shell: bash
+      run: build-tool taplo fmt --check
+
+    # for cache
+    - shell: bash
+      run: cargo metadata --all-features --format-version 1 > ./target/metadata.json

.github/actions/publish_image/action.yml

@@ -10,6 +10,12 @@ inputs:
   dockerhub_namespace:
     description: "The namespace of dockerhub"
     required: true
+  target:
+    description: ""
+    required: true
+  platform:
+    description: ""
+    required: true
 runs:
   using: "composite"
   steps:
@@ -31,7 +37,7 @@ runs:
 
     - uses: actions/download-artifact@v2
       with:
-        name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        name: release-${{ github.sha }}-${{ inputs.target }}
         path: ./target/release
 
     - name: Build and publish databend image
@@ -40,7 +46,7 @@ runs:
       with:
         push: true
         tags: ${{ inputs.dockerhub_namespace }}/databend:master # assume latest tag is the latest release tag
-        platforms: ${{ matrix.config.platform }}
+        platforms: ${{ inputs.platform }}
         context: .
         file: ./docker/Dockerfile
     - name: Image digest

.github/actions/setup_build_tool/action.yml

@@ -0,0 +1,46 @@
+name: "Setup Build Tool Binaries"
+description: "Setup Build Tool Binaries"
+inputs:
+  image:
+    description: "Build Tool Docker Image to use"
+    required: true
+    default: datafuselabs/build-tool:dev-debian-amd64
+  bypass_env_vars:
+    description: "Environment variables bypass to docker container"
+    required: false
+    default: RUSTFLAGS,RUST_LOG
+runs:
+  using: "composite"
+  steps:
+    - name: clean the existing toolchain
+      shell: bash
+      run: |
+        rm -rf /home/runner/.cargo
+        rm -rf /home/runner/.rustup
+
+    - name: setup build-tool binary
+      shell: bash
+      run: |
+        BIN_LOCAL=$HOME/.cargo/bin
+        mkdir -p $BIN_LOCAL
+        echo "$BIN_LOCAL" >> $GITHUB_PATH
+
+        cat <<EOF >$BIN_LOCAL/build-tool
+        #!/bin/bash
+        script_name=\$(basename "\$0")
+        export IMAGE=${{ inputs.image }}
+        export BYPASS_ENV_VARS=${{ inputs.bypass_env_vars }}
+        if [[ \${script_name} == "build-tool" ]]; then
+          scripts/setup/run_build_tool.sh \$@
+        else
+          scripts/setup/run_build_tool.sh \${script_name} \$@
+        fi
+        EOF
+        chmod +x $BIN_LOCAL/build-tool
+        ln -s build-tool $BIN_LOCAL/rustc
+        ln -s build-tool $BIN_LOCAL/cargo
+        ln -s build-tool $BIN_LOCAL/rustup
+
+    - name: check cargo version
+      shell: bash
+      run: rustup show

.github/actions/test_metactl/action.yml

@@ -1,24 +1,23 @@
-name: 'Test metactl'
+name: "Test metactl"
+description: ""
 inputs:
   profile:
-    description: 'The profile for this test'
+    description: "The profile for this test"
     required: true
-    default: 'debug'
+    default: "debug"
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
-
     - name: Maximize build space
       uses: ./.github/actions/cleanup
 
-    # If you need to reset the cache version, increment the number after `v`
-    - uses: Swatinem/rust-cache@v1
-      with:
-        sharedKey: unit-v1
-
     - uses: actions/download-artifact@v2
       with:
-        name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        name: ${{ inputs.profile }}-${{ github.sha }}-${{ inputs.target }}
         path: ./target/debug
 
     - name: Test metactl

.github/actions/test_stateful_standalone/action.yml

@@ -1,28 +1,23 @@
-name: 'Test Stateful Standalone'
-description: 'Running stateful tests in standalone mode'
+name: "Test Stateful Standalone"
+description: "Running stateful tests in standalone mode"
 inputs:
   profile:
-    description: 'The profile for this test'
+    description: "The profile for this test"
     required: true
-    default: 'debug'
+    default: "debug"
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
 runs:
   using: "composite"
   steps:
     - name: Maximize build space
       uses: ./.github/actions/cleanup
 
-    - name: Add rust target
-      shell: bash
-      run: rustup target add ${{ matrix.config.target }}
-
-    # If you need to reset the cache version, increment the number after `v`
-    - uses: Swatinem/rust-cache@v1
-      with:
-        sharedKey: stateful-v1
-
     - uses: actions/download-artifact@v2
       with:
-        name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        name: ${{ inputs.profile }}-${{ github.sha }}-${{ inputs.target }}
         path: ./target/debug
 
     - name: Set up file as executable

.github/actions/test_stateless_cluster_linux/action.yml

@@ -0,0 +1,34 @@
+name: "Test Stateless Cluster"
+description: "Running stateless tests in cluster mode"
+inputs:
+  profile:
+    description: "The profile for this test"
+    required: true
+    default: "debug"
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
+runs:
+  using: "composite"
+  steps:
+    - name: Maximize build space
+      uses: ./.github/actions/cleanup
+
+    - name: Setup Build Tool
+      uses: ./.github/actions/setup_build_tool
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ inputs.profile }}-${{ github.sha }}-${{ inputs.target }}
+        path: ./target/debug
+
+    - name: Set up file as executable
+      shell: bash
+      run: |
+        chmod +x ./target/debug/databend-*
+
+    - name: Run Stateless Tests with Cluster mode
+      shell: bash
+      run: |
+        build-tool bash ./scripts/ci/ci-run-stateless-tests-cluster.sh