Merge pull request #4630 from everpcpc/build-production

ci: production build with docker
databendlabs · Mar 30, 2022 · 7bccc47 · 7bccc47
2 parents bb58268 + c85f09d
commit 7bccc47
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 64 deletions.
diff --git a/.github/actions/build_release/action.yml b/.github/actions/build_release/action.yml
@@ -4,35 +4,45 @@ inputs:
   github_token:
     description: "Github Token"
     required: true
+  target:
+    description: ""
+    required: true
 runs:
   using: "composite"
   steps:
     - name: Maximize build space
       uses: ./.github/actions/cleanup
 
     - uses: arduino/setup-protoc@v1
+      if: "contains(inputs.target, 'apple')"
       with:
         version: '3.x'
         repo-token: ${{ inputs.github_token }}
 
-    - name: Add rust target
-      shell: bash
-      run: rustup target add ${{ matrix.config.target }}
+    - shell: bash
+      if: "contains(inputs.target, 'apple')"
+      run: rustup target add ${{ inputs.target }}
+
+    - name: Setup Build Tool
+      if: "contains(inputs.target, 'linux')"
+      uses: ./.github/actions/setup_build_tool
+      with:
+        image: datafuselabs/build-tool:multiarch
 
     # If you need to reset the cache version, increment the number after `v`
     - uses: Swatinem/rust-cache@v1
       with:
-        sharedKey: release-v1
+        sharedKey: release-${{ inputs.target }}-v2
 
-    - name: Build Databend
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        use-cross: ${{ matrix.config.cross }}
-        args: --release --target ${{ matrix.config.target }}
+    - shell: bash
+      run: cargo build --release --target ${{ inputs.target }}
 
     - name: Upload artifact
       uses: actions/upload-artifact@v2
       with:
-        name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
-        path: ./target/${{ matrix.config.target }}/release/databend-*
+        name: release-${{ github.sha }}-${{ inputs.target }}
+        path: ./target/${{ inputs.target }}/release/databend-*
+
+    # for cache
+    - shell: bash
+      run: cargo metadata --all-features --format-version 1 > ./target/metadata.json
diff --git a/.github/actions/publish_image/action.yml b/.github/actions/publish_image/action.yml
@@ -10,6 +10,12 @@ inputs:
   dockerhub_namespace:
     description: "The namespace of dockerhub"
     required: true
+  target:
+    description: ""
+    required: true
+  platform:
+    description: ""
+    required: true
 runs:
   using: "composite"
   steps:
@@ -31,7 +37,7 @@ runs:
 
     - uses: actions/download-artifact@v2
       with:
-        name: release-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        name: release-${{ github.sha }}-${{ inputs.target }}
         path: ./target/release
 
     - name: Build and publish databend image
@@ -40,7 +46,7 @@ runs:
       with:
         push: true
         tags: ${{ inputs.dockerhub_namespace }}/databend:master # assume latest tag is the latest release tag
-        platforms: ${{ matrix.config.platform }}
+        platforms: ${{ inputs.platform }}
         context: .
         file: ./docker/Dockerfile
     - name: Image digest

diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml
@@ -31,103 +31,91 @@ jobs:
     strategy:
       matrix:
         config:
-          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, cross: false }
-          - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, cross: true }
-          - { os: macos-11, target: x86_64-apple-darwin, cross: false }
-          - { os: macos-11, target: aarch64-apple-darwin, cross: false }
+          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
+          - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu }
+          - { os: macos-11, target: x86_64-apple-darwin }
+          - { os: macos-11, target: aarch64-apple-darwin }
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/build_release
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
+          target: ${{ matrix.config.target }}
 
   test_unit:
-    runs-on: ${{ matrix.config.os }}
+    runs-on: ubuntu-latest
     needs: check
-    strategy:
-      matrix:
-        config:
-          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/test_unit
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
           codecov_token: ${{ secrets.CODECOV_TOKEN }}
 
-  test_stateless_standalone:
-    runs-on: ${{ matrix.config.os }}
-    needs: build
-    strategy:
-      matrix:
-        config:
-          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
-          - { os: macos-11, target: x86_64-apple-darwin }
+  test_stateless_standalone_linux:
+    runs-on: ubuntu-latest
+    needs: build_linux
     steps:
       - uses: actions/checkout@v2
-      - uses: ./.github/actions/test_stateless_standalone
+      - uses: ./.github/actions/test_stateless_standalone_linux
         with:
           profile: release
 
-  test_stateless_cluster:
-    runs-on: ${{ matrix.config.os }}
-    needs: build
-    strategy:
-      matrix:
-        config:
-          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
-          - { os: macos-11, target: x86_64-apple-darwin }
+  test_stateless_standalone_macos:
+    runs-on: macos-11
+    needs: build_macos
     steps:
       - uses: actions/checkout@v2
-      - uses: ./.github/actions/test_stateless_cluster
+      - uses: ./.github/actions/test_stateless_standalone_macos
         with:
           profile: release
 
-  test_stateful_standalone:
-    runs-on: ${{ matrix.config.os }}
-    needs: build
-    strategy:
-      matrix:
-        config:
-          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
+  test_stateless_cluster_linux:
+    runs-on: ubuntu-latest
+    needs: build_linux
     steps:
       - uses: actions/checkout@v2
-      - uses: ./.github/actions/test_stateful_standalone
+      - uses: ./.github/actions/test_stateless_cluster_linux
         with:
           profile: release
 
-  test_management_mode:
-    runs-on: ${{ matrix.config.os }}
-    needs: build
-    strategy:
-      matrix:
-        config:
-          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
+  test_stateless_cluster_macos:
+    runs-on: macos-11
+    needs: build_macos
     steps:
       - uses: actions/checkout@v2
-      - uses: ./.github/actions/test_management_mode
+      - uses: ./.github/actions/test_stateless_cluster_macos
+        with:
+          profile: release
+
+  test_stateful_standalone:
+    runs-on: ubuntu-latest
+    needs: build_linux
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/test_stateful_standalone
         with:
           profile: release
 
   publish_image:
-    runs-on: ${{ matrix.config.os }}
+    runs-on: ubuntu-latest
     # Release after all tests have been passed.
     needs: [
       test_unit,
-      test_stateless_standalone,
-      test_stateless_cluster,
+      test_stateless_standalone_linux,
+      test_stateless_cluster_linux,
       test_stateful_standalone,
-      test_management_mode
     ]
     strategy:
       matrix:
         config:
-          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, platform: linux/amd64 }
-          - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, platform: linux/arm64 }
+          - { target: x86_64-unknown-linux-gnu, platform: linux/amd64 }
+          - { target: aarch64-unknown-linux-gnu, platform: linux/arm64 }
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/publish_image
         with:
           dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
           dockerhub_namespace: ${{ secrets.DOCKERHUB_NAMESPACE }}
+          target: ${{ matrix.config.target }}
+          platform: ${{ matrix.config.platform }}