Merge branch 'apache:master' into jdbc_druid_catalog

.asf.yaml

@@ -80,6 +80,31 @@ github:
         dismiss_stale_reviews: true
         required_approving_review_count: 1
 
+    branch-2.0:
+      required_status_checks:
+      # if strict is true, means "Require branches to be up to date before merging".
+        strict: false
+        contexts:
+          - License Check
+          - Clang Formatter
+          - CheckStyle
+          - P0 Regression (Doris Regression)
+          - P1 Regression (Doris Regression)
+          - External Regression (Doris External Regression)
+          - FE UT (Doris FE UT)
+          - BE UT (Doris BE UT)
+          - Build Broker
+          - Build Documents
+          - ShellCheck
+          - clickbench-new (clickbench)
+          - Build Third Party Libraries (Linux)
+          - Build Third Party Libraries (macOS)
+          - COMPILE (DORIS_COMPILE)
+
+        required_pull_request_reviews:
+          dismiss_stale_reviews: true
+          required_approving_review_count: 1
+
   collaborators:
     - LemonLiTree
     - Yukang-Lian

.clang-tidy

@@ -1,22 +1,33 @@
 ---
 Checks: |
+  -*,
   clang-diagnostic-*,
   clang-analyzer-*,
-  -*,
   bugprone-redundant-branch-condition,
   modernize-*,
   -modernize-use-trailing-return-type,
   -modernize-use-nodiscard,
+  -modernize-avoid-c-arrays,
   misc-redundant-expression,
   misc-unused-*,
   -misc-unused-parameters,
   readability-*,
   -readability-identifier-length,
   -readability-implicit-bool-conversion,
-  -readability-function-cognitive-complexity,
+  -readability-magic-numbers,
+  -readability-else-after-return,
+  -readability-inconsistent-declaration-parameter-name,
+  -readability-isolate-declaration,
+  -readability-named-parameter,
   portability-simd-intrinsics,
   performance-type-promotion-in-math-fn,
   performance-faster-string-find,
   performance-inefficient-algorithm,
   performance-move-const-arg
 WarningsAsErrors: '*'
+CheckOptions:
+  - key:             readability-function-size.LineThreshold
+    value:           '80'
+  - key:             readability-function-cognitive-complexity.Threshold
+    value:           '50'
+

.github/workflows/auto-pr-reply.yml

@@ -0,0 +1,66 @@
+name: Auto Reply to PR
+
+on:
+  pull_request:
+    types: [opened]
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Comment on PR
+      uses: peter-evans/create-or-update-comment@v1
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        body: |
+            Thank you for your contribution to Apache Doris. Here are some **TODOs** after submitting your PR:
+
+            1. Trigger CI Pipeline
+            
+                If your PR is ready, please **reply with `run buildall`**.
+                This command will automatically trigger all CI Pipelines.
+            
+                Currently on the `master branch`, the following admission tests are **`REQUIRED`**:
+            
+                - `License Check`: Check License Header
+                - `Clang Formatter`: Check BE code format
+                - `CheckStyle`: Check FE code format
+                - `P0 Regression (Doris Regression)`: P0 regression test
+                - `P0 Regression PipelineX (Doris Regression)`: P0 (with pipelinex) regression test
+                - `P1 Regression (Doris Regression)`: P1 regression test
+                - `External Regression (Doris External Regression)`: External table regression test
+                - `FE UT (Doris FE UT)`: FE unit test
+                - `BE UT (Doris BE UT)`: BE unit test
+                - `Build Broker`: Broker build
+                - `Build Documents`: Document build
+                - `ShellCheck`: Check Shell script format
+                - `clickbench-new (clickbench)`: Clickbench performance test
+                - `Build Third Party Libraries (Linux)`: Third-party library build (Linux)
+                - `Build Third Party Libraries (macOS)`: Third-party library build (macOS)
+                - `COMPILE (DORIS_COMPILE)`: Full compilation
+                - `Need_2_Approval`: Require approval from at least two reviewers
+                
+                The code can only be merged after all the above pipelines have passed.
+                Besides, there are some other pipelines, but they are **`NOT`** required, meaning their pass or fail does not affect PR merging.
+            
+                You can also reply with the following keywords to trigger specific pipelines individually:
+            
+                - `run compile`: COMPILE (DORIS_COMPILE)
+                - `run p0`: P0 Regression
+                - `run p1`: P1 Regression
+                - `run feut`: FE UT
+                - `run beut`: BE UT
+                - `run external`: External Regression
+                - `run clickbench`: clickbench-new
+                - `run pipelinex_p0`: P0 Regression PipelineX
+                - `run arm`: P0 Regression (ARM pipeline)
+                - `run tpch`: tpch-sf100
+            
+            2. Wait for Review
+            
+                Before merging the code, it requires **`approval from at least two reviewers`**, and one of them must be a Committer of Apache Doris.
+            
+            3. Merge the PR
+            
+                After all pipelines and reviews are passed, a Committer will manually merge the code.          
+

.github/workflows/auto_trigger_teamcity.yml

@@ -57,40 +57,32 @@ jobs:
           echo "latest_commit_id : ${{ env.LATEST_COMMIT }}"
           
           set -x
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "buildall" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression ${trigger_pipelines}"
+          reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*"
+          comment_trigger_type="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)"
+          comment_repeat_times="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)"
+          if [[ "${comment_trigger_type}" == "buildall" ]]; then 
+            trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression"
+          elif [[ "${comment_trigger_type}" == "compile" ]]; then 
+            trigger_pipelines="Doris_DorisCompile_Compile"
+          elif [[ "${comment_trigger_type}" == "p0" ]]; then 
+            trigger_pipelines="Doris_DorisRegression_P0Regression"
+          elif [[ "${comment_trigger_type}" == "p1" ]]; then 
+            trigger_pipelines="Doris_DorisRegression_P1Regression"
+          elif [[ "${comment_trigger_type}" == "feut" ]]; then 
+            trigger_pipelines="Doris_Doris_FeUt"
+          elif [[ "${comment_trigger_type}" == "beut" ]]; then 
+            trigger_pipelines="Doris_DorisBeUt_BeUt"
+          elif [[ "${comment_trigger_type}" == "external" ]]; then 
+            trigger_pipelines="Doris_External_Regression"
+          elif [[ "${comment_trigger_type}" == "clickbench" ]]; then 
+            trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew"
+          elif [[ "${comment_trigger_type}" == "pipelinex_p0" ]]; then 
+            trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX"
+          elif [[ "${comment_trigger_type}" == "arm" ]]; then 
+            trigger_pipelines="Doris_ArmPipeline_P0Regression"
+          elif [[ "${comment_trigger_type}" == "tpch" ]]; then
+            trigger_pipelines="Tpch_TpchSf100"
           fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ " p0" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_P0Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ " pipelinex_p0" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "p1" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_P1Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "feut" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Doris_FeUt  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "beut" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisBeUt_BeUt  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "compile" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisCompile_Compile  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "clickbench" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "arm" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_ArmPipeline_P0Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "external" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_External_Regression  ${trigger_pipelines}"
-          fi
-          if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ "just_for_test" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
-            trigger_pipelines="Doris_DorisRegression_ExternalRegression  ${trigger_pipelines}"
-          fi
-          
           if [ -z "${trigger_pipelines}" ];then
             echo "Just a general comment that doesn't match any pipeline rules"
           fi
@@ -133,7 +125,12 @@ jobs:
             if [ "_""${same_build_sign}" == "_false" ];then
               sleep 10s
               echo "there is no running build or queue build, so trigger a new !"
-              execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}"
+              echo "comment_repeat_times: ${comment_repeat_times}"
+              if [[ -n "${comment_repeat_times}" ]]; then
+                execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}\&name=env.repeat_times\&value=${comment_repeat_times}"
+              else
+                execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}"
+              fi
               echo "${execute_command}"
               eval "${execute_command}"
               echo "-----------------------------------------------------------------"

.github/workflows/clang-format.yml

@@ -31,14 +31,21 @@ jobs:
         uses: actions/checkout@v3
         with:
           persist-credentials: false
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
 
       - name: Paths filter
         uses: ./.github/actions/paths-filter
@@ -49,6 +56,15 @@ jobs:
               - 'be/src/**'
               - 'be/test/**'
 
+      - name: Checkout clang-format-lint-action
+        run: |
+          rm -rf ./.github/actions/clang-format-lint-action
+          git clone https://github.com/DoozyX/clang-format-lint-action .github/actions/clang-format-lint-action
+
+          pushd .github/actions/clang-format-lint-action &>/dev/null
+          git checkout 6adbe14579e5b8e19eb3e31e5ff2479f3bd302c7
+          popd &>/dev/null
+
       - name: "Format it!"
         if: ${{ steps.filter.outputs.be_changes == 'true' }}
         uses: ./.github/actions/clang-format-lint-action

.github/workflows/code-checks.yml

@@ -27,21 +27,22 @@ jobs:
       - name: Checkout ${{ github.ref }} ( ${{ github.sha }} )
         if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
-      - name: Patch
+      - name: Checkout action-sh-checker
         run: |
-          pushd .github/actions/action-sh-checker >/dev/null
+          rm -rf ./.github/actions/action-sh-checker
+          git clone https://github.com/luizm/action-sh-checker .github/actions/action-sh-checker
+
+          pushd .github/actions/action-sh-checker &>/dev/null
+          git checkout 76ab0b22e1f194e4a582edc7969df6485c4e9246
           sed -i 's/\[ "$GITHUB_EVENT_NAME" == "pull_request" \]/\[\[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "pull_request_target" \]\]/' entrypoint.sh
-          popd >/dev/null
+          popd &>/dev/null
 
       - name: Run ShellCheck
         uses: ./.github/actions/action-sh-checker
@@ -63,7 +64,15 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
 
       - name: Paths Filter
         uses: ./.github/actions/paths-filter
@@ -117,14 +126,22 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Download
         uses: actions/download-artifact@v3
         with:
           name: compile_commands
           path: ./be/build_Release
 
+      - name: Checkout clang-tidy review
+        run: |
+          rm -rf ./.github/actions/clang-tidy-review
+          git clone https://github.com/ZedThree/clang-tidy-review .github/actions/clang-tidy-review
+
+          pushd .github/actions/clang-tidy-review &>/dev/null
+          git checkout 2c55ef8cfc9acb3715d433e58aea086dcec9b206
+          popd &>/dev/null
+
       - name: Run clang-tidy review
         uses: ./.github/actions/clang-tidy-review
         id: review

.github/workflows/license-eyes.yml

@@ -30,15 +30,12 @@ jobs:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Check License
         uses: apache/skywalking-eyes@v0.2.0

.github/workflows/pr-approve-status.yml

@@ -33,7 +33,7 @@ jobs:
               echo "PR number is not set"
               exit 1
           fi
-          response=$(curl -s  -H "Authorization: token ${{ secrets.GITHUB_TOKEN }} "  "https://api.github.com/repos/apache/doris/pulls/${pr_num}/reviews")
+          response=$(curl -s  -H "Authorization: token ${{ secrets.GITHUB_TOKEN }} "  "https://api.github.com/repos/apache/doris/pulls/${pr_num}/reviews?per_page=100")
           # shellcheck disable=SC2207
           reviewers=($(echo $response | jq -r '.[] | .user.login'))
           # shellcheck disable=SC2207

.gitmodules

@@ -4,9 +4,6 @@
 [submodule ".github/actions/get-workflow-origin"]
 	path = .github/actions/get-workflow-origin
 	url = https://github.com/potiuk/get-workflow-origin.git
-[submodule ".github/actions/clang-format-lint-action"]
-	path = .github/actions/clang-format-lint-action
-	url = https://github.com/DoozyX/clang-format-lint-action.git
 [submodule ".github/actions/setup-maven"]
 	path = .github/actions/setup-maven
 	url = https://github.com/stCarolas/setup-maven.git
@@ -19,12 +16,6 @@
 [submodule ".github/actions/ccache-action"]
 	path = .github/actions/ccache-action
 	url = https://github.com/hendrikmuhs/ccache-action
-[submodule ".github/actions/action-sh-checker"]
-	path = .github/actions/action-sh-checker
-	url = https://github.com/luizm/action-sh-checker
-[submodule ".github/actions/clang-tidy-review"]
-	path = .github/actions/clang-tidy-review
-	url = https://github.com/ZedThree/clang-tidy-review.git
 [submodule "be/src/apache-orc"]
 	path = be/src/apache-orc
 	url = https://github.com/apache/doris-thirdparty.git