Merge branch 'master' into master-fix6

.asf.yaml

@@ -52,13 +52,12 @@ github:
           - Clang Formatter
           - CheckStyle
           - P0 Regression (Doris Regression)
-          - P0 Regression PipelineX (Doris Regression)
           - P1 Regression (Doris Regression)
           - External Regression (Doris External Regression)
+          - cloud_p1 (Doris Cloud Regression)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Build Broker
-          - Build Documents
           - ShellCheck          
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
@@ -94,7 +93,6 @@ github:
           - BE UT (Doris BE UT)
           - Build Broker
           - ShellCheck
-          - Build Documents
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)
@@ -112,8 +110,8 @@ github:
     - shuke987
     - wm1581066
     - KassieZ
-    - yujun777
     - gavinchou
+    - yujun777
 
 notifications:
     pullrequests_status:  [email protected]

.clang-tidy

@@ -23,6 +23,8 @@ Checks: |
   -readability-inconsistent-declaration-parameter-name,
   -readability-isolate-declaration,
   -readability-named-parameter,
+  -readability-avoid-const-params-in-decls,
+  -readability-convert-member-functions-to-static,
   portability-simd-intrinsics,
   performance-type-promotion-in-math-fn,
   performance-faster-string-find,

.github/workflows/build-extension.yml

@@ -61,7 +61,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           distribution: adopt
-          java-version: '8'
+          java-version: '17'
 
       - name: Setup thrift
         run: |
@@ -84,39 +84,39 @@ jobs:
       - name: Build broker
         run: |
           cd fs_brokers/apache_hdfs_broker/ && /bin/bash build.sh
-  build-docs:
-    name: Build Documents
-    needs: changes
-    if: ${{ needs.changes.outputs.docs_changes == 'true' }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout ${{ github.ref }}
-        uses: actions/checkout@v3
+  # build-docs:
+  #   name: Build Documents
+  #   needs: changes
+  #   if: ${{ needs.changes.outputs.docs_changes == 'true' }}
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout ${{ github.ref }}
+  #       uses: actions/checkout@v3
 
-      - name: Build docs
-        run: |
-          cd docs && /bin/bash build_help_zip.sh
-      - name: Build
-        run: |
-          git clone https://github.com/apache/doris-website.git website
-          cd website
-          echo "[\"current\"]" > versions.json 
-          mkdir -p docs
-          cp -R ../docs/en/docs/* docs/
-          cp -R ../docs/sidebars.json sidebars.json
-          mkdir -p i18n/zh-CN/docusaurus-plugin-content-docs/current
-          cp -R ../docs/zh-CN/docs/* i18n/zh-CN/docusaurus-plugin-content-docs/current/
-          cp -R ../docs/dev.json i18n/zh-CN/docusaurus-plugin-content-docs/current.json
+  #     - name: Build docs
+  #       run: |
+  #         cd docs && /bin/bash build_help_zip.sh
+  #     - name: Build
+  #       run: |
+  #         git clone https://github.com/apache/doris-website.git website
+  #         cd website
+  #         echo "[\"current\"]" > versions.json 
+  #         mkdir -p docs
+  #         cp -R ../docs/en/docs/* docs/
+  #         cp -R ../docs/sidebars.json sidebars.json
+  #         mkdir -p i18n/zh-CN/docusaurus-plugin-content-docs/current
+  #         cp -R ../docs/zh-CN/docs/* i18n/zh-CN/docusaurus-plugin-content-docs/current/
+  #         cp -R ../docs/dev.json i18n/zh-CN/docusaurus-plugin-content-docs/current.json
 
-          mkdir -p community
-          cp -R ../docs/en/community/* community/
-          mkdir -p i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
-          cp -R ../docs/zh-CN/community/* i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
-          cp -R ../docs/sidebarsCommunity.json .
+  #         mkdir -p community
+  #         cp -R ../docs/en/community/* community/
+  #         mkdir -p i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
+  #         cp -R ../docs/zh-CN/community/* i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
+  #         cp -R ../docs/sidebarsCommunity.json .
 
-          cp -R ../docs/images static/
-          npm install -g yarn
-          yarn cache clean
-          yarn && yarn build
-          cd ../
-          rm -rf website
+  #         cp -R ../docs/images static/
+  #         npm install -g yarn
+  #         yarn cache clean
+  #         yarn && yarn build
+  #         cd ../
+  #         rm -rf website

.github/workflows/clang-format.yml

@@ -17,7 +17,7 @@
 #
 
 ---
-name: BE Code Formatter
+name: Code Formatter
 
 on: [push, pull_request_target]
 
@@ -52,9 +52,11 @@ jobs:
         id: filter
         with:
           filters: |
-            be_changes:
+            changes:
               - 'be/src/**'
               - 'be/test/**'
+              - 'cloud/src/**'
+              - 'cloud/test/**'
 
       - name: Checkout clang-format-lint-action
         run: |
@@ -66,14 +68,14 @@ jobs:
           popd &>/dev/null
 
       - name: "Format it!"
-        if: ${{ steps.filter.outputs.be_changes == 'true' }}
+        if: ${{ steps.filter.outputs.changes == 'true' }}
         uses: ./.github/actions/clang-format-lint-action
-        id: be_clang_format
+        id: clang_format
         with:
-          source: "be/src be/test"
+          source: "be/src be/test cloud/src cloud/test"
           clangFormatVersion: 16
           inplace: False
 
       - name: Ignore it!
-        if: ${{ steps.filter.outputs.be_changes == 'false' }}
+        if: ${{ steps.filter.outputs.changes == 'false' }}
         run: echo 'No need to check.'

.github/workflows/code-checks.yml

@@ -105,7 +105,7 @@ jobs:
             popd
 
             export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
-            DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be --cloud
+            DISABLE_BE_JAVA_EXTENSIONS=ON DO_NOT_CHECK_JAVA_ENV=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be --cloud
           fi
 
           echo "should_check=${{ steps.filter.outputs.cpp_changes }}" >>${GITHUB_OUTPUT}

.github/workflows/comment-to-trigger-teamcity.yml

@@ -45,8 +45,8 @@ jobs:
             "${COMMENT_BODY}" == *'run p0'* ||
             "${COMMENT_BODY}" == *'run p1'* ||
             "${COMMENT_BODY}" == *'run external'* ||
-            "${COMMENT_BODY}" == *'run pipelinex_p0'* ||
             "${COMMENT_BODY}" == *'run cloud_p0'* ||
+            "${COMMENT_BODY}" == *'run cloud_p1'* ||
             "${COMMENT_BODY}" == *'run arm'* ||
             "${COMMENT_BODY}" == *'run performance'* ]]; then
             echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
@@ -64,7 +64,7 @@ jobs:
         echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
         echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT"
 
-        reg="run (buildall|compile|p0|p1|feut|beut|cloudut|external|clickbench|pipelinex_p0|cloud_p0|arm|performance)( [1-9]*[0-9]+)*"
+        reg="run (buildall|compile|p0|p1|feut|beut|cloudut|external|clickbench|cloud_p0|cloud_p1|arm|performance)( [1-9]*[0-9]+)*"
         COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
         COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
         echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT"
@@ -99,12 +99,10 @@ jobs:
           if file_changed_regression_p0; then
             echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
             echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
-            echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT"
             echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT"
           else
             echo "changed_p0=false" | tee -a "$GITHUB_OUTPUT"
             echo "changed_external=false" | tee -a "$GITHUB_OUTPUT"
-            echo "changed_pipelinex_p0=false" | tee -a "$GITHUB_OUTPUT"
             echo "changed_arm=false" | tee -a "$GITHUB_OUTPUT"
           fi
           if file_changed_regression_p1; then
@@ -122,18 +120,23 @@ jobs:
           else
             echo "changed_cloud_p0=false" | tee -a "$GITHUB_OUTPUT"
           fi
+          if file_changed_cloud_p1; then
+            echo "changed_cloud_p1=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_cloud_p1=false" | tee -a "$GITHUB_OUTPUT"
+          fi
         else
           echo "INFO: failed to _get_pr_changed_files, default trigger all"
           echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_cloud_ut=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
-          echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_cloud_p0=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_cloud_p1=true" | tee -a "$GITHUB_OUTPUT"
         fi
 
     # - name: "Setup tmate session"
@@ -182,8 +185,13 @@ jobs:
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
+        if [[ "${{ steps.changes.outputs.changed_p0 }}" == "true" ]] || [[ "${{ steps.changes.outputs.changed_cloud_p0 }}" == "true" ]]; then
+          changed="true"
+        else
+          changed="false"
+        fi
         trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_p0 }}" \
+          "${changed}" \
           "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
           "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
           "compile" \
@@ -235,35 +243,57 @@ jobs:
           "external" \
           "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
 
-    - name: "Trigger or Skip pipelinex_p0"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["pipelinex_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+    - name: "Trigger or Skip cloud_p0"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloud_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
-          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger pipelinex_p0" && exit
+          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit
         fi
         set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_pipelinex_p0 }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "pipelinex_p0" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+          echo "PR target branch in (master), need run cloud_p0"
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_cloud_p0 }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloud_p0" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch not in (master), skip run cloud_p0"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloud_p0" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        fi
 
-    - name: "Trigger or Skip cloud_p0"
-      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloud_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+    - name: "Trigger or Skip cloud_p1"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloud_p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then
-          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit
+          echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p1" && exit
         fi
         set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_cloud_p0 }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "cloud_p0" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+          echo "PR target branch in (master), need run cloud_p1"
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_cloud_p1 }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloud_p1" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch not in (master), skip run cloud_p1"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloud_p1" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        fi
 
     - name: "Trigger or Skip arm"
       if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}

.github/workflows/pr-approve-status.yml

@@ -19,42 +19,25 @@ name: Need_2_Approval
 
 on:
   pull_request_review:
-    types: [submitted]
+    types: [ submitted ]
+
 jobs:
   Need_2_Approval:
     runs-on: ubuntu-latest
-    timeout-minutes: 3
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v3
-      - run: |
-          pr_num=${{ github.event.pull_request.number }}
-          echo $pr_num
-          if [ -z "$pr_num" ]; then
-              echo "PR number is not set"
-              exit 1
-          fi
-          response=$(curl -s  -H "Authorization: token ${{ secrets.GITHUB_TOKEN }} "  "https://api.github.com/repos/apache/doris/pulls/${pr_num}/reviews?per_page=100")
-          # shellcheck disable=SC2207
-          reviewers=($(echo $response | jq -r '.[] | .user.login'))
-          # shellcheck disable=SC2207
-          statuses=($(echo $response | jq -r '.[] | .state'))
-          echo "${reviewers[@]}"
-          echo "${statuses[@]}"
-          approves=()
-          reviewers_unique=()
-          for ((i=${#reviewers[@]}-1;i>=0;i--)); do
-            if ! echo "${reviewers_unique[@]}" | grep -q -w "${reviewers[$i]}" && [ "${statuses[$i]}" != "COMMENTED" ]; then
-                  reviewers_unique+=( "${reviewers[$i]}" )
-                  if [ "${statuses[$i]}" == "APPROVED" ]; then
-                      approves+=( "${reviewers[$i]}" )
-                  fi
-              fi
-          done
-          echo "${approves[@]}"
-          if [ ${#approves[@]} -lt 2 ]; then
-              echo "PR ${pr_num} has not been approved by at least 2 reviewers"
-              # shellcheck disable=SC2242
-              exit 1
-          fi
-          echo "Thanks for your contribution to Doris."
-          exit 0
+      - name: Install Python dependencies
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'  # Adjust if needed
+      - name: Install match library
+        run: |
+          pip install --upgrade pip 
+          pip install match
+          pip install requests
+      - name: Run Python script
+        run: |
+          python tools/maintainers/check_review.py  ${{ github.event.pull_request.number }} ${{secrets.GITHUB_TOKEN}}
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sonarcloud.yml

@@ -143,7 +143,7 @@ jobs:
           popd
 
           export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
-          DISABLE_JAVA_UDF=ON ENABLE_PCH=OFF DORIS_TOOLCHAIN=clang OUTPUT_BE_BINARY=0 ./build.sh --be
+          DISABLE_BE_JAVA_EXTENSIONS=ON DO_NOT_CHECK_JAVA_ENV=ON ENABLE_PCH=OFF DORIS_TOOLCHAIN=clang OUTPUT_BE_BINARY=0 ./build.sh --be
 
       - name:  Install sonar-scanner and build-wrapper
         if: ${{ steps.filter.outputs.be_changes == 'true' }}

.gitignore

@@ -81,6 +81,7 @@ samples/doris-demo/remote-udf-cpp-demo/function_server_demo
 
 # FE
 fe/mocked/
+fe/gen/
 fe/fe-core/gen/
 fe/fe-core/src/main/resources/static/
 fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.tokens