Merge branch 'master' into extern_tutorial

.github/scripts/setup_generate_license.sh

@@ -18,6 +18,9 @@
 set -e
 
 sudo apt-get update && sudo apt-get install python3 -y
-curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
+# creating python virtual env
+python3 -m venv ~/.python3venv
+source ~/.python3venv/bin/activate
+sudo apt install python3-pip
 pip3 install wheel  # install wheel first explicitly
 pip3 install --upgrade pyyaml

.github/workflows/codeql.yml

@@ -40,7 +40,7 @@ jobs:
     - uses: actions/setup-java@v4
       with:
         distribution: 'zulu'
-        java-version: '8'
+        java-version: '17'
         cache: 'maven'
 
     # Initializes the CodeQL tools for scanning.

.github/workflows/cron-job-its.yml

@@ -28,7 +28,7 @@ on:
 jobs:
   build:
     if: (github.event_name == 'schedule' && github.repository == 'apache/druid')
-    name: build (jdk8)
+    name: build (jdk17)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
@@ -37,7 +37,7 @@ jobs:
       - name: setup java
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
 
       - name: Cache Maven m2 repository
@@ -60,8 +60,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 21.0.4
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       group: ${{ matrix.testing_group }}
@@ -74,8 +74,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 21.0.4
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: indexer
       group: ${{ matrix.testing_group }}
@@ -88,8 +88,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 21.0.4
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       override_config_path: ./environment-configs/test-groups/prepopulated-data
@@ -103,8 +103,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 21.0.4
       testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
       use_indexer: ${{ matrix.indexer }}
       group: other
@@ -122,7 +122,7 @@ jobs:
       - name: setup java
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
           cache: maven
 

.github/workflows/reusable-unit-tests.yml

@@ -117,27 +117,30 @@ jobs:
           MAVEN_PROJECTS: ${{ inputs.maven_projects }}
         run: ./.github/scripts/unit_tests_script.sh
 
-      - name: Check for .hprof files on failure
+      - name: Check for dumps on failure
         if: ${{ failure() }}
-        id: check_for_heap_dump
+        id: check_for_dumps
         run: |
-          if ls ${GITHUB_WORKSPACE}/target/*.hprof 1> /dev/null 2>&1; then
-            echo "found_hprof=true" >> "$GITHUB_ENV"
+          if test -n "$(find "${GITHUB_WORKSPACE}" \( -name '*.hprof' -or -name 'hs_err_pid*' -or -name 'replay_pid*' -or -regex '.*/core\.[0-9]*' \))"
+          then
+            echo "found_dumps=true" >> "$GITHUB_ENV"
           else
-            echo "found_hprof=false" >> "$GITHUB_ENV"
+            echo "found_dumps=false" >> "$GITHUB_ENV"
           fi
 
-      - name: Collect tarball hprof dumps if they exist on failure
-        if: ${{ failure() && env.found_hprof == 'true' }}
+      - name: Collect dumps if they exist on failure
+        if: ${{ failure() && env.found_dumps == 'true' }}
         run: |
-          tar cvzf ${RUNNER_TEMP}/hprof-dumps.tgz ${GITHUB_WORKSPACE}/target/*.hprof
+          find "${GITHUB_WORKSPACE}" \
+            \( -name '*.hprof' -or -name 'hs_err_pid*' -or -name 'replay_pid*' -or -regex '.*/core\.[0-9]*' \) \
+            -exec tar -cvzf ${RUNNER_TEMP}/failure-dumps.tar.gz {} +
 
-      - name: Upload hprof dumps to GitHub if they exist on failure
-        if: ${{ failure() && env.found_hprof == 'true' }}
+      - name: Upload dumps to GitHub if they exist on failure
+        if: ${{ failure() && env.found_dumps == 'true' }}
         uses: actions/upload-artifact@master
         with:
-          name: Hprof-${{ inputs.group }} hprof dumps (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }})
-          path: ${{ runner.temp }}/hprof-dumps.tgz
+          name: Failure-${{ inputs.group }} failure dumps (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }})
+          path: ${{ runner.temp }}/failure-dumps.tar.gz
 
       - name: set outputs on failure
         id: set_outputs

.github/workflows/revised-its.yml

@@ -66,8 +66,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        #jdk: [8, 11, 17]
-        jdk: [8]
+        # jdk: [11, 17]
+        jdk: [17]
         it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query]
         #indexer: [indexer, middleManager]
         indexer: [middleManager]
@@ -86,8 +86,8 @@ jobs:
     uses: ./.github/workflows/reusable-revised-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 21.0.4
       use_indexer: middleManager
       script: ./it.sh github S3DeepStorage
       it: S3DeepStorage
@@ -103,8 +103,8 @@ jobs:
     uses: ./.github/workflows/reusable-revised-its.yml
     if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       use_indexer: middleManager
       script: ./it.sh github BackwardCompatibilityMain
       it: BackwardCompatibilityMain

.github/workflows/standard-its.yml

@@ -51,8 +51,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       override_config_path: ./environment-configs/test-groups/prepopulated-data
       use_indexer: middleManager
@@ -67,8 +67,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: indexer
       group: ${{ matrix.testing_group }}
@@ -82,8 +82,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       override_config_path: ./environment-configs/test-groups/prepopulated-data
@@ -94,7 +94,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [8, 17, 21]
+        jdk: [11, 17, 21]
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
@@ -115,8 +115,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=shuffle-deep-store
       use_indexer: ${{ matrix.indexer }}
       override_config_path: ./environment-configs/test-groups/shuffle-deep-store
@@ -127,16 +127,16 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=custom-coordinator-duties
       use_indexer: middleManager
       override_config_path: ./environment-configs/test-groups/custom-coordinator-duties
       group: custom coordinator duties
 
   integration-k8s-leadership-tests:
     needs: changes
-    name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test
+    name: (Compile=openjdk17, Run=openjdk17, Cluster Build On K8s) ITNestedQueryPushDownTest integration test
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     runs-on: ubuntu-22.04
     env:
@@ -154,7 +154,7 @@ jobs:
       - name: setup java
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
 
       # the build step produces SNAPSHOT artifacts into the local maven repository,
@@ -195,8 +195,8 @@ jobs:
         indexer: [middleManager, indexer]
     uses: ./.github/workflows/reusable-standard-its.yml
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-datasource-schema,cds-task-schema-publish-disabled,cds-coordinator-metadata-query-disabled
       use_indexer: ${{ matrix.indexer }}
       group: other

.github/workflows/static-checks.yml

@@ -41,7 +41,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [ '8', '11', '17', '21' ]
+        # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429
+        java: [ '11', '17', '21.0.4' ]
     runs-on: ubuntu-latest
     steps:
       - name: checkout branch
@@ -63,7 +64,7 @@ jobs:
 
       - name: script checks
         # who watches the watchers?
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: ./check_test_suite_test.py
 
       - name: (openjdk17) strict compilation
@@ -73,77 +74,45 @@ jobs:
         run: ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
       - name: maven install
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: |
           echo 'Running Maven install...' &&
           ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
           ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
       - name: checkstyle
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: ${MVN} checkstyle:checkstyle --fail-at-end
 
       - name: license checks
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: ./.github/scripts/license_checks_script.sh
 
       - name: analyze dependencies
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: |
           ./.github/scripts/analyze_dependencies_script.sh
 
       - name: animal sniffer checks
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: ${MVN} animal-sniffer:check --fail-at-end
 
       - name: enforcer checks
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: ${MVN} enforcer:enforce --fail-at-end
 
       - name: forbidden api checks
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
 
       - name: pmd checks
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: ${MVN} pmd:check --fail-at-end  # TODO: consider adding pmd:cpd-check
 
       - name: spotbugs checks
-        if: ${{ matrix.java == '8' }}
+        if: ${{ matrix.java == '17' }}
         run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
 
-  intellij-inspections:
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout branch
-        uses: actions/checkout@v4
-
-      - uses: actions/setup-java@v4
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-
-      - name: maven install
-        run: |
-          echo 'Running Maven install...' &&
-          ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
-          ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
-
-      - name: intellij inspections
-        run: |
-          docker run --rm \
-          -v $(pwd):/project \
-          -v ~/.m2:/home/inspect/.m2 \
-          -v $(pwd)/.idea/misc-for-inspection.xml:/project/.idea/misc.xml \
-          ccaominh/intellij-inspect:1.0.0 \
-          /project/pom.xml \
-          /project/.idea/inspectionProfiles/Druid.xml \
-          --levels ERROR \
-          --scope JavaInspectionsScope
-
   openrewrite:
     runs-on: ubuntu-latest
     steps:
@@ -153,7 +122,7 @@ jobs:
       - uses: actions/setup-java@v4
         with:
           distribution: 'zulu'
-          java-version: '8'
+          java-version: '17'
           cache: 'maven'
 
       - name: maven install