Merge branch 'master' into restore-examples

.github/labeler.yml

@@ -88,7 +88,7 @@
 'Kubernetes':
   - changed-files:
     - any-glob-to-any-file:
-      - 'extensions-contrib/kubernetes-overlord-extensions/**'
+      - 'extensions-core/kubernetes-overlord-extensions/**'
 
 'GHA':
   - changed-files:

.github/scripts/setup_generate_license.sh

@@ -18,6 +18,9 @@
 set -e
 
 sudo apt-get update && sudo apt-get install python3 -y
-curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
+# creating python virtual env
+python3 -m venv ~/.python3venv
+source ~/.python3venv/bin/activate
+sudo apt install python3-pip
 pip3 install wheel  # install wheel first explicitly
 pip3 install --upgrade pyyaml

.github/scripts/unit_tests_script.sh

@@ -21,8 +21,8 @@ unset _JAVA_OPTIONS
 
 # Set MAVEN_OPTS for Surefire launcher.
 MAVEN_OPTS='-Xmx2500m' ${MVN} test -pl ${MAVEN_PROJECTS} \
-${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} \
--DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}"
+${MAVEN_SKIP} \
+-DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}" -Pci
 sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
 free -m
 ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report || { echo "coverage_failure=false" >> "$GITHUB_ENV" && false; }

.github/workflows/codeql.yml

@@ -40,7 +40,7 @@ jobs:
     - uses: actions/setup-java@v4
       with:
         distribution: 'zulu'
-        java-version: '8'
+        java-version: '17'
         cache: 'maven'
 
     # Initializes the CodeQL tools for scanning.

.github/workflows/cron-job-its.yml

@@ -28,7 +28,7 @@ on:
 jobs:
   build:
     if: (github.event_name == 'schedule' && github.repository == 'apache/druid')
-    name: build (jdk8)
+    name: build (jdk17)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
@@ -37,15 +37,15 @@ jobs:
       - name: setup java
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
 
       - name: Cache Maven m2 repository
         id: maven
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: maven-${{ runner.os }}-8-${{ github.sha }}
+          key: maven-${{ runner.os }}-17-${{ github.sha }}
 
       - name: Maven build
         id: maven_build
@@ -60,8 +60,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       group: ${{ matrix.testing_group }}
@@ -74,8 +74,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: indexer
       group: ${{ matrix.testing_group }}
@@ -88,8 +88,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       override_config_path: ./environment-configs/test-groups/prepopulated-data
@@ -103,8 +103,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
       use_indexer: ${{ matrix.indexer }}
       group: other
@@ -122,7 +122,7 @@ jobs:
       - name: setup java
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
           cache: maven
 

.github/workflows/reusable-unit-tests.yml

@@ -21,11 +21,6 @@ on:
         required: true
         type: string
         description: 'JDK version used to test Druid'
-      sql_compatibility:
-        required: false
-        type: boolean
-        default: true
-        description: 'For SQL compatibility'
       module:
         required: true
         type: string
@@ -84,13 +79,6 @@ jobs:
         run: |
           export base_ref=${{ github.base_ref }}
           echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
-          # If sql_compatibilty is true, we want to set default_value_for_null
-          # which enables compatibility mode
-          if (${{ inputs.sql_compatibility }} == true); then
-            echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false" >> $GITHUB_ENV
-          else
-            echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true" >> $GITHUB_ENV
-          fi
 
       - name: test profiling
         run: | 
@@ -117,27 +105,30 @@ jobs:
           MAVEN_PROJECTS: ${{ inputs.maven_projects }}
         run: ./.github/scripts/unit_tests_script.sh
 
-      - name: Check for .hprof files on failure
+      - name: Check for dumps on failure
         if: ${{ failure() }}
-        id: check_for_heap_dump
+        id: check_for_dumps
         run: |
-          if ls ${GITHUB_WORKSPACE}/target/*.hprof 1> /dev/null 2>&1; then
-            echo "found_hprof=true" >> "$GITHUB_ENV"
+          if test -n "$(find "${GITHUB_WORKSPACE}" \( -name '*.hprof' -or -name 'hs_err_pid*' -or -name 'replay_pid*' -or -regex '.*/core\.[0-9]*' \))"
+          then
+            echo "found_dumps=true" >> "$GITHUB_ENV"
           else
-            echo "found_hprof=false" >> "$GITHUB_ENV"
+            echo "found_dumps=false" >> "$GITHUB_ENV"
           fi
 
-      - name: Collect tarball hprof dumps if they exist on failure
-        if: ${{ failure() && env.found_hprof == 'true' }}
+      - name: Collect dumps if they exist on failure
+        if: ${{ failure() && env.found_dumps == 'true' }}
         run: |
-          tar cvzf ${RUNNER_TEMP}/hprof-dumps.tgz ${GITHUB_WORKSPACE}/target/*.hprof
+          find "${GITHUB_WORKSPACE}" \
+            \( -name '*.hprof' -or -name 'hs_err_pid*' -or -name 'replay_pid*' -or -regex '.*/core\.[0-9]*' \) \
+            -exec tar -cvzf ${RUNNER_TEMP}/failure-dumps.tar.gz {} +
 
-      - name: Upload hprof dumps to GitHub if they exist on failure
-        if: ${{ failure() && env.found_hprof == 'true' }}
+      - name: Upload dumps to GitHub if they exist on failure
+        if: ${{ failure() && env.found_dumps == 'true' }}
         uses: actions/upload-artifact@master
         with:
-          name: Hprof-${{ inputs.group }} hprof dumps (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }})
-          path: ${{ runner.temp }}/hprof-dumps.tgz
+          name: Failure-${{ inputs.group }} failure dumps (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }})
+          path: ${{ runner.temp }}/failure-dumps.tar.gz
 
       - name: set outputs on failure
         id: set_outputs

.github/workflows/revised-its.yml

@@ -66,8 +66,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        #jdk: [8, 11, 17]
-        jdk: [8]
+        # jdk: [11, 17]
+        jdk: [17]
         it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query]
         #indexer: [indexer, middleManager]
         indexer: [middleManager]
@@ -86,8 +86,8 @@ jobs:
     uses: ./.github/workflows/reusable-revised-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       use_indexer: middleManager
       script: ./it.sh github S3DeepStorage
       it: S3DeepStorage
@@ -103,8 +103,8 @@ jobs:
     uses: ./.github/workflows/reusable-revised-its.yml
     if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       use_indexer: middleManager
       script: ./it.sh github BackwardCompatibilityMain
       it: BackwardCompatibilityMain

.github/workflows/standard-its.yml

@@ -51,8 +51,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       override_config_path: ./environment-configs/test-groups/prepopulated-data
       use_indexer: middleManager
@@ -67,8 +67,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: indexer
       group: ${{ matrix.testing_group }}
@@ -82,8 +82,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       override_config_path: ./environment-configs/test-groups/prepopulated-data
@@ -94,7 +94,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [8, 17, 21]
+        jdk: [11, 17, 21]
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
@@ -115,8 +115,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=shuffle-deep-store
       use_indexer: ${{ matrix.indexer }}
       override_config_path: ./environment-configs/test-groups/shuffle-deep-store
@@ -127,16 +127,16 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=custom-coordinator-duties
       use_indexer: middleManager
       override_config_path: ./environment-configs/test-groups/custom-coordinator-duties
       group: custom coordinator duties
 
   integration-k8s-leadership-tests:
     needs: changes
-    name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test
+    name: (Compile=openjdk17, Run=openjdk17, Cluster Build On K8s) ITNestedQueryPushDownTest integration test
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     runs-on: ubuntu-22.04
     env:
@@ -154,7 +154,7 @@ jobs:
       - name: setup java
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
 
       # the build step produces SNAPSHOT artifacts into the local maven repository,
@@ -164,7 +164,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ~/.m2/repository
-          key: maven-${{ runner.os }}-8-${{ github.sha }}
+          key: maven-${{ runner.os }}-17-${{ github.sha }}
           restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }}
 
       - name: Maven build
@@ -195,8 +195,8 @@ jobs:
         indexer: [middleManager, indexer]
     uses: ./.github/workflows/reusable-standard-its.yml
     with:
-      build_jdk: 8
-      runtime_jdk: 8
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-datasource-schema,cds-task-schema-publish-disabled,cds-coordinator-metadata-query-disabled
       use_indexer: ${{ matrix.indexer }}
       group: other