diff --git a/.asf.yaml b/.asf.yaml
index c0492350a3638..bd16084e4fe84 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -84,6 +84,9 @@ github:
branch-2.11: {}
branch-3.0: {}
branch-3.1: {}
+ branch-3.2: {}
+ branch-3.3: {}
+ branch-4.0: {}
notifications:
commits: commits@pulsar.apache.org
diff --git a/.github/actions/clean-disk/action.yml b/.github/actions/clean-disk/action.yml
index d74c3f25fc64c..e67ce59a08ddb 100644
--- a/.github/actions/clean-disk/action.yml
+++ b/.github/actions/clean-disk/action.yml
@@ -46,6 +46,15 @@ runs:
time df -BM / /mnt
echo "::endgroup::"
done
+ if [[ "${{ inputs.mode }}" == "full" ]]; then
+ echo "::group::Moving /var/lib/docker to /mnt/docker"
+ sudo systemctl stop docker
+ echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
+ sudo mv /var/lib/docker /mnt/docker
+ sudo systemctl start docker
+ time df -BM / /mnt
+ echo "::endgroup::"
+ fi
echo "::group::Cleaning apt state"
time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
time df -BM / /mnt
diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml
index a673a30843417..6b0310e487123 100644
--- a/.github/workflows/ci-maven-cache-update.yaml
+++ b/.github/workflows/ci-maven-cache-update.yaml
@@ -50,7 +50,7 @@ jobs:
name: Update Maven dependency cache for ${{ matrix.name }}
env:
JOB_NAME: Update Maven dependency cache for ${{ matrix.name }}
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
runs-on: ${{ matrix.runs-on }}
timeout-minutes: 45
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index a1c6dd594d3a2..4a2e12a06377a 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -24,7 +24,7 @@ on:
workflow_dispatch:
env:
- MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+ MAVEN_OPTS: -Xss1500k -Xmx2048m -XX:+UnlockDiagnosticVMOptions -XX:GCLockerRetryAllocationCount=100 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
JDK_DISTRIBUTION: corretto
NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
@@ -34,7 +34,7 @@ jobs:
name: Check ${{ matrix.branch }}
env:
JOB_NAME: Check ${{ matrix.branch }}
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
runs-on: ubuntu-22.04
timeout-minutes: 75
strategy:
diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml
index bfc5140943172..c1d0e3ae3f539 100644
--- a/.github/workflows/pulsar-ci-flaky.yaml
+++ b/.github/workflows/pulsar-ci-flaky.yaml
@@ -65,7 +65,7 @@ concurrency:
cancel-in-progress: true
env:
- MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+ MAVEN_OPTS: -Xss1500k -Xmx2048m -XX:+UnlockDiagnosticVMOptions -XX:GCLockerRetryAllocationCount=100 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
# defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
# it's possible to rerun individual failed jobs when the build artifacts are available
# if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
@@ -148,7 +148,7 @@ jobs:
env:
JOB_NAME: Flaky tests suite
COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index dd93003eecce6..0dfe0ce3a68a8 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -25,9 +25,9 @@ on:
- branch-*
- pulsar-*
schedule:
- # scheduled job with JDK 17
- - cron: '0 12 * * *'
# scheduled job with JDK 21
+ - cron: '0 12 * * *'
+ # scheduled job with JDK 17
# if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job
- cron: '0 6 * * *'
workflow_dispatch:
@@ -44,7 +44,7 @@ on:
options:
- '17'
- '21'
- default: '17'
+ default: '21'
trace_test_resource_cleanup:
description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
required: true
@@ -65,7 +65,7 @@ concurrency:
cancel-in-progress: true
env:
- MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+ MAVEN_OPTS: -Xss1500k -Xmx2048m -XX:+UnlockDiagnosticVMOptions -XX:GCLockerRetryAllocationCount=100 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
# defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
# it's possible to rerun individual failed jobs when the build artifacts are available
# if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
@@ -95,13 +95,13 @@ jobs:
- name: Select JDK major version
id: jdk_major_version
run: |
- # use JDK 21 for the scheduled build with cron expression '0 6 * * *'
+ # use JDK 17 for the scheduled build with cron expression '0 6 * * *'
if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then
- echo "jdk_major_version=21" >> $GITHUB_OUTPUT
+ echo "jdk_major_version=17" >> $GITHUB_OUTPUT
exit 0
fi
- # use JDK 17 for build unless overridden with workflow_dispatch input
- echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT
+ # use JDK 21 for build unless overridden with workflow_dispatch input
+ echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '21'}}" >> $GITHUB_OUTPUT
- name: checkout
if: ${{ github.event_name == 'pull_request' }}
@@ -147,7 +147,7 @@ jobs:
name: Build and License check
env:
JOB_NAME: Build and License check
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
runs-on: ubuntu-22.04
timeout-minutes: 60
@@ -224,7 +224,7 @@ jobs:
env:
JOB_NAME: CI - Unit - ${{ matrix.name }}
COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
@@ -472,7 +472,7 @@ jobs:
- linux/amd64
- linux/arm64
env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
@@ -550,7 +550,7 @@ jobs:
env:
JOB_NAME: CI - Integration - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
@@ -603,6 +603,9 @@ jobs:
- name: Metrics
group: METRICS
+ - name: Upgrade
+ group: UPGRADE
+
steps:
- name: checkout
uses: actions/checkout@v4
@@ -828,7 +831,7 @@ jobs:
needs: ['preconditions', 'build-and-license-check']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
@@ -890,8 +893,10 @@ jobs:
run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz
- name: Run Trivy container scan
- uses: aquasecurity/trivy-action@master
+ id: trivy_scan
+ uses: aquasecurity/trivy-action@0.26.0
if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
+ continue-on-error: true
with:
image-ref: "apachepulsar/pulsar:latest"
scanners: vuln
@@ -902,7 +907,8 @@ jobs:
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v3
- if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
+ if: ${{ steps.trivy_scan.outcome == 'success' && github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
+ continue-on-error: true
with:
sarif_file: 'trivy-results.sarif'
@@ -951,7 +957,7 @@ jobs:
env:
JOB_NAME: CI - System - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
@@ -1181,7 +1187,7 @@ jobs:
env:
JOB_NAME: CI Flaky - System - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
@@ -1324,7 +1330,7 @@ jobs:
needs: ['preconditions', 'integration-tests']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
- name: checkout
@@ -1364,7 +1370,7 @@ jobs:
contents: read
security-events: write
env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
CODEQL_LANGUAGE: java-kotlin
steps:
@@ -1425,7 +1431,7 @@ jobs:
needs: [ 'preconditions', 'integration-tests' ]
if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
steps:
@@ -1488,15 +1494,21 @@ jobs:
restore-keys: |
owasp-dependency-check-data-
+ - name: Log warning when skipped
+ if: ${{ !steps.restore-owasp-dependency-check-data.outputs.cache-matched-key }}
+ run: |
+ echo "::warning::OWASP Dependency Check was skipped since the OWASP Dependency check data wasn't found in the cache. Run ci-owasp-dependency-check.yaml workflow to update the cache."
+
# Projects dependent on flume, hdfs, and hbase currently excluded from the scan.
- name: trigger dependency check
+ if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-matched-key }}
run: |
mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
- -pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
+ -pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
- name: Upload report
uses: actions/upload-artifact@v4
- if: ${{ cancelled() || failure() }}
+ if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-matched-key && (cancelled() || failure()) }}
continue-on-error: true
with:
name: dependency report
diff --git a/.gitignore b/.gitignore
index 80d760cd29df7..fe6e44915e628 100644
--- a/.gitignore
+++ b/.gitignore
@@ -99,3 +99,5 @@ test-reports/
.mvn/.gradle-enterprise/
# Gradle Develocity
.mvn/.develocity/
+.vscode
+effective-pom.xml
diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml
index 5c0fbb47c7217..399b37899985d 100644
--- a/.mvn/develocity.xml
+++ b/.mvn/develocity.xml
@@ -24,16 +24,16 @@
#{(env['GRADLE_ENTERPRISE_ACCESS_KEY']?.trim() > '' or env['DEVELOCITY_ACCESS_KEY']?.trim() > '') and !(env['GITHUB_HEAD_REF']?.matches('(?i).*(experiment|wip|private).*') or env['GITHUB_REPOSITORY']?.matches('(?i).*(experiment|wip|private).*'))}
+ pulsar
- https://ge.apache.org
+ https://develocity.apache.org
false
-
- true
- true
-
#{isFalse(env['GITHUB_ACTIONS'])}
+
+ authenticated
+
#{{'0.0.0.0'}}
diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml
index eb998dc3471b8..8ceede33b9cdc 100644
--- a/.mvn/extensions.xml
+++ b/.mvn/extensions.xml
@@ -24,11 +24,11 @@
com.gradle
develocity-maven-extension
- 1.21.6
+ 1.22.2
com.gradle
common-custom-user-data-maven-extension
- 2.0
+ 2.0.1
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
index db95c131dde6f..d58dfb70bab56 100644
--- a/.mvn/wrapper/maven-wrapper.properties
+++ b/.mvn/wrapper/maven-wrapper.properties
@@ -5,14 +5,15 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.5/apache-maven-3.8.5-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar
+wrapperVersion=3.3.2
+distributionType=only-script
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip
diff --git a/bin/bookkeeper b/bin/bookkeeper
index 668c5d4db70a8..6be45bffdfa87 100755
--- a/bin/bookkeeper
+++ b/bin/bookkeeper
@@ -201,7 +201,7 @@ OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk
OPTS="-cp $BOOKIE_CLASSPATH $OPTS"
# Disable ipv6 as it can cause issues
-OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
+OPTS="-Djava.net.preferIPv4Stack=true $OPTS"
OPTS="$OPTS $BOOKIE_MEM $BOOKIE_GC $BOOKIE_GC_LOG $BOOKIE_EXTRA_OPTS"
@@ -232,20 +232,20 @@ OPTS="$OPTS $BK_METADATA_OPTIONS"
#Change to BK_HOME to support relative paths
cd "$BK_HOME"
if [ $COMMAND == "bookie" ]; then
- exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.server.Main --conf $BOOKIE_CONF $@
+ exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.server.Main --conf $BOOKIE_CONF "$@"
elif [ $COMMAND == "autorecovery" ]; then
- exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.replication.AutoRecoveryMain --conf $BOOKIE_CONF $@
+ exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.replication.AutoRecoveryMain --conf $BOOKIE_CONF "$@"
elif [ $COMMAND == "localbookie" ]; then
NUMBER=$1
shift
- exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.util.LocalBookKeeper $NUMBER $BOOKIE_CONF $@
+ exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.util.LocalBookKeeper $NUMBER $BOOKIE_CONF "$@"
elif [ $COMMAND == "upgrade" ]; then
- exec $JAVA $OPTS org.apache.bookkeeper.bookie.FileSystemUpgrade --conf $BOOKIE_CONF $@
+ exec $JAVA $OPTS org.apache.bookkeeper.bookie.FileSystemUpgrade --conf $BOOKIE_CONF "$@"
elif [ $COMMAND == "shell" ]; then
ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}"
- exec $JAVA $OPTS $ENTRY_FORMATTER_ARG org.apache.bookkeeper.bookie.BookieShell -conf $BOOKIE_CONF $@
+ exec $JAVA $OPTS $ENTRY_FORMATTER_ARG org.apache.bookkeeper.bookie.BookieShell -conf $BOOKIE_CONF "$@"
elif [ $COMMAND == "help" -o $COMMAND == "--help" -o $COMMAND == "-h" ]; then
bookkeeper_help;
else
- exec $JAVA $OPTS $COMMAND $@
+ exec $JAVA $OPTS $COMMAND "$@"
fi
diff --git a/bin/function-localrunner b/bin/function-localrunner
index b2405db724e72..90971277906db 100755
--- a/bin/function-localrunner
+++ b/bin/function-localrunner
@@ -52,7 +52,14 @@ done
PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"}
# Garbage collection options
-PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}
+if [ -z "$PULSAR_GC" ]; then
+ PULSAR_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"
+ if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then
+ PULSAR_GC="-XX:+UseZGC -XX:+ZGenerational ${PULSAR_GC}"
+ else
+ PULSAR_GC="-XX:+UseZGC ${PULSAR_GC}"
+ fi
+fi
# Garbage collection log.
PULSAR_GC_LOG_DIR=${PULSAR_GC_LOG_DIR:-logs}
@@ -124,7 +131,7 @@ fi
# Ensure we can read bigger content from ZK. (It might be
# rarely needed when trying to list many z-nodes under a
# directory)
-OPTS="$OPTS -Djute.maxbuffer=10485760 -Djava.net.preferIPv4Stack=true"
+OPTS="-Djava.net.preferIPv4Stack=true $OPTS -Djute.maxbuffer=10485760"
OPTS="-cp $PULSAR_CLASSPATH $OPTS"
diff --git a/bin/pulsar b/bin/pulsar
index 09be2ac50e279..4ff959a332c9f 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -273,7 +273,7 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
# Ensure we can read bigger content from ZK. (It might be
# rarely needed when trying to list many z-nodes under a
# directory)
-OPTS="$OPTS -Djute.maxbuffer=10485760 -Djava.net.preferIPv4Stack=true"
+OPTS="-Djava.net.preferIPv4Stack=true $OPTS -Djute.maxbuffer=10485760"
# Enable TCP keepalive for all Zookeeper client connections
OPTS="$OPTS -Dzookeeper.clientTcpKeepAlive=true"
@@ -303,8 +303,6 @@ else
fi
PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"}
-PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}
-PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"}
PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"}
if [ ! -d "$PULSAR_LOG_DIR" ]; then
mkdir -p "$PULSAR_LOG_DIR"
@@ -314,8 +312,14 @@ PULSAR_LOG_IMMEDIATE_FLUSH="${PULSAR_LOG_IMMEDIATE_FLUSH:-"false"}"
#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
-OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
-OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
+if [ -n "$PULSAR_LOG_LEVEL" ]; then
+ OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
+fi
+if [ -n "$PULSAR_LOG_ROOT_LEVEL" ]; then
+ OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
+elif [ -n "$PULSAR_LOG_LEVEL" ]; then
+ OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_LEVEL"
+fi
OPTS="$OPTS -Dpulsar.log.immediateFlush=$PULSAR_LOG_IMMEDIATE_FLUSH"
OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT"
# Configure log4j2 to disable servlet webapp detection so that Garbage free logging can be used
@@ -346,56 +350,56 @@ fi
cd "$PULSAR_HOME"
if [ $COMMAND == "broker" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-broker.log"}
- exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarBrokerStarter --broker-conf $PULSAR_BROKER_CONF $@
+ exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarBrokerStarter --broker-conf $PULSAR_BROKER_CONF "$@"
elif [ $COMMAND == "bookie" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"bookkeeper.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.server.Main --conf $PULSAR_BOOKKEEPER_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.server.Main --conf $PULSAR_BOOKKEEPER_CONF "$@"
elif [ $COMMAND == "zookeeper" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"zookeeper.log"}
- exec $JAVA ${ZK_OPTS} $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_ZK_CONF $@
+ exec $JAVA ${ZK_OPTS} $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_ZK_CONF "$@"
elif [ $COMMAND == "global-zookeeper" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"global-zookeeper.log"}
# Allow global ZK to turn into read-only mode when it cannot reach the quorum
OPTS="${OPTS} ${ZK_OPTS} -Dreadonlymode.enabled=true"
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_GLOBAL_ZK_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_GLOBAL_ZK_CONF "$@"
elif [ $COMMAND == "configuration-store" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"configuration-store.log"}
# Allow global ZK to turn into read-only mode when it cannot reach the quorum
OPTS="${OPTS} ${ZK_OPTS} -Dreadonlymode.enabled=true"
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_CONFIGURATION_STORE_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_CONFIGURATION_STORE_CONF "$@"
elif [ $COMMAND == "proxy" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-proxy.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.proxy.server.ProxyServiceStarter --config $PULSAR_PROXY_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.proxy.server.ProxyServiceStarter --config $PULSAR_PROXY_CONF "$@"
elif [ $COMMAND == "websocket" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-websocket.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.websocket.service.WebSocketServiceStarter $PULSAR_WEBSOCKET_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.websocket.service.WebSocketServiceStarter $PULSAR_WEBSOCKET_CONF "$@"
elif [ $COMMAND == "functions-worker" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-functions-worker.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF "$@"
elif [ $COMMAND == "standalone" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-standalone.log"}
- exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS ${ZK_OPTS} -Dpulsar.log.file=$PULSAR_LOG_FILE -Dpulsar.config.file=$PULSAR_STANDALONE_CONF org.apache.pulsar.PulsarStandaloneStarter $@
+ exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS ${ZK_OPTS} -Dpulsar.log.file=$PULSAR_LOG_FILE -Dpulsar.config.file=$PULSAR_STANDALONE_CONF org.apache.pulsar.PulsarStandaloneStarter "$@"
elif [ ${COMMAND} == "autorecovery" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-autorecovery.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.replication.AutoRecoveryMain --conf $PULSAR_BOOKKEEPER_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.replication.AutoRecoveryMain --conf $PULSAR_BOOKKEEPER_CONF "$@"
elif [ $COMMAND == "initialize-cluster-metadata" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataSetup $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataSetup "$@"
elif [ $COMMAND == "delete-cluster-metadata" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataTeardown $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataTeardown "$@"
elif [ $COMMAND == "initialize-transaction-coordinator-metadata" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarTransactionCoordinatorMetadataSetup $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarTransactionCoordinatorMetadataSetup "$@"
elif [ $COMMAND == "initialize-namespace" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarInitialNamespaceSetup $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarInitialNamespaceSetup "$@"
elif [ $COMMAND == "zookeeper-shell" ]; then
- exec $JAVA $OPTS org.apache.zookeeper.ZooKeeperMain $@
+ exec $JAVA $OPTS org.apache.zookeeper.ZooKeeperMain "$@"
elif [ $COMMAND == "broker-tool" ]; then
- exec $JAVA $OPTS org.apache.pulsar.broker.tools.BrokerTool $@
+ exec $JAVA $OPTS org.apache.pulsar.broker.tools.BrokerTool "$@"
elif [ $COMMAND == "compact-topic" ]; then
- exec $JAVA $OPTS org.apache.pulsar.compaction.CompactorTool --broker-conf $PULSAR_BROKER_CONF $@
+ exec $JAVA $OPTS org.apache.pulsar.compaction.CompactorTool --broker-conf $PULSAR_BROKER_CONF "$@"
elif [ $COMMAND == "tokens" ]; then
- exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils $@
+ exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils "$@"
elif [ $COMMAND == "version" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarVersionStarter $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarVersionStarter "$@"
elif [ $COMMAND == "help" -o $COMMAND == "--help" -o $COMMAND == "-h" ]; then
pulsar_help;
else
diff --git a/bin/pulsar-admin-common.cmd b/bin/pulsar-admin-common.cmd
index c59f0e9b424d3..265af6bafa8ad 100644
--- a/bin/pulsar-admin-common.cmd
+++ b/bin/pulsar-admin-common.cmd
@@ -60,7 +60,7 @@ for %%a in ("%PULSAR_LOG_CONF%") do SET "PULSAR_LOG_CONF_BASENAME=%%~nxa"
set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_LOG_CONF_DIR%"
set "OPTS=%OPTS% -Dlog4j.configurationFile="%PULSAR_LOG_CONF_BASENAME%""
-set "OPTS=%OPTS% -Djava.net.preferIPv4Stack=true"
+set "OPTS=-Djava.net.preferIPv4Stack=true %OPTS%"
REM Allow Netty to use reflection access
set "OPTS=%OPTS% -Dio.netty.tryReflectionSetAccessible=true"
@@ -83,14 +83,18 @@ set "OPTS=%OPTS% %PULSAR_EXTRA_OPTS%"
if "%PULSAR_LOG_DIR%" == "" set "PULSAR_LOG_DIR=%PULSAR_HOME%\logs"
if "%PULSAR_LOG_APPENDER%" == "" set "PULSAR_LOG_APPENDER=RoutingAppender"
-if "%PULSAR_LOG_LEVEL%" == "" set "PULSAR_LOG_LEVEL=info"
-if "%PULSAR_LOG_ROOT_LEVEL%" == "" set "PULSAR_LOG_ROOT_LEVEL=%PULSAR_LOG_LEVEL%"
if "%PULSAR_ROUTING_APPENDER_DEFAULT%" == "" set "PULSAR_ROUTING_APPENDER_DEFAULT=Console"
if "%PULSAR_LOG_IMMEDIATE_FLUSH%" == "" set "PULSAR_LOG_IMMEDIATE_FLUSH=false"
set "OPTS=%OPTS% -Dpulsar.log.appender=%PULSAR_LOG_APPENDER%"
set "OPTS=%OPTS% -Dpulsar.log.dir=%PULSAR_LOG_DIR%"
-set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%"
-set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%"
+if not "%PULSAR_LOG_LEVEL%" == "" set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%"
+if not "%PULSAR_LOG_ROOT_LEVEL%" == "" (
+ set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%"
+) else (
+ if not "%PULSAR_LOG_LEVEL%" == "" (
+ set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_LEVEL%"
+ )
+)
set "OPTS=%OPTS% -Dpulsar.log.immediateFlush=%PULSAR_LOG_IMMEDIATE_FLUSH%"
set "OPTS=%OPTS% -Dpulsar.routing.appender.default=%PULSAR_ROUTING_APPENDER_DEFAULT%"
\ No newline at end of file
diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh
index 336ff43c1a861..d83408bdf8bf7 100755
--- a/bin/pulsar-admin-common.sh
+++ b/bin/pulsar-admin-common.sh
@@ -104,7 +104,7 @@ fi
PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
-OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
+OPTS="-Djava.net.preferIPv4Stack=true $OPTS"
# Allow Netty to use reflection access
OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
diff --git a/bin/pulsar-perf b/bin/pulsar-perf
index 9108a42ef994f..2e957e4a3e9dc 100755
--- a/bin/pulsar-perf
+++ b/bin/pulsar-perf
@@ -101,7 +101,7 @@ fi
PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
-OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF` -Djava.net.preferIPv4Stack=true"
+OPTS="-Djava.net.preferIPv4Stack=true $OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
# Allow Netty to use reflection access
OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
@@ -125,14 +125,18 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS"
# log directory & file
PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-perftest.log"}
-PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}
-PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"}
PULSAR_LOG_IMMEDIATE_FLUSH="${PULSAR_LOG_IMMEDIATE_FLUSH:-"false"}"
#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
-OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
-OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
+if [ -n "$PULSAR_LOG_LEVEL" ]; then
+ OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
+fi
+if [ -n "$PULSAR_LOG_ROOT_LEVEL" ]; then
+ OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
+elif [ -n "$PULSAR_LOG_LEVEL" ]; then
+ OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_LEVEL"
+fi
OPTS="$OPTS -Dpulsar.log.immediateFlush=$PULSAR_LOG_IMMEDIATE_FLUSH"
OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
OPTS="$OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE"
diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml
index e440923af6de5..a8c4a9656db44 100644
--- a/bouncy-castle/bc/pom.xml
+++ b/bouncy-castle/bc/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
bouncy-castle-parent
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
bouncy-castle-bc
diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml
index f4478174b86dd..441d200327970 100644
--- a/bouncy-castle/bcfips-include-test/pom.xml
+++ b/bouncy-castle/bcfips-include-test/pom.xml
@@ -24,7 +24,7 @@
org.apache.pulsar
bouncy-castle-parent
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
bcfips-include-test
diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml
index 250b3db6b9b08..2910f15080aa5 100644
--- a/bouncy-castle/bcfips/pom.xml
+++ b/bouncy-castle/bcfips/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
bouncy-castle-parent
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
bouncy-castle-bcfips
diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml
index 4d85a163104a2..63c2508c45de2 100644
--- a/bouncy-castle/pom.xml
+++ b/bouncy-castle/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
pulsar
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh
index 2d82fce08878d..63b92d4e0a798 100755
--- a/build/run_integration_group.sh
+++ b/build/run_integration_group.sh
@@ -177,6 +177,10 @@ test_group_standalone() {
mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-standalone.xml -DintegrationTests
}
+test_group_upgrade() {
+ mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-upgrade.xml -DintegrationTests
+}
+
test_group_transaction() {
mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-transaction.xml -DintegrationTests
}
diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
index cdaf69e351b6d..40f83efaf2fb9 100755
--- a/build/run_unit_group.sh
+++ b/build/run_unit_group.sh
@@ -77,6 +77,8 @@ alias echo='{ [[ $- =~ .*x.* ]] && trace_enabled=1 || trace_enabled=0; set +x; }
# Test Groups -- start --
function test_group_broker_group_1() {
mvn_test -pl pulsar-broker -Dgroups='broker' -DtestReuseFork=true
+ # run tests in broker-isolated group individually (instead of with -Dgroups=broker-isolated) to avoid scanning all test classes
+ mvn_test -pl pulsar-broker -Dtest=org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGeneratorWithNoUnsafeTest -DtestForkCount=1 -DtestReuseFork=false
}
function test_group_broker_group_2() {
diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index b1ae0cd9b73f3..7c78924a371e2 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -31,12 +31,19 @@
org.apache.pulsar
buildtools
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
jar
Pulsar Build Tools
+
+
+ Apache Pulsar developers
+ http://pulsar.apache.org/
+
+
+
- 2024-08-09T08:42:01Z
+ 2024-10-14T13:32:50Z
1.8
1.8
3.1.0
@@ -47,7 +54,7 @@
4.1
10.14.2
3.1.2
- 4.1.104.Final
+ 4.1.118.Final
4.2.3
32.1.2-jre
1.10.12
diff --git a/conf/bkenv.sh b/conf/bkenv.sh
index b41532d3a0c91..8beea47cee312 100644
--- a/conf/bkenv.sh
+++ b/conf/bkenv.sh
@@ -37,9 +37,6 @@ BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"${PULSAR_LOG_DIR}"}
# Memory size options
BOOKIE_MEM=${BOOKIE_MEM:-${PULSAR_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=2g"}}
-# Garbage collection options
-BOOKIE_GC=${BOOKIE_GC:-${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}}
-
if [ -z "$JAVA_HOME" ]; then
JAVA_BIN=java
else
@@ -60,6 +57,17 @@ for token in $("$JAVA_BIN" -version 2>&1 | grep 'version "'); do
fi
done
+# Garbage collection options
+BOOKIE_GC="${BOOKIE_GC:-${PULSAR_GC}}"
+if [ -z "$BOOKIE_GC" ]; then
+ BOOKIE_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"
+ if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then
+ BOOKIE_GC="-XX:+UseZGC -XX:+ZGenerational ${BOOKIE_GC}"
+ else
+ BOOKIE_GC="-XX:+UseZGC ${BOOKIE_GC}"
+ fi
+fi
+
if [[ -z "$BOOKIE_GC_LOG" ]]; then
# fallback to PULSAR_GC_LOG if it is set
BOOKIE_GC_LOG="$PULSAR_GC_LOG"
diff --git a/conf/broker.conf b/conf/broker.conf
index 125b2aa8c1b39..f68306ec7b4d7 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -198,6 +198,10 @@ allowAutoTopicCreation=true
# The type of topic that is allowed to be automatically created.(partitioned/non-partitioned)
allowAutoTopicCreationType=non-partitioned
+# If 'allowAutoTopicCreation' is true and the name of the topic contains 'cluster',
+# the topic cannot be automatically created.
+allowAutoTopicCreationWithLegacyNamingScheme=true
+
# Enable subscription auto creation if new consumer connected (disable auto creation with value false)
allowAutoSubscriptionCreation=true
@@ -243,6 +247,10 @@ messageExpiryCheckIntervalInMinutes=5
# How long to delay rewinding cursor and dispatching messages when active consumer is changed
activeConsumerFailoverDelayTimeMillis=1000
+# Enable consistent hashing for selecting the active consumer in partitioned topics with Failover subscription type.
+# For non-partitioned topics, consistent hashing is used by default.
+activeConsumerFailoverConsistentHashing=false
+
# How long to delete inactive subscriptions from last consuming
# When it is 0, inactive subscriptions are not deleted automatically
subscriptionExpirationTimeMinutes=0
@@ -489,12 +497,12 @@ dispatcherReadFailureBackoffMandatoryStopTimeInMs=0
# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered
# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff
# delay. This parameter sets the initial backoff delay in milliseconds.
-dispatcherRetryBackoffInitialTimeInMs=100
+dispatcherRetryBackoffInitialTimeInMs=1
# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered
# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff
# delay. This parameter sets the maximum backoff delay in milliseconds.
-dispatcherRetryBackoffMaxTimeInMs=1000
+dispatcherRetryBackoffMaxTimeInMs=10
# Precise dispatcher flow control according to history message number of each entry
preciseDispatcherFlowControl=false
@@ -1689,6 +1697,8 @@ exposePublisherStats=true
statsUpdateFrequencyInSecs=60
statsUpdateInitialDelayInSecs=60
+healthCheckMetricsUpdateTimeInSeconds=-1
+
# Enable expose the precise backlog stats.
# Set false to use published counter and consumed counter to calculate, this would be more efficient but may be inaccurate.
# Default is false.
diff --git a/conf/proxy.conf b/conf/proxy.conf
index 6e6c960e8009e..46d84744e1297 100644
--- a/conf/proxy.conf
+++ b/conf/proxy.conf
@@ -59,10 +59,21 @@ bindAddress=0.0.0.0
# If not set, the value of `InetAddress.getLocalHost().getCanonicalHostName()` is used.
advertisedAddress=
+# Specifies the interval (in seconds) for sending ping messages to the client. Set to 0 to disable
+# ping messages. This setting applies to client connections used for topic lookups and
+# partition metadata requests. When a client establishes a broker connection via the proxy,
+# the client and broker will communicate directly without the proxy intercepting the messages.
+# In that case, the broker's keepAliveIntervalSeconds configuration becomes relevant.
+keepAliveIntervalSeconds=30
+
# Enable or disable the HAProxy protocol.
# If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
haProxyProtocolEnabled=false
+# Default http header map to add into http-proxy for the any security requirements.
+# eg: {"header1":"value"}
+proxyHttpResponseHeadersJson=
+
# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
webServiceHaProxyProtocolEnabled=false
diff --git a/conf/pulsar_env.sh b/conf/pulsar_env.sh
index 3a069e31fdc90..f95d0ac83c13a 100755
--- a/conf/pulsar_env.sh
+++ b/conf/pulsar_env.sh
@@ -44,9 +44,6 @@
# Extra options to be passed to the jvm
PULSAR_MEM=${PULSAR_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=4g"}
-# Garbage collection options
-PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}
-
if [ -z "$JAVA_HOME" ]; then
JAVA_BIN=java
else
@@ -67,6 +64,16 @@ for token in $("$JAVA_BIN" -version 2>&1 | grep 'version "'); do
fi
done
+# Garbage collection options
+if [ -z "$PULSAR_GC" ]; then
+ PULSAR_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"
+ if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then
+ PULSAR_GC="-XX:+UseZGC -XX:+ZGenerational ${PULSAR_GC}"
+ else
+ PULSAR_GC="-XX:+UseZGC ${PULSAR_GC}"
+ fi
+fi
+
PULSAR_GC_LOG_DIR=${PULSAR_GC_LOG_DIR:-"${PULSAR_LOG_DIR}"}
if [[ -z "$PULSAR_GC_LOG" ]]; then
diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh
index 9d22b73905df3..96ee304bf0b3a 100755
--- a/conf/pulsar_tools_env.sh
+++ b/conf/pulsar_tools_env.sh
@@ -57,9 +57,6 @@ if [ -n "$PULSAR_MEM" ]; then
fi
PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"}
-# Garbage collection options
-PULSAR_GC=${PULSAR_GC:-" -client "}
-
# Extra options to be passed to the jvm
PULSAR_EXTRA_OPTS="${PULSAR_MEM} ${PULSAR_GC} ${PULSAR_GC_LOG} -Dio.netty.leakDetectionLevel=disabled ${PULSAR_EXTRA_OPTS}"
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 622949bf6c325..2036556da4385 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -152,6 +152,10 @@ maxMessageSizeCheckIntervalInSeconds=60
# How long to delay rewinding cursor and dispatching messages when active consumer is changed
activeConsumerFailoverDelayTimeMillis=1000
+# Enable consistent hashing for selecting the active consumer in partitioned topics with Failover subscription type.
+# For non-partitioned topics, consistent hashing is used by default.
+activeConsumerFailoverConsistentHashing=false
+
# How long to delete inactive subscriptions from last consuming
# When it is 0, inactive subscriptions are not deleted automatically
subscriptionExpirationTimeMinutes=0
@@ -305,12 +309,12 @@ dispatcherReadFailureBackoffMandatoryStopTimeInMs=0
# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered
# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff
# delay. This parameter sets the initial backoff delay in milliseconds.
-dispatcherRetryBackoffInitialTimeInMs=100
+dispatcherRetryBackoffInitialTimeInMs=1
# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered
# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff
# delay. This parameter sets the maximum backoff delay in milliseconds.
-dispatcherRetryBackoffMaxTimeInMs=1000
+dispatcherRetryBackoffMaxTimeInMs=10
# Precise dispatcher flow control according to history message number of each entry
preciseDispatcherFlowControl=false
@@ -1177,6 +1181,10 @@ allowAutoTopicCreation=true
# The type of topic that is allowed to be automatically created.(partitioned/non-partitioned)
allowAutoTopicCreationType=non-partitioned
+# If 'allowAutoTopicCreation' is true and the name of the topic contains 'cluster',
+# the topic cannot be automatically created.
+allowAutoTopicCreationWithLegacyNamingScheme=true
+
# Enable subscription auto creation if new consumer connected (disable auto creation with value false)
allowAutoSubscriptionCreation=true
diff --git a/deployment/terraform-ansible/deploy-pulsar.yaml b/deployment/terraform-ansible/deploy-pulsar.yaml
index db2fd1257ca41..3a9f0fd942c17 100644
--- a/deployment/terraform-ansible/deploy-pulsar.yaml
+++ b/deployment/terraform-ansible/deploy-pulsar.yaml
@@ -147,7 +147,6 @@
# - file
# - flume
# - hbase
-# - hdfs2
# - hdfs3
# - influxdb
# - jdbc-clickhouse
diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf
index ff3677174024c..bae55cb69f1ee 100644
--- a/deployment/terraform-ansible/templates/broker.conf
+++ b/deployment/terraform-ansible/templates/broker.conf
@@ -148,6 +148,10 @@ messageExpiryCheckIntervalInMinutes=5
# How long to delay rewinding cursor and dispatching messages when active consumer is changed
activeConsumerFailoverDelayTimeMillis=1000
+# Enable consistent hashing for selecting the active consumer in partitioned topics with Failover subscription type.
+# For non-partitioned topics, consistent hashing is used by default.
+activeConsumerFailoverConsistentHashing=false
+
# How long to delete inactive subscriptions from last consuming
# When it is 0, inactive subscriptions are not deleted automatically
subscriptionExpirationTimeMinutes=0
diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
index 813c4d26d9391..2c9ee5dbd68a6 100644
--- a/distribution/io/pom.xml
+++ b/distribution/io/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
distribution
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
pulsar-io-distribution
diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml
index f98ee14bb20c9..cf7731b4c85ab 100644
--- a/distribution/io/src/assemble/io.xml
+++ b/distribution/io/src/assemble/io.xml
@@ -63,7 +63,6 @@
${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar
${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar
${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar
- ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar
${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar
${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar
${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar
diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml
index 38beeacde8ba4..1708684cf04f9 100644
--- a/distribution/offloaders/pom.xml
+++ b/distribution/offloaders/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
distribution
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
pulsar-offloader-distribution
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 67604e145dd73..3523fce8fcec4 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
pulsar
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
distribution
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
index 36641dea20f0c..c8815fdd8da8d 100644
--- a/distribution/server/pom.xml
+++ b/distribution/server/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
distribution
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
pulsar-server-distribution
@@ -49,7 +49,6 @@
${project.groupId}
jetcd-core-shaded
${project.version}
- shaded
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 579613b0d8f2f..6cddc05ecddf5 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -262,6 +262,7 @@ The Apache Software License, Version 2.0
- com.fasterxml.jackson.module-jackson-module-parameter-names-2.17.2.jar
* Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar
* Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
+ * Fastutil -- it.unimi.dsi-fastutil-8.5.14.jar
* Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.17.0.jar
* Bitbucket -- org.bitbucket.b_c-jose4j-0.9.4.jar
* Gson
@@ -284,7 +285,7 @@ The Apache Software License, Version 2.0
- commons-cli-commons-cli-1.5.0.jar
- commons-codec-commons-codec-1.15.jar
- commons-configuration-commons-configuration-1.10.jar
- - commons-io-commons-io-2.8.0.jar
+ - commons-io-commons-io-2.18.0.jar
- commons-lang-commons-lang-2.6.jar
- commons-logging-commons-logging-1.1.1.jar
- org.apache.commons-commons-collections4-4.4.jar
@@ -292,36 +293,36 @@ The Apache Software License, Version 2.0
- org.apache.commons-commons-lang3-3.11.jar
- org.apache.commons-commons-text-1.10.0.jar
* Netty
- - io.netty-netty-buffer-4.1.113.Final.jar
- - io.netty-netty-codec-4.1.113.Final.jar
- - io.netty-netty-codec-dns-4.1.113.Final.jar
- - io.netty-netty-codec-http-4.1.113.Final.jar
- - io.netty-netty-codec-http2-4.1.113.Final.jar
- - io.netty-netty-codec-socks-4.1.113.Final.jar
- - io.netty-netty-codec-haproxy-4.1.113.Final.jar
- - io.netty-netty-common-4.1.113.Final.jar
- - io.netty-netty-handler-4.1.113.Final.jar
- - io.netty-netty-handler-proxy-4.1.113.Final.jar
- - io.netty-netty-resolver-4.1.113.Final.jar
- - io.netty-netty-resolver-dns-4.1.113.Final.jar
- - io.netty-netty-resolver-dns-classes-macos-4.1.113.Final.jar
- - io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar
- - io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar
- - io.netty-netty-transport-4.1.113.Final.jar
- - io.netty-netty-transport-classes-epoll-4.1.113.Final.jar
- - io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar
- - io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar
- - io.netty-netty-transport-native-unix-common-4.1.113.Final.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar
- - io.netty-netty-tcnative-classes-2.0.66.Final.jar
- - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.24.Final.jar
- - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar
- - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar
+ - io.netty-netty-buffer-4.1.118.Final.jar
+ - io.netty-netty-codec-4.1.118.Final.jar
+ - io.netty-netty-codec-dns-4.1.118.Final.jar
+ - io.netty-netty-codec-http-4.1.118.Final.jar
+ - io.netty-netty-codec-http2-4.1.118.Final.jar
+ - io.netty-netty-codec-socks-4.1.118.Final.jar
+ - io.netty-netty-codec-haproxy-4.1.118.Final.jar
+ - io.netty-netty-common-4.1.118.Final.jar
+ - io.netty-netty-handler-4.1.118.Final.jar
+ - io.netty-netty-handler-proxy-4.1.118.Final.jar
+ - io.netty-netty-resolver-4.1.118.Final.jar
+ - io.netty-netty-resolver-dns-4.1.118.Final.jar
+ - io.netty-netty-resolver-dns-classes-macos-4.1.118.Final.jar
+ - io.netty-netty-resolver-dns-native-macos-4.1.118.Final-osx-aarch_64.jar
+ - io.netty-netty-resolver-dns-native-macos-4.1.118.Final-osx-x86_64.jar
+ - io.netty-netty-transport-4.1.118.Final.jar
+ - io.netty-netty-transport-classes-epoll-4.1.118.Final.jar
+ - io.netty-netty-transport-native-epoll-4.1.118.Final-linux-aarch_64.jar
+ - io.netty-netty-transport-native-epoll-4.1.118.Final-linux-x86_64.jar
+ - io.netty-netty-transport-native-unix-common-4.1.118.Final.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar
+ - io.netty-netty-tcnative-classes-2.0.70.Final.jar
+ - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar
+ - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
+ - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar
* Prometheus client
- io.prometheus.jmx-collector-0.16.1.jar
- io.prometheus-simpleclient-0.16.0.jar
@@ -337,12 +338,12 @@ The Apache Software License, Version 2.0
- io.prometheus-simpleclient_tracer_otel-0.16.0.jar
- io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar
* Prometheus exporter
- - io.prometheus-prometheus-metrics-config-1.2.1.jar
- - io.prometheus-prometheus-metrics-exporter-common-1.2.1.jar
- - io.prometheus-prometheus-metrics-exporter-httpserver-1.2.1.jar
- - io.prometheus-prometheus-metrics-exposition-formats-1.2.1.jar
- - io.prometheus-prometheus-metrics-model-1.2.1.jar
- - io.prometheus-prometheus-metrics-shaded-protobuf-1.2.1.jar
+ - io.prometheus-prometheus-metrics-config-1.3.4.jar
+ - io.prometheus-prometheus-metrics-exporter-common-1.3.4.jar
+ - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.4.jar
+ - io.prometheus-prometheus-metrics-exposition-formats-1.3.4.jar
+ - io.prometheus-prometheus-metrics-model-1.3.4.jar
+ - io.prometheus-prometheus-metrics-exposition-textformats-1.3.4.jar
* Jakarta Bean Validation API
- jakarta.validation-jakarta.validation-api-2.0.2.jar
- javax.validation-validation-api-1.1.0.Final.jar
@@ -389,28 +390,28 @@ The Apache Software License, Version 2.0
* AirCompressor
- io.airlift-aircompressor-0.27.jar
* AsyncHttpClient
- - org.asynchttpclient-async-http-client-2.12.1.jar
- - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar
+ - org.asynchttpclient-async-http-client-2.12.4.jar
+ - org.asynchttpclient-async-http-client-netty-utils-2.12.4.jar
* Jetty
- - org.eclipse.jetty-jetty-client-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-continuation-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-http-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-io-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-proxy-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-security-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-server-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-servlet-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-servlets-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-util-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-util-ajax-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-api-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-client-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-common-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-server-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-servlet-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-alpn-server-9.4.54.v20240208.jar
+ - org.eclipse.jetty-jetty-client-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-continuation-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-http-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-io-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-proxy-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-security-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-server-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-servlet-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-servlets-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-util-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-util-ajax-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-api-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-client-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-common-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-server-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-servlet-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-alpn-server-9.4.56.v20240826.jar
* SnakeYaml -- org.yaml-snakeyaml-2.0.jar
* RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar
* Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.24.0.jar
@@ -460,12 +461,12 @@ The Apache Software License, Version 2.0
* zt-zip
- org.zeroturnaround-zt-zip-1.17.jar
* Apache Avro
- - org.apache.avro-avro-1.11.3.jar
- - org.apache.avro-avro-protobuf-1.11.3.jar
+ - org.apache.avro-avro-1.11.4.jar
+ - org.apache.avro-avro-protobuf-1.11.4.jar
* Apache Curator
- - org.apache.curator-curator-client-5.1.0.jar
- - org.apache.curator-curator-framework-5.1.0.jar
- - org.apache.curator-curator-recipes-5.1.0.jar
+ - org.apache.curator-curator-client-5.7.1.jar
+ - org.apache.curator-curator-framework-5.7.1.jar
+ - org.apache.curator-curator-recipes-5.7.1.jar
* Apache Yetus
- org.apache.yetus-audience-annotations-0.12.0.jar
* Kubernetes Client
@@ -480,8 +481,8 @@ The Apache Software License, Version 2.0
* Prometheus
- io.prometheus-simpleclient_httpserver-0.16.0.jar
* Oxia
- - io.streamnative.oxia-oxia-client-api-0.4.5.jar
- - io.streamnative.oxia-oxia-client-0.4.5.jar
+ - io.streamnative.oxia-oxia-client-api-0.5.0.jar
+ - io.streamnative.oxia-oxia-client-0.5.0.jar
* OpenHFT
- net.openhft-zero-allocation-hashing-0.16.jar
* Java JSON WebTokens
@@ -491,15 +492,15 @@ The Apache Software License, Version 2.0
* JCTools - Java Concurrency Tools for the JVM
- org.jctools-jctools-core-2.1.2.jar
* Vertx
- - io.vertx-vertx-auth-common-4.5.8.jar
- - io.vertx-vertx-bridge-common-4.5.8.jar
- - io.vertx-vertx-core-4.5.8.jar
- - io.vertx-vertx-web-4.5.8.jar
- - io.vertx-vertx-web-common-4.5.8.jar
+ - io.vertx-vertx-auth-common-4.5.10.jar
+ - io.vertx-vertx-bridge-common-4.5.10.jar
+ - io.vertx-vertx-core-4.5.10.jar
+ - io.vertx-vertx-web-4.5.10.jar
+ - io.vertx-vertx-web-common-4.5.10.jar
* Apache ZooKeeper
- - org.apache.zookeeper-zookeeper-3.9.2.jar
- - org.apache.zookeeper-zookeeper-jute-3.9.2.jar
- - org.apache.zookeeper-zookeeper-prometheus-metrics-3.9.2.jar
+ - org.apache.zookeeper-zookeeper-3.9.3.jar
+ - org.apache.zookeeper-zookeeper-jute-3.9.3.jar
+ - org.apache.zookeeper-zookeeper-prometheus-metrics-3.9.3.jar
* Snappy Java
- org.xerial.snappy-snappy-java-1.1.10.5.jar
* Google HTTP Client
@@ -515,27 +516,27 @@ The Apache Software License, Version 2.0
* RoaringBitmap
- org.roaringbitmap-RoaringBitmap-1.2.0.jar
* OpenTelemetry
- - io.opentelemetry-opentelemetry-api-1.38.0.jar
- - io.opentelemetry-opentelemetry-api-incubator-1.38.0-alpha.jar
- - io.opentelemetry-opentelemetry-context-1.38.0.jar
- - io.opentelemetry-opentelemetry-exporter-common-1.38.0.jar
- - io.opentelemetry-opentelemetry-exporter-otlp-1.38.0.jar
- - io.opentelemetry-opentelemetry-exporter-otlp-common-1.38.0.jar
- - io.opentelemetry-opentelemetry-exporter-prometheus-1.38.0-alpha.jar
- - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-common-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-logs-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-metrics-1.38.0.jar
- - io.opentelemetry-opentelemetry-sdk-trace-1.38.0.jar
- - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.3.jar
- - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.3-alpha.jar
- - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.3-alpha.jar
- - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.3-alpha.jar
- - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.3-alpha.jar
- - io.opentelemetry.semconv-opentelemetry-semconv-1.25.0-alpha.jar
+ - io.opentelemetry-opentelemetry-api-1.45.0.jar
+ - io.opentelemetry-opentelemetry-api-incubator-1.45.0-alpha.jar
+ - io.opentelemetry-opentelemetry-context-1.45.0.jar
+ - io.opentelemetry-opentelemetry-exporter-common-1.45.0.jar
+ - io.opentelemetry-opentelemetry-exporter-otlp-1.45.0.jar
+ - io.opentelemetry-opentelemetry-exporter-otlp-common-1.45.0.jar
+ - io.opentelemetry-opentelemetry-exporter-prometheus-1.45.0-alpha.jar
+ - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.45.0.jar
+ - io.opentelemetry-opentelemetry-sdk-1.45.0.jar
+ - io.opentelemetry-opentelemetry-sdk-common-1.45.0.jar
+ - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.45.0.jar
+ - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.45.0.jar
+ - io.opentelemetry-opentelemetry-sdk-logs-1.45.0.jar
+ - io.opentelemetry-opentelemetry-sdk-metrics-1.45.0.jar
+ - io.opentelemetry-opentelemetry-sdk-trace-1.45.0.jar
+ - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.6.jar
+ - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.6-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.6-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.6-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.6-alpha.jar
+ - io.opentelemetry.semconv-opentelemetry-semconv-1.29.0-alpha.jar
* Spotify completable-futures
- com.spotify-completable-futures-0.3.6.jar
@@ -565,16 +566,14 @@ MIT License
- com.auth0-jwks-rsa-0.22.0.jar
Protocol Buffers License
* Protocol Buffers
- - com.google.protobuf-protobuf-java-3.22.3.jar -- ../licenses/LICENSE-protobuf.txt
- - com.google.protobuf-protobuf-java-util-3.22.3.jar -- ../licenses/LICENSE-protobuf.txt
+ - com.google.protobuf-protobuf-java-3.25.5.jar -- ../licenses/LICENSE-protobuf.txt
+ - com.google.protobuf-protobuf-java-util-3.25.5.jar -- ../licenses/LICENSE-protobuf.txt
CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
* Java Annotations API
- - com.sun.activation-javax.activation-1.2.0.jar
- - javax.xml.bind-jaxb-api-2.3.1.jar
+ - com.sun.activation-jakarta.activation-1.2.2.jar
* Java Servlet API -- javax.servlet-javax.servlet-api-3.1.0.jar
* WebSocket Server API -- javax.websocket-javax.websocket-client-api-1.0.jar
- * Java Web Service REST API -- javax.ws.rs-javax.ws.rs-api-2.1.jar
* HK2 - Dependency Injection Kernel
- org.glassfish.hk2-hk2-api-2.6.1.jar
- org.glassfish.hk2-hk2-locator-2.6.1.jar
diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml
index 45108aba68f87..bbe00fc82e041 100644
--- a/distribution/shell/pom.xml
+++ b/distribution/shell/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
distribution
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
pulsar-shell-distribution
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 9ab22ae83e42e..7d0b9adbd3bc4 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -340,42 +340,42 @@ The Apache Software License, Version 2.0
* Apache Commons
- commons-codec-1.15.jar
- commons-configuration-1.10.jar
- - commons-io-2.8.0.jar
+ - commons-io-2.18.0.jar
- commons-lang-2.6.jar
- commons-logging-1.2.jar
- commons-lang3-3.11.jar
- commons-text-1.10.0.jar
- commons-compress-1.26.0.jar
* Netty
- - netty-buffer-4.1.113.Final.jar
- - netty-codec-4.1.113.Final.jar
- - netty-codec-dns-4.1.113.Final.jar
- - netty-codec-http-4.1.113.Final.jar
- - netty-codec-socks-4.1.113.Final.jar
- - netty-codec-haproxy-4.1.113.Final.jar
- - netty-common-4.1.113.Final.jar
- - netty-handler-4.1.113.Final.jar
- - netty-handler-proxy-4.1.113.Final.jar
- - netty-resolver-4.1.113.Final.jar
- - netty-resolver-dns-4.1.113.Final.jar
- - netty-transport-4.1.113.Final.jar
- - netty-transport-classes-epoll-4.1.113.Final.jar
- - netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar
- - netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar
- - netty-transport-native-unix-common-4.1.113.Final.jar
- - netty-tcnative-boringssl-static-2.0.66.Final.jar
- - netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar
- - netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar
- - netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar
- - netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar
- - netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar
- - netty-tcnative-classes-2.0.66.Final.jar
- - netty-incubator-transport-classes-io_uring-0.0.24.Final.jar
- - netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar
- - netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar
- - netty-resolver-dns-classes-macos-4.1.113.Final.jar
- - netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar
- - netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar
+ - netty-buffer-4.1.118.Final.jar
+ - netty-codec-4.1.118.Final.jar
+ - netty-codec-dns-4.1.118.Final.jar
+ - netty-codec-http-4.1.118.Final.jar
+ - netty-codec-socks-4.1.118.Final.jar
+ - netty-codec-haproxy-4.1.118.Final.jar
+ - netty-common-4.1.118.Final.jar
+ - netty-handler-4.1.118.Final.jar
+ - netty-handler-proxy-4.1.118.Final.jar
+ - netty-resolver-4.1.118.Final.jar
+ - netty-resolver-dns-4.1.118.Final.jar
+ - netty-transport-4.1.118.Final.jar
+ - netty-transport-classes-epoll-4.1.118.Final.jar
+ - netty-transport-native-epoll-4.1.118.Final-linux-aarch_64.jar
+ - netty-transport-native-epoll-4.1.118.Final-linux-x86_64.jar
+ - netty-transport-native-unix-common-4.1.118.Final.jar
+ - netty-tcnative-boringssl-static-2.0.70.Final.jar
+ - netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar
+ - netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar
+ - netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar
+ - netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar
+ - netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar
+ - netty-tcnative-classes-2.0.70.Final.jar
+ - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar
+ - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar
+ - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
+ - netty-resolver-dns-classes-macos-4.1.118.Final.jar
+ - netty-resolver-dns-native-macos-4.1.118.Final-osx-aarch_64.jar
+ - netty-resolver-dns-native-macos-4.1.118.Final-osx-x86_64.jar
* Prometheus client
- simpleclient-0.16.0.jar
- simpleclient_log4j2-0.16.0.jar
@@ -388,9 +388,9 @@ The Apache Software License, Version 2.0
- log4j-slf4j2-impl-2.23.1.jar
- log4j-web-2.23.1.jar
* OpenTelemetry
- - opentelemetry-api-1.38.0.jar
- - opentelemetry-api-incubator-1.38.0-alpha.jar
- - opentelemetry-context-1.38.0.jar
+ - opentelemetry-api-1.45.0.jar
+ - opentelemetry-api-incubator-1.45.0-alpha.jar
+ - opentelemetry-context-1.45.0.jar
* BookKeeper
- bookkeeper-common-allocator-4.17.1.jar
@@ -399,25 +399,27 @@ The Apache Software License, Version 2.0
* AirCompressor
- aircompressor-0.27.jar
* AsyncHttpClient
- - async-http-client-2.12.1.jar
- - async-http-client-netty-utils-2.12.1.jar
+ - async-http-client-2.12.4.jar
+ - async-http-client-netty-utils-2.12.4.jar
* Jetty
- - jetty-client-9.4.54.v20240208.jar
- - jetty-http-9.4.54.v20240208.jar
- - jetty-io-9.4.54.v20240208.jar
- - jetty-util-9.4.54.v20240208.jar
- - javax-websocket-client-impl-9.4.54.v20240208.jar
- - websocket-api-9.4.54.v20240208.jar
- - websocket-client-9.4.54.v20240208.jar
- - websocket-common-9.4.54.v20240208.jar
+ - jetty-client-9.4.56.v20240826.jar
+ - jetty-http-9.4.56.v20240826.jar
+ - jetty-io-9.4.56.v20240826.jar
+ - jetty-util-9.4.56.v20240826.jar
+ - javax-websocket-client-impl-9.4.56.v20240826.jar
+ - websocket-api-9.4.56.v20240826.jar
+ - websocket-client-9.4.56.v20240826.jar
+ - websocket-common-9.4.56.v20240826.jar
* SnakeYaml -- snakeyaml-2.0.jar
* Google Error Prone Annotations - error_prone_annotations-2.24.0.jar
* Javassist -- javassist-3.25.0-GA.jar
* Apache Avro
- - avro-1.11.3.jar
- - avro-protobuf-1.11.3.jar
+ - avro-1.11.4.jar
+ - avro-protobuf-1.11.4.jar
* RE2j -- re2j-1.7.jar
* Spotify completable-futures -- completable-futures-0.3.6.jar
+ * RoaringBitmap -- RoaringBitmap-1.2.0.jar
+ * Fastutil -- fastutil-8.5.14.jar
BSD 3-clause "New" or "Revised" License
* JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt
@@ -429,17 +431,10 @@ MIT License
* The Checker Framework
- checker-qual-3.33.0.jar
-Protocol Buffers License
- * Protocol Buffers
- - protobuf-java-3.22.3.jar -- ../licenses/LICENSE-protobuf.txt
-
CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
* Java Annotations API
- - javax.annotation-api-1.3.2.jar
- - javax.activation-1.2.0.jar
- - jaxb-api-2.3.1.jar
+ - jakarta.activation-1.2.2.jar
* WebSocket Server API -- javax.websocket-client-api-1.0.jar
- * Java Web Service REST API -- javax.ws.rs-api-2.1.jar
* HK2 - Dependency Injection Kernel
- hk2-api-2.6.1.jar
- hk2-locator-2.6.1.jar
diff --git a/docker/glibc-package/Dockerfile b/docker/glibc-package/Dockerfile
deleted file mode 100644
index 016e5c622365f..0000000000000
--- a/docker/glibc-package/Dockerfile
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-ARG GLIBC_VERSION=2.38
-ARG ALPINE_VERSION=3.20
-
-FROM ubuntu:22.04 as build
-ARG GLIBC_VERSION
-
-RUN apt-get -q update \
- && apt-get -qy install \
- bison \
- build-essential \
- gawk \
- gettext \
- openssl \
- python3 \
- texinfo \
- wget
-
-# Build GLibc
-RUN wget -qO- https://ftpmirror.gnu.org/libc/glibc-${GLIBC_VERSION}.tar.gz | tar zxf -
-RUN mkdir /glibc-build
-WORKDIR /glibc-build
-RUN /glibc-${GLIBC_VERSION}/configure \
- --prefix=/usr/glibc-compat \
- --libdir=/usr/glibc-compat/lib \
- --libexecdir=/usr/glibc-compat/lib \
- --enable-multi-arch \
- --enable-stack-protector=strong
-RUN make -j$(nproc)
-RUN make install
-RUN tar --dereference --hard-dereference -zcf /glibc-bin.tar.gz /usr/glibc-compat
-
-
-################################################
-## Build the APK package
-FROM alpine:$ALPINE_VERSION as apk
-ARG GLIBC_VERSION
-
-RUN apk add abuild sudo build-base
-
-RUN mkdir /build
-WORKDIR build
-
-COPY --from=build /glibc-bin.tar.gz /build
-
-COPY ./scripts /build
-
-RUN echo "pkgver=\"${GLIBC_VERSION}\"" >> /build/APKBUILD
-RUN echo "sha512sums=\"$(sha512sum glibc-bin.tar.gz ld.so.conf)\"" >> /build/APKBUILD
-
-RUN abuild-keygen -a -i -n
-RUN abuild -F -c -r
-
-################################################
-## Last stage - Only leaves the packages
-FROM busybox
-ARG GLIBC_VERSION
-
-RUN mkdir -p /root/packages
-COPY --from=apk /root/packages/*/glibc-${GLIBC_VERSION}-r0.apk /root/packages
-COPY --from=apk /root/packages/*/glibc-bin-${GLIBC_VERSION}-r0.apk /root/packages
diff --git a/docker/glibc-package/scripts/APKBUILD b/docker/glibc-package/scripts/APKBUILD
deleted file mode 100644
index 0545508f0a7d4..0000000000000
--- a/docker/glibc-package/scripts/APKBUILD
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-pkgname="glibc"
-pkgrel="0"
-pkgdesc="GNU C Library compatibility layer"
-arch="all"
-url="https:/pulsar.apache.org"
-license="LGPL"
-options="!check"
-source="glibc-bin.tar.gz
-ld.so.conf"
-subpackages="${pkgname}-bin ${pkgname}-dev"
-triggers="glibc-bin.trigger=/lib:/usr/lib:/usr/glibc-compat/lib"
-depends="libuuid libgcc"
-
-package() {
- mkdir -p $pkgdir/lib $pkgdir/usr/glibc-compat/lib/locale $pkgdir/usr/glibc-compat/lib64 $pkgdir/etc $pkgdir/usr/glibc-compat/etc/
- cp -a $srcdir/usr $pkgdir
- cp $srcdir/ld.so.conf $pkgdir/usr/glibc-compat/etc/ld.so.conf
- cd $pkgdir/usr/glibc-compat
- rm -rf etc/rpc bin sbin lib/gconv lib/getconf lib/audit share var include
-
- FILENAME=$(ls $pkgdir/usr/glibc-compat/lib/ld-linux-*.so.*)
- LIBNAME=$(basename $FILENAME)
- ln -s /usr/glibc-compat/lib/$LIBNAME $pkgdir/lib/$LIBNAME
- ln -s /usr/glibc-compat/lib/$LIBNAME $pkgdir/usr/glibc-compat/lib64/$LIBNAME
- ln -s /usr/glibc-compat/etc/ld.so.cache $pkgdir/etc/ld.so.cache
-}
-
-bin() {
- depends="$pkgname libc6-compat"
- mkdir -p $subpkgdir/usr/glibc-compat
- cp -a $srcdir/usr/glibc-compat/bin $subpkgdir/usr/glibc-compat
- cp -a $srcdir/usr/glibc-compat/sbin $subpkgdir/usr/glibc-compat
-}
-
diff --git a/docker/glibc-package/scripts/glibc-bin.trigger b/docker/glibc-package/scripts/glibc-bin.trigger
deleted file mode 100755
index 5bae5d7ca2bda..0000000000000
--- a/docker/glibc-package/scripts/glibc-bin.trigger
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-/usr/glibc-compat/sbin/ldconfig
\ No newline at end of file
diff --git a/docker/glibc-package/scripts/ld.so.conf b/docker/glibc-package/scripts/ld.so.conf
deleted file mode 100644
index 6548b9300bb9c..0000000000000
--- a/docker/glibc-package/scripts/ld.so.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-/usr/local/lib
-/usr/glibc-compat/lib
-/usr/lib
-/lib
diff --git a/docker/kinesis-producer-alpine/Dockerfile b/docker/kinesis-producer-alpine/Dockerfile
new file mode 100644
index 0000000000000..ffdf44f55d083
--- /dev/null
+++ b/docker/kinesis-producer-alpine/Dockerfile
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+ARG ALPINE_VERSION=3.20
+
+# Builds an Alpine image with kinesis_producer compiled for Alpine Linux / musl
+
+# Build stage
+FROM alpine:$ALPINE_VERSION AS kinesis-producer-build
+ENV KINESIS_PRODUCER_LIB_VERSION=0.15.12
+
+# Install build dependencies
+RUN apk update && apk add --no-cache \
+ git \
+ binutils \
+ coreutils \
+ alpine-sdk \
+ util-linux \
+ cmake \
+ autoconf \
+ automake \
+ libtool \
+ curl \
+ bash \
+ tar \
+ libuuid \
+ linux-headers \
+ zlib \
+ zlib-dev \
+ perl \
+ wget \
+ boost-dev \
+ openssl-dev \
+ curl-dev \
+ build-base \
+ util-linux-dev \
+ g++ \
+ make \
+ upx
+
+ENV LANG=C.UTF-8
+
+RUN mkdir /build
+COPY kinesis_producer_alpine.patch /build/
+
+# Clone KPL and copy build script
+RUN cd /build && \
+ git clone --depth 1 --single-branch --branch v${KINESIS_PRODUCER_LIB_VERSION} https://github.com/awslabs/amazon-kinesis-producer && \
+ cd amazon-kinesis-producer && \
+ git apply ../kinesis_producer_alpine.patch
+
+# Copy and execute build script
+COPY build-alpine.sh /build/
+RUN chmod +x /build/build-alpine.sh
+RUN /build/build-alpine.sh
+
+# Final stage
+FROM alpine:$ALPINE_VERSION
+COPY --from=kinesis-producer-build /opt/amazon-kinesis-producer /opt/amazon-kinesis-producer
+RUN apk update && apk add --no-cache \
+ brotli-libs \
+ c-ares \
+ libcrypto3 \
+ libcurl \
+ libgcc \
+ libidn2 \
+ libpsl \
+ libssl3 \
+ libunistring \
+ nghttp2-libs \
+ zlib \
+ zstd-libs \
+ libuuid
+WORKDIR /opt/amazon-kinesis-producer/bin
diff --git a/docker/glibc-package/README.md b/docker/kinesis-producer-alpine/README.md
similarity index 54%
rename from docker/glibc-package/README.md
rename to docker/kinesis-producer-alpine/README.md
index ee1f643705ad2..4526f08c65ec1 100644
--- a/docker/glibc-package/README.md
+++ b/docker/kinesis-producer-alpine/README.md
@@ -19,21 +19,19 @@
-->
-# GLibc compatibility package
+# Alpine image with kinesis_producer compiled for Alpine Linux / musl
-This directory includes the Docker scripts to build an image with GLibc compiled for Alpine Linux.
+This directory includes the Docker scripts to build an image with `kinesis_producer` for Alpine Linux.
+`kinesis_producer` is a native executable that is required by [Amazon Kinesis Producer library (KPL)](https://github.com/awslabs/amazon-kinesis-producer) which is used by the Pulsar IO Kinesis Sink connector. The default `kinesis_producer` binary is compiled for glibc, and it does not work on Alpine Linux which uses musl.
-This is used to ensure plugins that are going to be used in the Pulsar image and that are depeding on GLibc, will
-still be working correctly in the Alpine Image. (eg: Netty Tc-Native and Kinesis Producer Library).
-
-This image only needs to be re-created when we want to upgrade to a newer version of GLibc.
+This image only needs to be re-created when we want to upgrade to a newer version of `kinesis_producer`.
# Steps
1. Change the version in the Dockerfile for this directory.
2. Rebuild the image and push it to Docker Hub:
```
-docker buildx build --platform=linux/amd64,linux/arm64 -t apachepulsar/glibc-base:2.38 . --push
+docker buildx build --platform=linux/amd64,linux/arm64 -t apachepulsar/pulsar-io-kinesis-sink-kinesis_producer:0.15.12 . --push
```
-The image tag is then used in `docker/pulsar/Dockerfile`.
+The image tag is then used in `docker/pulsar-all/Dockerfile`. The `kinesis_producer` binary is copied from the image to the `pulsar-all` image that is used by Pulsar Functions to run the Pulsar IO Kinesis Sink connector. The environment variable `PULSAR_IO_KINESIS_KPL_PATH` is set to `/opt/amazon-kinesis-producer/bin/kinesis_producer` and this is how the Kinesis Sink connector knows where to find the `kinesis_producer` binary.
\ No newline at end of file
diff --git a/docker/kinesis-producer-alpine/build-alpine.sh b/docker/kinesis-producer-alpine/build-alpine.sh
new file mode 100644
index 0000000000000..23718450bbc83
--- /dev/null
+++ b/docker/kinesis-producer-alpine/build-alpine.sh
@@ -0,0 +1,139 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+set -x
+
+INSTALL_DIR=/build/third_party
+AWS_SDK_CPP_VERSION="1.11.420"
+PROTOBUF_VERSION="3.11.4"
+BOOST_VERSION="1.76.0"
+BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}"
+
+# Create install directory
+mkdir -p $INSTALL_DIR
+
+# Setup environment variables
+export CC="gcc"
+export CXX="g++"
+export CXXFLAGS="-I$INSTALL_DIR/include -O3 -Wno-implicit-fallthrough -Wno-int-in-bool-context"
+export LDFLAGS="-L$INSTALL_DIR/lib"
+export LD_LIBRARY_PATH="$INSTALL_DIR/lib:$LD_LIBRARY_PATH"
+
+cd $INSTALL_DIR
+
+# Build protobuf
+if [ ! -d "protobuf-${PROTOBUF_VERSION}" ]; then
+ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz
+ tar xf protobuf-all-${PROTOBUF_VERSION}.tar.gz
+ rm protobuf-all-${PROTOBUF_VERSION}.tar.gz
+
+ cd protobuf-${PROTOBUF_VERSION}
+ ./configure --prefix=${INSTALL_DIR} \
+ --disable-shared \
+ CFLAGS="-fPIC" \
+ CXXFLAGS="-fPIC ${CXXFLAGS}" \
+ --with-pic
+ make -j4
+ make install
+ cd ..
+fi
+
+# Build Boost
+if [ ! -d "boost_${BOOST_VERSION_UNDERSCORED}" ]; then
+ curl -LO https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORED}.tar.gz
+ tar xf boost_${BOOST_VERSION_UNDERSCORED}.tar.gz
+ rm boost_${BOOST_VERSION_UNDERSCORED}.tar.gz
+
+ cd boost_${BOOST_VERSION_UNDERSCORED}
+
+ BOOST_LIBS="regex,thread,log,system,random,filesystem,chrono,atomic,date_time,program_options,test"
+
+ ./bootstrap.sh --with-libraries=$BOOST_LIBS --with-toolset=gcc
+
+ ./b2 \
+ -j4 \
+ variant=release \
+ link=static \
+ threading=multi \
+ runtime-link=static \
+ --prefix=${INSTALL_DIR} \
+ cxxflags="-fPIC ${CXXFLAGS}" \
+ install
+
+ cd ..
+fi
+
+# Download and build AWS SDK
+if [ ! -d "aws-sdk-cpp" ]; then
+ git clone --depth 1 --branch ${AWS_SDK_CPP_VERSION} https://github.com/awslabs/aws-sdk-cpp.git aws-sdk-cpp
+ pushd aws-sdk-cpp
+ git config submodule.fetchJobs 8
+ git submodule update --init --depth 1 --recursive
+ popd
+
+ rm -rf aws-sdk-cpp-build
+ mkdir aws-sdk-cpp-build
+ cd aws-sdk-cpp-build
+
+ cmake \
+ -DBUILD_ONLY="kinesis;monitoring;sts" \
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+ -DSTATIC_LINKING=1 \
+ -DCMAKE_PREFIX_PATH="$INSTALL_DIR" \
+ -DCMAKE_C_COMPILER="$CC" \
+ -DCMAKE_CXX_COMPILER="$CXX" \
+ -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
+ -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
+ -DCMAKE_FIND_FRAMEWORK=LAST \
+ -DENABLE_TESTING="OFF" \
+ ../aws-sdk-cpp
+ make -j4
+ make install
+ cd ..
+fi
+
+# Build the native kinesis producer
+cd /build/amazon-kinesis-producer
+ln -fs ../third_party
+cmake -DCMAKE_PREFIX_PATH="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=RelWithDebInfo .
+make -j4
+
+FINAL_DIR=/opt/amazon-kinesis-producer
+# copy the binary
+mkdir -p $FINAL_DIR/bin
+cp kinesis_producer $FINAL_DIR/bin/kinesis_producer.original
+
+# capture version information
+git describe --long --tags > $FINAL_DIR/bin/.version
+git rev-parse HEAD > $FINAL_DIR/bin/.revision
+uname -a > $FINAL_DIR/bin/.system_info
+cat /etc/os-release > $FINAL_DIR/bin/.os_info
+date > $FINAL_DIR/bin/.build_time
+
+# copy tests
+mkdir -p $FINAL_DIR/tests
+cp tests $FINAL_DIR/tests/
+cp test_driver $FINAL_DIR/tests/
+
+# Strip and compress the binary
+cd $FINAL_DIR/bin
+strip -o kinesis_producer.stripped kinesis_producer.original
+upx --best -o kinesis_producer kinesis_producer.stripped
\ No newline at end of file
diff --git a/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch b/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch
new file mode 100644
index 0000000000000..d1ddd6a85501c
--- /dev/null
+++ b/docker/kinesis-producer-alpine/kinesis_producer_alpine.patch
@@ -0,0 +1,127 @@
+From 96ba2eb7145363586529e6c770dcc0920bf04ac2 Mon Sep 17 00:00:00 2001
+From: Lari Hotari
+Date: Wed, 18 Dec 2024 17:16:02 +0200
+Subject: [PATCH] Adapt build for Alpine, fix issue with NULL_BACKTRACE support
+
+- also use dynamic linking to some libraries (zlib, openssl, libz, libcurl, libcrypto)
+ to reduce binary size
+---
+ CMakeLists.txt | 20 +++++++++++---------
+ aws/utils/backtrace/bsd_backtrace.cc | 2 +-
+ aws/utils/backtrace/gcc_backtrace.cc | 3 +--
+ aws/utils/backtrace/null_backtrace.cc | 5 ++---
+ aws/utils/signal_handler.cc | 1 -
+ 5 files changed, 15 insertions(+), 16 deletions(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 2dd7084..2ba47e6 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -213,22 +213,24 @@ set(STATIC_LIBS
+ boost_chrono)
+
+ find_package(Threads)
+-find_package(ZLIB)
++find_package(ZLIB REQUIRED)
+ find_package(AWSSDK REQUIRED COMPONENTS kinesis monitoring sts)
++find_package(OpenSSL REQUIRED)
++find_package(CURL REQUIRED)
+
+-add_library(LibCrypto STATIC IMPORTED)
+-set_property(TARGET LibCrypto PROPERTY IMPORTED_LOCATION ${THIRD_PARTY_LIB_DIR}/libcrypto.a)
++add_library(LibCrypto SHARED IMPORTED)
++set_property(TARGET LibCrypto PROPERTY IMPORTED_LOCATION ${OPENSSL_CRYPTO_LIBRARY})
+ set_property(TARGET LibCrypto PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES ${LIBDL_LIBRARIES})
+
+-add_library(LibSsl STATIC IMPORTED)
+-set_property(TARGET LibSsl PROPERTY IMPORTED_LOCATION ${THIRD_PARTY_LIB_DIR}/libssl.a)
++add_library(LibSsl SHARED IMPORTED)
++set_property(TARGET LibSsl PROPERTY IMPORTED_LOCATION ${OPENSSL_SSL_LIBRARY})
+ set_property(TARGET LibSsl PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES LibCrypto)
+
+-add_library(LibZ STATIC IMPORTED)
+-set_property(TARGET LibZ PROPERTY IMPORTED_LOCATION ${THIRD_PARTY_LIB_DIR}/libz.a)
++add_library(LibZ SHARED IMPORTED)
++set_property(TARGET LibZ PROPERTY IMPORTED_LOCATION ${ZLIB_LIBRARIES})
+
+-add_library(LibCurl STATIC IMPORTED)
+-set_property(TARGET LibCurl PROPERTY IMPORTED_LOCATION ${THIRD_PARTY_LIB_DIR}/libcurl.a)
++add_library(LibCurl SHARED IMPORTED)
++set_property(TARGET LibCurl PROPERTY IMPORTED_LOCATION ${CURL_LIBRARIES})
+ set_property(TARGET LibCurl PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES ${CMAKE_THREAD_LIBS_INIT} ${LIBRT_LIBRARIES} ${LIBDL_LIBRARIES} LibSsl LibZ)
+
+ add_library(LibProto STATIC IMPORTED)
+diff --git a/aws/utils/backtrace/bsd_backtrace.cc b/aws/utils/backtrace/bsd_backtrace.cc
+index fb5dbe3..cd6c5fe 100644
+--- a/aws/utils/backtrace/bsd_backtrace.cc
++++ b/aws/utils/backtrace/bsd_backtrace.cc
+@@ -15,10 +15,10 @@
+
+ #include "backtrace.h"
+ #include
+-#include
+ #include
+
+ #ifdef BSD_BACKTRACE
++#include
+ namespace aws {
+ namespace utils {
+ namespace backtrace {
+diff --git a/aws/utils/backtrace/gcc_backtrace.cc b/aws/utils/backtrace/gcc_backtrace.cc
+index 446ede9..32a866d 100644
+--- a/aws/utils/backtrace/gcc_backtrace.cc
++++ b/aws/utils/backtrace/gcc_backtrace.cc
+@@ -15,7 +15,6 @@
+
+ #include "backtrace.h"
+ #include
+-#include
+ #include
+ #include
+ #include
+@@ -23,7 +22,7 @@
+ #include
+
+ #ifdef LIB_BACKTRACE
+-
++#include
+ #include
+
+ namespace {
+diff --git a/aws/utils/backtrace/null_backtrace.cc b/aws/utils/backtrace/null_backtrace.cc
+index 69d57f9..d443eae 100644
+--- a/aws/utils/backtrace/null_backtrace.cc
++++ b/aws/utils/backtrace/null_backtrace.cc
+@@ -15,10 +15,9 @@
+
+ #include "backtrace.h"
+ #include
+-#include
+ #include
+
+-#ifdef NULL_STACKTRACE
++#ifdef NULL_BACKTRACE
+
+ namespace aws {
+ namespace utils {
+@@ -36,4 +35,4 @@ void stack_trace_for_signal(int skip, bool /*signaled*/) {
+ }
+ }
+
+-#endif // NULL_STACKTRACE
++#endif // NULL_BACKTRACE
+diff --git a/aws/utils/signal_handler.cc b/aws/utils/signal_handler.cc
+index b58ab0e..f483c77 100644
+--- a/aws/utils/signal_handler.cc
++++ b/aws/utils/signal_handler.cc
+@@ -19,7 +19,6 @@
+ #include "backtrace/backtrace.h"
+
+ #include
+-#include
+ #include
+ #include
+ #include
+--
+2.47.1
+
diff --git a/docker/pom.xml b/docker/pom.xml
index a5ea238241c6a..ceeb34a97bea3 100644
--- a/docker/pom.xml
+++ b/docker/pom.xml
@@ -26,7 +26,7 @@
org.apache.pulsar
pulsar
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
docker-images
Apache Pulsar :: Docker Images
@@ -68,7 +68,6 @@
false
true
- true
false
diff --git a/docker/pulsar-all/Dockerfile b/docker/pulsar-all/Dockerfile
index 81ad74b65000f..59ab86e45569e 100644
--- a/docker/pulsar-all/Dockerfile
+++ b/docker/pulsar-all/Dockerfile
@@ -17,16 +17,42 @@
# under the License.
#
+# global arguments (only to be used in FROM clauses)
ARG PULSAR_IMAGE
-FROM busybox as pulsar-all
+ARG PULSAR_IO_KINESIS_KPL_IMAGE
-ARG PULSAR_IO_DIR
-ARG PULSAR_OFFLOADER_TARBALL
+FROM busybox AS pulsar-extensions
+ARG PULSAR_IO_DIR
ADD ${PULSAR_IO_DIR} /connectors
+ARG PULSAR_OFFLOADER_TARBALL
ADD ${PULSAR_OFFLOADER_TARBALL} /
RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders
-FROM $PULSAR_IMAGE
-COPY --from=pulsar-all /connectors /pulsar/connectors
-COPY --from=pulsar-all /offloaders /pulsar/offloaders
+FROM ${PULSAR_IO_KINESIS_KPL_IMAGE} AS pulsar-io-kinesis-sink-kinesis_producer
+
+FROM ${PULSAR_IMAGE}
+COPY --from=pulsar-extensions /connectors /pulsar/connectors
+COPY --from=pulsar-extensions /offloaders /pulsar/offloaders
+# Copy the kinesis_producer native executable compiled for Alpine musl to the pulsar-all image
+# This is required to support the Pulsar IO Kinesis sink connector
+COPY --from=pulsar-io-kinesis-sink-kinesis_producer /opt/amazon-kinesis-producer/bin/kinesis_producer /opt/amazon-kinesis-producer/bin/.os_info /opt/amazon-kinesis-producer/bin/.build_time /opt/amazon-kinesis-producer/bin/.revision /opt/amazon-kinesis-producer/bin/.system_info /opt/amazon-kinesis-producer/bin/.version /opt/amazon-kinesis-producer/bin/
+# Set the environment variable to point to the kinesis_producer native executable
+ENV PULSAR_IO_KINESIS_KPL_PATH=/opt/amazon-kinesis-producer/bin/kinesis_producer
+# Install the required dependencies for the kinesis_producer native executable
+USER 0
+RUN apk update && apk add --no-cache \
+ brotli-libs \
+ c-ares \
+ libcrypto3 \
+ libcurl \
+ libgcc \
+ libidn2 \
+ libpsl \
+ libssl3 \
+ libunistring \
+ nghttp2-libs \
+ zlib \
+ zstd-libs \
+ libuuid
+USER 10000
\ No newline at end of file
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
index b43121dd0f613..5281e2ab38db2 100644
--- a/docker/pulsar-all/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -23,7 +23,7 @@
org.apache.pulsar
docker-images
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
4.0.0
pulsar-all-docker-image
@@ -80,6 +80,10 @@
docker
+
+
+ apachepulsar/pulsar-io-kinesis-sink-kinesis_producer:0.15.12
+
${project.groupId}
@@ -161,6 +165,7 @@
target/apache-pulsar-io-connectors-${project.version}-bin
target/pulsar-offloader-distribution-${project.version}-bin.tar.gz
${docker.organization}/${docker.image}:${project.version}-${git.commit.id.abbrev}
+ ${PULSAR_IO_KINESIS_KPL_IMAGE}
diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
index 81446ae5ee5ce..aec8355963958 100644
--- a/docker/pulsar/Dockerfile
+++ b/docker/pulsar/Dockerfile
@@ -17,7 +17,8 @@
# under the License.
#
-ARG ALPINE_VERSION=3.20
+ARG ALPINE_VERSION=3.21
+ARG IMAGE_JDK_MAJOR_VERSION=21
# First create a stage with just the Pulsar tarball and scripts
FROM alpine:$ALPINE_VERSION as pulsar
@@ -54,7 +55,7 @@ RUN chmod -R o+rx /pulsar
RUN echo 'OPTS="$OPTS -Dorg.xerial.snappy.use.systemlib=true"' >> /pulsar/conf/bkenv.sh
### Create one stage to include JVM distribution
-FROM amazoncorretto:21-alpine AS jvm
+FROM amazoncorretto:${IMAGE_JDK_MAJOR_VERSION}-alpine${ALPINE_VERSION} AS jvm
RUN apk add --no-cache binutils
@@ -74,7 +75,7 @@ ARG SNAPPY_VERSION
RUN apk add git alpine-sdk util-linux cmake autoconf automake libtool openjdk17 maven curl bash tar
ENV JAVA_HOME=/usr
RUN curl -Ls https://github.com/xerial/snappy-java/archive/refs/tags/v$SNAPPY_VERSION.tar.gz | tar zxf - && cd snappy-java-$SNAPPY_VERSION && make clean-native native
-FROM apachepulsar/glibc-base:2.38 as glibc
+
## Create final stage from Alpine image
## and add OpenJDK and Python dependencies (for Pulsar functions)
@@ -89,10 +90,15 @@ RUN apk add --no-cache \
py3-grpcio \
py3-yaml \
gcompat \
+ libgcc \
+ libstdc++ \
+ libuuid \
ca-certificates \
procps \
curl \
- bind-tools
+ bind-tools \
+ openssl \
+ coreutils
# Upgrade all packages to get latest versions with security fixes
RUN apk upgrade --no-cache
@@ -121,10 +127,6 @@ fastavro>=1.9.2\n\
RUN pip3 install --break-system-packages --no-cache-dir --only-binary grpcio -r /requirements.txt
RUN rm /requirements.txt
-# Install GLibc compatibility library
-COPY --from=glibc /root/packages /root/packages
-RUN apk add --allow-untrusted --force-overwrite /root/packages/glibc-*.apk
-
COPY --from=jvm /opt/jvm /opt/jvm
ENV JAVA_HOME=/opt/jvm
@@ -133,12 +135,14 @@ COPY --from=snappy-java /tmp/libsnappyjava.so /usr/lib/libsnappyjava.so
# The default is /pulsat/bin and cannot be written.
ENV PULSAR_PID_DIR=/pulsar/logs
-ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE
-
COPY --from=pulsar /pulsar /pulsar
WORKDIR /pulsar
ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin
+# Use musl libc library for RocksDB
+ENV ROCKSDB_MUSL_LIBC=true
+# Preload gcompat library for glibc compatibility with Netty native libraries
+ENV LD_PRELOAD=/lib/libgcompat.so.0
# The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value.
ARG DEFAULT_USERNAME=pulsar
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 68d82ae552825..edb6346957d9e 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -23,7 +23,7 @@
org.apache.pulsar
docker-images
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
4.0.0
pulsar-docker-image
@@ -83,6 +83,7 @@
target/pulsar-server-distribution-${project.version}-bin.tar.gz
${pulsar.client.python.version}
${snappy.version}
+ ${IMAGE_JDK_MAJOR_VERSION}
${project.basedir}
diff --git a/grafana/dashboards/offloader.json b/grafana/dashboards/offloader.json
new file mode 100644
index 0000000000000..12cf21cfcc3b6
--- /dev/null
+++ b/grafana/dashboards/offloader.json
@@ -0,0 +1,599 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 2,
+ "iteration": 1644809446689,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "id": 14,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_writeError{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_writeError",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "id": 18,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.3",
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_readLedgerLatencyBuckets{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_readLedgerLatencyBuckets",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "id": 6,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_readOffloadError{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_readOffloadError",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "id": 16,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_writeRate{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_writeRate",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 16
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_offloadError{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_offloadError",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 16
+ },
+ "id": 8,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "exemplar": true,
+ "expr": "pulsar_ledgeroffloader_readOffloadRate{cluster=~\"$cluster\",namespace=~\"$namespace\",job=~\"$job\"}",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "pulsar_ledgeroffloader_readOffloadRate",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 34,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "standalone",
+ "value": "standalone"
+ },
+ "definition": "{cluster=~\".+\"}",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Cluster",
+ "multi": false,
+ "name": "cluster",
+ "options": [],
+ "query": {
+ "query": "{cluster=~\".+\"}",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "/.*[^_]cluster=\\\"([^\\\"]+)\\\".*/",
+ "skipUrlSync": false,
+ "sort": 1,
+ "type": "query"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "public/functions",
+ "value": "public/functions"
+ },
+ "definition": "pulsar_rate_in{namespace=~\".+\", cluster=~\"$cluster\"}",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Namespace",
+ "multi": false,
+ "name": "namespace",
+ "options": [],
+ "query": {
+ "query": "pulsar_rate_in{namespace=~\".+\", cluster=~\"$cluster\"}",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "/.*namespace=\\\"([^\\\"]+)\\\".*/",
+ "skipUrlSync": false,
+ "sort": 0,
+ "type": "query"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "Pulsar",
+ "value": "Pulsar"
+ },
+ "definition": "pulsar_storage_size{job=~\".+\"}",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Job",
+ "multi": false,
+ "name": "job",
+ "options": [],
+ "query": {
+ "query": "pulsar_storage_size{job=~\".+\"}",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 1,
+ "regex": "/.*[^_]job=\\\"([^\\\"]+)\\\".*/",
+ "skipUrlSync": false,
+ "sort": 1,
+ "type": "query"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Pulsar Offloader",
+ "uid": "_qAaz517k",
+ "version": 25,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml
index dd19faad904bc..b8451cd626d0a 100644
--- a/jclouds-shaded/pom.xml
+++ b/jclouds-shaded/pom.xml
@@ -26,7 +26,7 @@
org.apache.pulsar
pulsar
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
jclouds-shaded
diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml
index a0885f8509547..2a5536987cd42 100644
--- a/jetcd-core-shaded/pom.xml
+++ b/jetcd-core-shaded/pom.xml
@@ -26,7 +26,7 @@
org.apache.pulsar
pulsar
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
jetcd-core-shaded
@@ -45,6 +45,10 @@
io.netty
*
+
+ javax.annotation
+ javax.annotation-api
+
@@ -74,6 +78,7 @@
+ ${project.artifactId}-${project.version}
org.apache.maven.plugins
@@ -93,6 +98,10 @@
io.etcd:*
io.vertx:*
+
+
+ org.apache.pulsar:jetcd-core-shaded
+
@@ -141,54 +150,6 @@
${project.basedir}/dependency-reduced-pom.xml
-
- true
- shaded
-
-
-
-
-
-
- org.codehaus.mojo
- build-helper-maven-plugin
-
-
- attach-shaded-jar
- package
-
- attach-artifact
-
-
-
-
- ${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar
- jar
- shaded
-
-
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-antrun-plugin
- ${maven-antrun-plugin.version}
-
-
- unpack-shaded-jar
- package
-
- run
-
-
-
-
-
diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
index 22b093f7aafd7..2b83fcb49177b 100644
--- a/managed-ledger/pom.xml
+++ b/managed-ledger/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
pulsar
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
managed-ledger
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
index 042e03998696c..4e5e12365480c 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
@@ -660,6 +660,31 @@ void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate
void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition,
FindEntryCallback callback, Object ctx, boolean isFindFromLedger);
+
+ /**
+ * Find the newest entry that matches the given predicate.
+ *
+ * @param constraint
+ * search only active entries or all entries
+ * @param condition
+ * predicate that reads an entry an applies a condition
+ * @param callback
+ * callback object returning the resultant position
+ * @param startPosition
+ * start position to search from.
+ * @param endPosition
+ * end position to search to.
+ * @param ctx
+ * opaque context
+ * @param isFindFromLedger
+ * find the newest entry from ledger
+ */
+ default void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition,
+ Position startPosition, Position endPosition, FindEntryCallback callback,
+ Object ctx, boolean isFindFromLedger) {
+ asyncFindNewestMatching(constraint, condition, callback, ctx, isFindFromLedger);
+ }
+
/**
* reset the cursor to specified position to enable replay of messages.
*
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
index 03439f93ccad8..6fc39170e851a 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
@@ -64,6 +64,7 @@ public class ManagedLedgerConfig {
private long retentionTimeMs = 0;
private long retentionSizeInMB = 0;
private boolean autoSkipNonRecoverableData;
+ private boolean ledgerForceRecovery;
private boolean lazyCursorRecovery = false;
private long metadataOperationsTimeoutSeconds = 60;
private long readEntryTimeoutSeconds = 120;
@@ -86,10 +87,14 @@ public class ManagedLedgerConfig {
private int minimumBacklogEntriesForCaching = 1000;
private int maxBacklogBetweenCursorsForCaching = 1000;
private boolean triggerOffloadOnTopicLoad = false;
-
+ @Getter
+ @Setter
+ private String storageClassName;
@Getter
@Setter
private String shadowSourceName;
+ @Getter
+ private boolean persistIndividualAckAsLongArray;
public boolean isCreateIfMissing() {
return createIfMissing;
@@ -100,6 +105,11 @@ public ManagedLedgerConfig setCreateIfMissing(boolean createIfMissing) {
return this;
}
+ public ManagedLedgerConfig setPersistIndividualAckAsLongArray(boolean persistIndividualAckAsLongArray) {
+ this.persistIndividualAckAsLongArray = persistIndividualAckAsLongArray;
+ return this;
+ }
+
/**
* @return the lazyCursorRecovery
*/
@@ -465,6 +475,17 @@ public void setAutoSkipNonRecoverableData(boolean skipNonRecoverableData) {
this.autoSkipNonRecoverableData = skipNonRecoverableData;
}
+ /**
+ * Skip managed ledger failure to recover managed ledger forcefully.
+ */
+ public boolean isLedgerForceRecovery() {
+ return ledgerForceRecovery;
+ }
+
+ public void setLedgerForceRecovery(boolean ledgerForceRecovery) {
+ this.ledgerForceRecovery = ledgerForceRecovery;
+ }
+
/**
* @return max unacked message ranges that will be persisted and recovered.
*
@@ -505,8 +526,10 @@ public int getMaxUnackedRangesToPersistInMetadataStore() {
return maxUnackedRangesToPersistInMetadataStore;
}
- public void setMaxUnackedRangesToPersistInMetadataStore(int maxUnackedRangesToPersistInMetadataStore) {
+ public ManagedLedgerConfig setMaxUnackedRangesToPersistInMetadataStore(
+ int maxUnackedRangesToPersistInMetadataStore) {
this.maxUnackedRangesToPersistInMetadataStore = maxUnackedRangesToPersistInMetadataStore;
+ return this;
}
/**
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
index 386310b3ccbae..af538262ed44a 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryConfig.java
@@ -61,6 +61,18 @@ public class ManagedLedgerFactoryConfig {
*/
private long managedLedgerMaxReadsInFlightSize = 0;
+ /**
+ * Maximum time to wait for acquiring permits for max reads in flight when managedLedgerMaxReadsInFlightSizeInMB is
+ * set (>0) and the limit is reached.
+ */
+ private long managedLedgerMaxReadsInFlightPermitsAcquireTimeoutMillis = 60000;
+
+ /**
+ * Maximum number of reads that can be queued for acquiring permits for max reads in flight when
+ * managedLedgerMaxReadsInFlightSizeInMB is set (>0) and the limit is reached.
+ */
+ private int managedLedgerMaxReadsInFlightPermitsAcquireQueueSize = 10000;
+
/**
* Whether trace managed ledger task execution time.
*/
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java
index 363336e83113e..6e9bf78b14cfe 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java
@@ -18,6 +18,8 @@
*/
package org.apache.bookkeeper.mledger.impl;
+import javax.annotation.Nullable;
+
/**
* Interface to manage the ackSet state attached to a position.
* Helpers in {@link AckSetStateUtil} to create positions with
@@ -28,7 +30,7 @@ public interface AckSetState {
* Get the ackSet bitset information encoded as a long array.
* @return the ackSet
*/
- long[] getAckSet();
+ @Nullable long[] getAckSet();
/**
* Set the ackSet bitset information as a long array.
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
index 4ac409a2e9bfe..4107949de51b9 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerMetadataUtils.java
@@ -80,7 +80,7 @@ static Map buildAdditionalMetadataForCursor(String name) {
* Build additional metadata for a CompactedLedger.
*
* @param compactedTopic reference to the compacted topic.
- * @param compactedToMessageId last mesasgeId.
+ * @param compactedToMessageId last messageId.
* @return an immutable map which describes the compacted ledger
*/
public static Map buildMetadataForCompactedLedger(String compactedTopic,
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index e27814eadd0b5..203d48933f0a5 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -35,13 +35,14 @@
import java.time.Clock;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -59,7 +60,10 @@
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
import java.util.function.Predicate;
+import java.util.stream.Collectors;
import java.util.stream.LongStream;
+import javax.annotation.Nullable;
+import lombok.Getter;
import org.apache.bookkeeper.client.AsyncCallback.CloseCallback;
import org.apache.bookkeeper.client.AsyncCallback.OpenCallback;
import org.apache.bookkeeper.client.BKException;
@@ -91,17 +95,19 @@
import org.apache.bookkeeper.mledger.ScanOutcome;
import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongListMap;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo.Builder;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.StringProperty;
+import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats;
import org.apache.pulsar.common.util.DateFormatter;
import org.apache.pulsar.common.util.FutureUtil;
-import org.apache.pulsar.common.util.collections.BitSetRecyclable;
import org.apache.pulsar.common.util.collections.LongPairRangeSet;
import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer;
import org.apache.pulsar.common.util.collections.LongPairRangeSet.RangeBoundConsumer;
@@ -174,10 +180,12 @@ public class ManagedCursorImpl implements ManagedCursor {
protected volatile long messagesConsumedCounter;
// Current ledger used to append the mark-delete position
- private volatile LedgerHandle cursorLedger;
+ @VisibleForTesting
+ volatile LedgerHandle cursorLedger;
// Wether the current cursorLedger is read-only or writable
private boolean isCursorLedgerReadOnly = true;
+ private boolean ledgerForceRecovery;
// Stat of the cursor z-node
// NOTE: Don't update cursorLedgerStat alone,
@@ -195,7 +203,9 @@ public class ManagedCursorImpl implements ManagedCursor {
// Maintain the deletion status for batch messages
// (ledgerId, entryId) -> deletion indexes
- protected final ConcurrentSkipListMap batchDeletedIndexes;
+ @Getter
+ @VisibleForTesting
+ @Nullable protected final ConcurrentSkipListMap batchDeletedIndexes;
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private RateLimiter markDeleteLimiter;
@@ -328,6 +338,7 @@ public interface VoidCallback {
markDeleteLimiter = null;
}
this.mbean = new ManagedCursorMXBeanImpl(this);
+ this.ledgerForceRecovery = getConfig().isLedgerForceRecovery();
}
private void updateCursorLedgerStat(ManagedCursorInfo cursorInfo, Stat stat) {
@@ -543,10 +554,10 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
if (log.isInfoEnabled()) {
log.info("[{}] Opened ledger {} for cursor {}. rc={}", ledger.getName(), ledgerId, name, rc);
}
- if (isBkErrorNotRecoverable(rc)) {
+ if (isBkErrorNotRecoverable(rc) || (rc != BKException.Code.OK && ledgerForceRecovery)) {
log.error("[{}] Error opening metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name,
BKException.getMessage(rc));
- // Rewind to oldest entry available
+ // Rewind to the oldest entry available
initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback);
return;
} else if (rc != BKException.Code.OK) {
@@ -571,10 +582,10 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
if (log.isDebugEnabled()) {
log.debug("[{}} readComplete rc={} entryId={}", ledger.getName(), rc1, lh1.getLastAddConfirmed());
}
- if (isBkErrorNotRecoverable(rc1)) {
+ if (isBkErrorNotRecoverable(rc1) || (rc1 != BKException.Code.OK && ledgerForceRecovery)) {
log.error("[{}] Error reading from metadata ledger {} for cursor {}: {}", ledger.getName(),
ledgerId, name, BKException.getMessage(rc1));
- // Rewind to oldest entry available
+ // Rewind to the oldest entry available
initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback);
return;
} else if (rc1 != BKException.Code.OK) {
@@ -606,9 +617,7 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
}
Position position = PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId());
- if (positionInfo.getIndividualDeletedMessagesCount() > 0) {
- recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList());
- }
+ recoverIndividualDeletedMessages(positionInfo);
if (getConfig().isDeletionAtBatchIndexLevelEnabled()
&& positionInfo.getBatchedEntryDeletionIndexInfoCount() > 0) {
recoverBatchDeletedIndexes(positionInfo.getBatchedEntryDeletionIndexInfoList());
@@ -627,6 +636,60 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
}
}
+ public void recoverIndividualDeletedMessages(PositionInfo positionInfo) {
+ if (positionInfo.getIndividualDeletedMessagesCount() > 0) {
+ recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList());
+ } else if (positionInfo.getIndividualDeletedMessageRangesCount() > 0) {
+ List rangeList = positionInfo.getIndividualDeletedMessageRangesList();
+ try {
+ Map rangeMap = rangeList.stream().collect(Collectors.toMap(LongListMap::getKey,
+ list -> list.getValuesList().stream().mapToLong(i -> i).toArray()));
+ // Guarantee compatability for the config "unackedRangesOpenCacheSetEnabled".
+ if (getConfig().isUnackedRangesOpenCacheSetEnabled()) {
+ individualDeletedMessages.build(rangeMap);
+ } else {
+ RangeSetWrapper rangeSetWrapperV2 = new RangeSetWrapper<>(positionRangeConverter,
+ positionRangeReverseConverter, true,
+ getConfig().isPersistentUnackedRangesWithMultipleEntriesEnabled());
+ rangeSetWrapperV2.build(rangeMap);
+ rangeSetWrapperV2.forEach(range -> {
+ individualDeletedMessages.addOpenClosed(range.lowerEndpoint().getLedgerId(),
+ range.lowerEndpoint().getEntryId(), range.upperEndpoint().getLedgerId(),
+ range.upperEndpoint().getEntryId());
+ return true;
+ });
+ rangeSetWrapperV2.clear();
+ }
+ } catch (Exception e) {
+ log.warn("[{}]-{} Failed to recover individualDeletedMessages from serialized data", ledger.getName(),
+ name, e);
+ }
+ }
+ }
+
+ private List buildLongPropertiesMap(Map properties) {
+ if (properties.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List longListMap = new ArrayList<>();
+ MutableInt serializedSize = new MutableInt();
+ properties.forEach((id, ranges) -> {
+ if (ranges == null || ranges.length <= 0) {
+ return;
+ }
+ org.apache.bookkeeper.mledger.proto.MLDataFormats.LongListMap.Builder lmBuilder = LongListMap.newBuilder()
+ .setKey(id);
+ for (long range : ranges) {
+ lmBuilder.addValues(range);
+ }
+ LongListMap lm = lmBuilder.build();
+ longListMap.add(lm);
+ serializedSize.add(lm.getSerializedSize());
+ });
+ individualDeletedMessagesSerializedSize = serializedSize.toInteger();
+ return longListMap;
+ }
+
private void recoverIndividualDeletedMessages(List individualDeletedMessagesList) {
lock.writeLock().lock();
try {
@@ -666,6 +729,7 @@ private void recoverIndividualDeletedMessages(List i
private void recoverBatchDeletedIndexes (
List batchDeletedIndexInfoList) {
+ Objects.requireNonNull(batchDeletedIndexes);
lock.writeLock().lock();
try {
this.batchDeletedIndexes.clear();
@@ -677,8 +741,7 @@ private void recoverBatchDeletedIndexes (
}
this.batchDeletedIndexes.put(
PositionFactory.create(batchDeletedIndexInfo.getPosition().getLedgerId(),
- batchDeletedIndexInfo.getPosition().getEntryId()),
- BitSetRecyclable.create().resetWords(array));
+ batchDeletedIndexInfo.getPosition().getEntryId()), BitSet.valueOf(array));
}
});
} finally {
@@ -1229,27 +1292,55 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate
@Override
public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition,
FindEntryCallback callback, Object ctx, boolean isFindFromLedger) {
- OpFindNewest op;
- Position startPosition = null;
- long max = 0;
+ asyncFindNewestMatching(constraint, condition, null, null, callback, ctx,
+ isFindFromLedger);
+ }
+
+
+ @Override
+ public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition,
+ Position start, Position end, FindEntryCallback callback,
+ Object ctx, boolean isFindFromLedger) {
+ Position startPosition;
switch (constraint) {
- case SearchAllAvailableEntries:
- startPosition = getFirstPosition();
- max = ledger.getNumberOfEntries() - 1;
- break;
- case SearchActiveEntries:
- startPosition = ledger.getNextValidPosition(markDeletePosition);
- max = getNumberOfEntriesInStorage();
- break;
- default:
- callback.findEntryFailed(new ManagedLedgerException("Unknown position constraint"), Optional.empty(), ctx);
- return;
+ case SearchAllAvailableEntries ->
+ startPosition = start == null ? getFirstPosition() : start;
+ case SearchActiveEntries -> {
+ if (start == null) {
+ startPosition = ledger.getNextValidPosition(markDeletePosition);
+ } else {
+ startPosition = start;
+ startPosition = startPosition.compareTo(markDeletePosition) <= 0
+ ? ledger.getNextValidPosition(startPosition) : startPosition;
+ }
+ }
+ default -> {
+ callback.findEntryFailed(
+ new ManagedLedgerException("Unknown position constraint"), Optional.empty(), ctx);
+ return;
+ }
}
+ // startPosition can't be null, should never go here.
if (startPosition == null) {
callback.findEntryFailed(new ManagedLedgerException("Couldn't find start position"),
Optional.empty(), ctx);
return;
}
+ // Calculate the end position
+ Position endPosition = end == null ? ledger.lastConfirmedEntry : end;
+ endPosition = endPosition.compareTo(ledger.lastConfirmedEntry) > 0 ? ledger.lastConfirmedEntry : endPosition;
+ // Calculate the number of entries between the startPosition and endPosition
+ long max = 0;
+ if (startPosition.compareTo(endPosition) <= 0) {
+ max = ledger.getNumberOfEntries(Range.closed(startPosition, endPosition));
+ }
+
+ if (max <= 0) {
+ callback.findEntryComplete(null, ctx);
+ return;
+ }
+
+ OpFindNewest op;
if (isFindFromLedger) {
op = new OpFindNewest(this.ledger, startPosition, condition, max, callback, ctx);
} else {
@@ -1338,14 +1429,12 @@ public void operationComplete() {
lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, isCompactionCursor()
? getProperties() : Collections.emptyMap(), null, null);
individualDeletedMessages.clear();
- if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
- batchDeletedIndexes.values().forEach(BitSetRecyclable::recycle);
+ if (batchDeletedIndexes != null) {
batchDeletedIndexes.clear();
AckSetStateUtil.maybeGetAckSetState(newReadPosition).ifPresent(ackSetState -> {
long[] resetWords = ackSetState.getAckSet();
if (resetWords != null) {
- BitSetRecyclable ackSet = BitSetRecyclable.create().resetWords(resetWords);
- batchDeletedIndexes.put(newReadPosition, ackSet);
+ batchDeletedIndexes.put(newReadPosition, BitSet.valueOf(resetWords));
}
});
}
@@ -1526,7 +1615,7 @@ public Set asyncReplayEntries(Set positi
Set alreadyAcknowledgedPositions = new HashSet<>();
lock.readLock().lock();
try {
- positions.stream().filter(this::isMessageDeleted).forEach(alreadyAcknowledgedPositions::add);
+ positions.stream().filter(this::internalIsMessageDeleted).forEach(alreadyAcknowledgedPositions::add);
} finally {
lock.readLock().unlock();
}
@@ -1974,47 +2063,7 @@ public void asyncMarkDelete(final Position position, Map propertie
log.debug("[{}] Mark delete cursor {} up to position: {}", ledger.getName(), name, position);
}
- Position newPosition = position;
-
- Optional ackSetStateOptional = AckSetStateUtil.maybeGetAckSetState(newPosition);
- if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
- if (ackSetStateOptional.isPresent()) {
- AtomicReference bitSetRecyclable = new AtomicReference<>();
- BitSetRecyclable givenBitSet =
- BitSetRecyclable.create().resetWords(ackSetStateOptional.map(AckSetState::getAckSet).get());
- // In order to prevent the batch index recorded in batchDeletedIndexes from rolling back,
- // only update batchDeletedIndexes when the submitted batch index is greater
- // than the recorded index.
- batchDeletedIndexes.compute(newPosition,
- (k, v) -> {
- if (v == null) {
- return givenBitSet;
- }
- if (givenBitSet.nextSetBit(0) > v.nextSetBit(0)) {
- bitSetRecyclable.set(v);
- return givenBitSet;
- } else {
- bitSetRecyclable.set(givenBitSet);
- return v;
- }
- });
- if (bitSetRecyclable.get() != null) {
- bitSetRecyclable.get().recycle();
- }
- newPosition = ledger.getPreviousPosition(newPosition);
- }
- Map subMap = batchDeletedIndexes.subMap(PositionFactory.EARLIEST, newPosition);
- subMap.values().forEach(BitSetRecyclable::recycle);
- subMap.clear();
- } else {
- if (ackSetStateOptional.isPresent()) {
- AckSetState ackSetState = ackSetStateOptional.get();
- if (ackSetState.getAckSet() != null) {
- newPosition = ledger.getPreviousPosition(newPosition);
- }
- }
- }
-
+ Position newPosition = ackBatchPosition(position);
if (ledger.getLastConfirmedEntry().compareTo(newPosition) < 0) {
boolean shouldCursorMoveForward = false;
try {
@@ -2060,6 +2109,31 @@ public void asyncMarkDelete(final Position position, Map propertie
internalAsyncMarkDelete(newPosition, properties, callback, ctx);
}
+ private Position ackBatchPosition(Position position) {
+ return AckSetStateUtil.maybeGetAckSetState(position)
+ .map(AckSetState::getAckSet)
+ .map(ackSet -> {
+ if (batchDeletedIndexes == null) {
+ return ledger.getPreviousPosition(position);
+ }
+ // In order to prevent the batch index recorded in batchDeletedIndexes from rolling back,
+ // only update batchDeletedIndexes when the submitted batch index is greater
+ // than the recorded index.
+ final var givenBitSet = BitSet.valueOf(ackSet);
+ batchDeletedIndexes.compute(position, (k, v) -> {
+ if (v == null || givenBitSet.nextSetBit(0) > v.nextSetBit(0)) {
+ return givenBitSet;
+ } else {
+ return v;
+ }
+ });
+ final var newPosition = ledger.getPreviousPosition(position);
+ batchDeletedIndexes.subMap(PositionFactory.EARLIEST, newPosition).clear();
+ return newPosition;
+ })
+ .orElse(position);
+ }
+
protected void internalAsyncMarkDelete(final Position newPosition, Map properties,
final MarkDeleteCallback callback, final Object ctx) {
ledger.mbean.addMarkDeleteOp();
@@ -2165,12 +2239,10 @@ public void operationComplete() {
try {
individualDeletedMessages.removeAtMost(mdEntry.newPosition.getLedgerId(),
mdEntry.newPosition.getEntryId());
- if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
- Map subMap = batchDeletedIndexes.subMap(PositionFactory.EARLIEST,
+ if (batchDeletedIndexes != null) {
+ batchDeletedIndexes.subMap(PositionFactory.EARLIEST,
false, PositionFactory.create(mdEntry.newPosition.getLedgerId(),
- mdEntry.newPosition.getEntryId()), true);
- subMap.values().forEach(BitSetRecyclable::recycle);
- subMap.clear();
+ mdEntry.newPosition.getEntryId()), true).clear();
}
persistentMarkDeletePosition = mdEntry.newPosition;
} finally {
@@ -2204,6 +2276,8 @@ public void operationFailed(ManagedLedgerException exception) {
if (State.NoLedger.equals(STATE_UPDATER.get(this))) {
if (ledger.isNoMessagesAfterPos(mdEntry.newPosition)) {
+ log.error("[{}][{}] Metadata ledger creation failed, try to persist the position in the metadata"
+ + " store.", ledger.getName(), name);
persistPositionToMetaStore(mdEntry, cb);
} else {
cb.operationFailed(new ManagedLedgerException("Switch new cursor ledger failed"));
@@ -2302,12 +2376,9 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
return;
}
- if (isMessageDeleted(position)) {
- if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
- BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
- if (bitSetRecyclable != null) {
- bitSetRecyclable.recycle();
- }
+ if (internalIsMessageDeleted(position)) {
+ if (batchDeletedIndexes != null) {
+ batchDeletedIndexes.remove(position);
}
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Position was already deleted {}", ledger.getName(), name, position);
@@ -2316,15 +2387,19 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
}
long[] ackSet = AckSetStateUtil.getAckSetArrayOrNull(position);
if (ackSet == null) {
- if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
- BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
- if (bitSetRecyclable != null) {
- bitSetRecyclable.recycle();
- }
+ if (batchDeletedIndexes != null) {
+ batchDeletedIndexes.remove(position);
}
// Add a range (prev, pos] to the set. Adding the previous entry as an open limit to the range will
// make the RangeSet recognize the "continuity" between adjacent Positions.
- Position previousPosition = ledger.getPreviousPosition(position);
+ // Before https://github.com/apache/pulsar/pull/21105 is merged, the range does not support crossing
+ // multi ledgers, so the first position's entryId maybe "-1".
+ Position previousPosition;
+ if (position.getEntryId() == 0) {
+ previousPosition = PositionFactory.create(position.getLedgerId(), -1);
+ } else {
+ previousPosition = ledger.getPreviousPosition(position);
+ }
individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(),
previousPosition.getEntryId(), position.getLedgerId(), position.getEntryId());
MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
@@ -2333,12 +2408,11 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
log.debug("[{}] [{}] Individually deleted messages: {}", ledger.getName(), name,
individualDeletedMessages);
}
- } else if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
- BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(ackSet);
- BitSetRecyclable bitSet = batchDeletedIndexes.computeIfAbsent(position, (v) -> givenBitSet);
+ } else if (batchDeletedIndexes != null) {
+ final var givenBitSet = BitSet.valueOf(ackSet);
+ final var bitSet = batchDeletedIndexes.computeIfAbsent(position, __ -> givenBitSet);
if (givenBitSet != bitSet) {
bitSet.and(givenBitSet);
- givenBitSet.recycle();
}
if (bitSet.isEmpty()) {
Position previousPosition = ledger.getPreviousPosition(position);
@@ -2346,10 +2420,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
previousPosition.getEntryId(),
position.getLedgerId(), position.getEntryId());
MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
- BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
- if (bitSetRecyclable != null) {
- bitSetRecyclable.recycle();
- }
+ batchDeletedIndexes.remove(position);
}
}
}
@@ -2847,6 +2918,54 @@ public void deleteFailed(ManagedLedgerException ex, Object ctx) {
}, null);
}
+ /**
+ * Manually acknowledge all entries from startPosition to endPosition.
+ * - Since this is an uncommon event, we focus on maintainability. So we do not modify
+ * {@link #individualDeletedMessages} and {@link #batchDeletedIndexes}, but call
+ * {@link #asyncDelete(Position, AsyncCallbacks.DeleteCallback, Object)}.
+ * - This method is valid regardless of the consumer ACK type.
+ * - If there is a consumer ack request after this event, it will also work.
+ */
+ public void skipNonRecoverableEntries(Position startPosition, Position endPosition){
+ long ledgerId = startPosition.getLedgerId();
+ LedgerInfo ledgerInfo = ledger.getLedgersInfo().get(ledgerId);
+ if (ledgerInfo == null) {
+ return;
+ }
+
+ long startEntryId = Math.max(0, startPosition.getEntryId());
+ long endEntryId = ledgerId != endPosition.getLedgerId() ? ledgerInfo.getEntries() : endPosition.getEntryId();
+ if (startEntryId >= endEntryId) {
+ return;
+ }
+
+ lock.writeLock().lock();
+ log.warn("[{}] [{}] Since these entry for ledger [{}] is lost and the autoSkipNonRecoverableData is true, "
+ + "these entries [{}:{}) will be auto acknowledge in subscription",
+ ledger.getName(), name, ledgerId, startEntryId, endEntryId);
+ try {
+ for (long i = startEntryId; i < endEntryId; i++) {
+ if (!individualDeletedMessages.contains(ledgerId, i)) {
+ asyncDelete(PositionFactory.create(ledgerId, i), new AsyncCallbacks.DeleteCallback() {
+ @Override
+ public void deleteComplete(Object ctx) {
+ // ignore.
+ }
+
+ @Override
+ public void deleteFailed(ManagedLedgerException ex, Object ctx) {
+ // The method internalMarkDelete already handled the failure operation. We only need to
+ // make sure the memory state is updated.
+ // If the broker crashed, the non-recoverable ledger will be detected again.
+ }
+ }, null);
+ }
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
// //////////////////////////////////////////////////
void startCreatingNewMetadataLedger() {
@@ -2878,9 +2997,7 @@ public void operationComplete() {
@Override
public void operationFailed(ManagedLedgerException exception) {
- log.error("[{}][{}] Metadata ledger creation failed {}, try to persist the position in the metadata"
- + " store.", ledger.getName(), name, exception);
-
+ log.error("[{}][{}] Metadata ledger creation failed {}", ledger.getName(), name, exception);
synchronized (pendingMarkDeleteOps) {
// At this point we don't have a ledger ready
STATE_UPDATER.set(ManagedCursorImpl.this, State.NoLedger);
@@ -3094,7 +3211,7 @@ private List buildIndividualDeletedMessageRanges() {
private List buildBatchEntryDeletionIndexInfoList() {
lock.readLock().lock();
try {
- if (!getConfig().isDeletionAtBatchIndexLevelEnabled() || batchDeletedIndexes.isEmpty()) {
+ if (batchDeletedIndexes == null || batchDeletedIndexes.isEmpty()) {
return Collections.emptyList();
}
MLDataFormats.NestedPositionInfo.Builder nestedPositionBuilder = MLDataFormats.NestedPositionInfo
@@ -3102,9 +3219,9 @@ private List buildBatchEntryDeletio
MLDataFormats.BatchedEntryDeletionIndexInfo.Builder batchDeletedIndexInfoBuilder = MLDataFormats
.BatchedEntryDeletionIndexInfo.newBuilder();
List result = new ArrayList<>();
- Iterator> iterator = batchDeletedIndexes.entrySet().iterator();
+ final var iterator = batchDeletedIndexes.entrySet().iterator();
while (iterator.hasNext() && result.size() < getConfig().getMaxBatchDeletedIndexToPersist()) {
- Map.Entry entry = iterator.next();
+ final var entry = iterator.next();
nestedPositionBuilder.setLedgerId(entry.getKey().getLedgerId());
nestedPositionBuilder.setEntryId(entry.getKey().getEntryId());
batchDeletedIndexInfoBuilder.setPosition(nestedPositionBuilder.build());
@@ -3125,12 +3242,34 @@ private List buildBatchEntryDeletio
void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, final VoidCallback callback) {
Position position = mdEntry.newPosition;
- PositionInfo pi = PositionInfo.newBuilder().setLedgerId(position.getLedgerId())
+ Builder piBuilder = PositionInfo.newBuilder().setLedgerId(position.getLedgerId())
.setEntryId(position.getEntryId())
- .addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges())
.addAllBatchedEntryDeletionIndexInfo(buildBatchEntryDeletionIndexInfoList())
- .addAllProperties(buildPropertiesMap(mdEntry.properties)).build();
-
+ .addAllProperties(buildPropertiesMap(mdEntry.properties));
+
+ Map internalRanges = null;
+ /**
+ * Cursor will create the {@link #individualDeletedMessages} typed {@link LongPairRangeSet.DefaultRangeSet} if
+ * disabled the config {@link ManagedLedgerConfig#unackedRangesOpenCacheSetEnabled}.
+ * {@link LongPairRangeSet.DefaultRangeSet} never implemented the methods below:
+ * - {@link LongPairRangeSet#toRanges(int)}, which is used to serialize cursor metadata.
+ * - {@link LongPairRangeSet#build(Map)}, which is used to deserialize cursor metadata.
+ * Do not enable the feature that https://github.com/apache/pulsar/pull/9292 introduced, to avoid serialization
+ * and deserialization error.
+ */
+ if (getConfig().isUnackedRangesOpenCacheSetEnabled() && getConfig().isPersistIndividualAckAsLongArray()) {
+ try {
+ internalRanges = individualDeletedMessages.toRanges(getConfig().getMaxUnackedRangesToPersist());
+ } catch (Exception e) {
+ log.warn("[{}]-{} Failed to serialize individualDeletedMessages", ledger.getName(), name, e);
+ }
+ }
+ if (internalRanges != null && !internalRanges.isEmpty()) {
+ piBuilder.addAllIndividualDeletedMessageRanges(buildLongPropertiesMap(internalRanges));
+ } else {
+ piBuilder.addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges());
+ }
+ PositionInfo pi = piBuilder.build();
if (log.isDebugEnabled()) {
log.debug("[{}] Cursor {} Appending to ledger={} position={}", ledger.getName(), name, lh.getId(),
@@ -3152,6 +3291,13 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin
mbean.addWriteCursorLedgerSize(data.length);
callback.operationComplete();
} else {
+ if (state == State.Closed) {
+ // After closed the cursor, the in-progress persistence task will get a
+ // BKException.Code.LedgerClosedException.
+ callback.operationFailed(new CursorAlreadyClosedException(String.format("%s %s skipped this"
+ + " persistence, because the cursor already closed", ledger.getName(), name)));
+ return;
+ }
log.warn("[{}] Error updating cursor {} position {} in meta-ledger {}: {}", ledger.getName(), name,
position, lh1.getId(), BKException.getMessage(rc));
// If we've had a write error, the ledger will be automatically closed, we need to create a new one,
@@ -3489,22 +3635,28 @@ public Position processIndividuallyDeletedMessagesAndGetMarkDeletedPosition(
public boolean isMessageDeleted(Position position) {
lock.readLock().lock();
try {
- return position.compareTo(markDeletePosition) <= 0
- || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId());
+ return internalIsMessageDeleted(position);
} finally {
lock.readLock().unlock();
}
}
+ // When this method is called while the external has already acquired a write lock or a read lock,
+ // it avoids unnecessary lock nesting.
+ private boolean internalIsMessageDeleted(Position position) {
+ return position.compareTo(markDeletePosition) <= 0
+ || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId());
+ }
+
//this method will return a copy of the position's ack set
@Override
public long[] getBatchPositionAckSet(Position position) {
if (batchDeletedIndexes != null) {
- BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.get(position);
- if (bitSetRecyclable == null) {
+ final var bitSet = batchDeletedIndexes.get(position);
+ if (bitSet == null) {
return null;
} else {
- return bitSetRecyclable.toLongArray();
+ return bitSet.toLongArray();
}
} else {
return null;
@@ -3607,8 +3759,8 @@ private ManagedCursorImpl cursorImpl() {
@Override
public long[] getDeletedBatchIndexesAsLongArray(Position position) {
- if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
- BitSetRecyclable bitSet = batchDeletedIndexes.get(position);
+ if (batchDeletedIndexes != null) {
+ final var bitSet = batchDeletedIndexes.get(position);
return bitSet == null ? null : bitSet.toLongArray();
} else {
return null;
@@ -3736,9 +3888,9 @@ public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) thro
lock.readLock().unlock();
}
if (batchDeletedIndexes != null) {
- for (Map.Entry entry : this.batchDeletedIndexes.entrySet()) {
- BitSetRecyclable copiedBitSet = BitSetRecyclable.valueOf(entry.getValue());
- newNonDurableCursor.batchDeletedIndexes.put(entry.getKey(), copiedBitSet);
+ Objects.requireNonNull(newNonDurableCursor.batchDeletedIndexes);
+ for (final var entry : this.batchDeletedIndexes.entrySet()) {
+ newNonDurableCursor.batchDeletedIndexes.put(entry.getKey(), (BitSet) entry.getValue().clone());
}
}
return newNonDurableCursor;
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index 586beb412d297..225f4dba493d5 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -104,7 +104,6 @@
import org.apache.pulsar.common.util.DateFormatter;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.Runnables;
-import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.Stat;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
@@ -229,7 +228,7 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
compressionConfigForManagedCursorInfo);
this.config = config;
this.mbean = new ManagedLedgerFactoryMBeanImpl(this);
- this.entryCacheManager = new RangeEntryCacheManagerImpl(this, openTelemetry);
+ this.entryCacheManager = new RangeEntryCacheManagerImpl(this, scheduledExecutor, openTelemetry);
this.statsTask = scheduledExecutor.scheduleWithFixedDelay(catchingAndLoggingThrowables(this::refreshStats),
0, config.getStatsPeriodSeconds(), TimeUnit.SECONDS);
this.flushCursorsTask = scheduledExecutor.scheduleAtFixedRate(catchingAndLoggingThrowables(this::flushCursors),
@@ -411,11 +410,8 @@ public void asyncOpen(final String name, final ManagedLedgerConfig config, final
new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
config.getBookKeeperEnsemblePlacementPolicyProperties()))
.thenAccept(bk -> {
- final ManagedLedgerImpl newledger = config.getShadowSource() == null
- ? new ManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name,
- mlOwnershipChecker)
- : new ShadowManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name,
- mlOwnershipChecker);
+ final ManagedLedgerImpl newledger =
+ createManagedLedger(bk, store, name, config, mlOwnershipChecker);
PendingInitializeManagedLedger pendingLedger = new PendingInitializeManagedLedger(newledger);
pendingInitializeLedgers.put(name, pendingLedger);
newledger.initialize(new ManagedLedgerInitializeLedgerCallback() {
@@ -441,6 +437,7 @@ public void initializeFailed(ManagedLedgerException e) {
// Clean the map if initialization fails
ledgers.remove(name, future);
+ entryCacheManager.removeEntryCache(name);
if (pendingInitializeLedgers.remove(name, pendingLedger)) {
pendingLedger.ledger.asyncClose(new CloseCallback() {
@@ -472,6 +469,14 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) {
});
}
+ protected ManagedLedgerImpl createManagedLedger(BookKeeper bk, MetaStore store, String name,
+ ManagedLedgerConfig config,
+ Supplier> mlOwnershipChecker) {
+ return config.getShadowSource() == null
+ ? new ManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, mlOwnershipChecker) :
+ new ShadowManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, mlOwnershipChecker);
+ }
+
@Override
public void asyncOpenReadOnlyManagedLedger(String managedLedgerName,
AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback,
@@ -1207,8 +1212,7 @@ public void calculateCursorBacklogs(final TopicName topicName,
BookKeeper bk = getBookKeeper().get();
final CountDownLatch allCursorsCounter = new CountDownLatch(1);
final long errorInReadingCursor = -1;
- ConcurrentOpenHashMap ledgerRetryMap =
- ConcurrentOpenHashMap.newBuilder().build();
+ final var ledgerRetryMap = new ConcurrentHashMap();
final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue();
final Position lastLedgerPosition =
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index cb19bd94bce01..607b6d09cc239 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -574,6 +574,7 @@ public void operationFailed(MetaStoreException e) {
executor.execute(() -> {
mbean.endDataLedgerCreateOp();
if (rc != BKException.Code.OK) {
+ log.error("[{}] Error creating ledger rc={} {}", name, rc, BKException.getMessage(rc));
callback.initializeFailed(createManagedLedgerException(rc));
return;
}
@@ -631,7 +632,7 @@ public void operationComplete(List consumers, Stat s) {
for (final String cursorName : consumers) {
log.info("[{}] Loading cursor {}", name, cursorName);
final ManagedCursorImpl cursor;
- cursor = new ManagedCursorImpl(bookKeeper, ManagedLedgerImpl.this, cursorName);
+ cursor = createCursor(ManagedLedgerImpl.this.bookKeeper, cursorName);
cursor.recover(new VoidCallback() {
@Override
@@ -662,7 +663,7 @@ public void operationFailed(ManagedLedgerException exception) {
log.debug("[{}] Recovering cursor {} lazily", name, cursorName);
}
final ManagedCursorImpl cursor;
- cursor = new ManagedCursorImpl(bookKeeper, ManagedLedgerImpl.this, cursorName);
+ cursor = createCursor(ManagedLedgerImpl.this.bookKeeper, cursorName);
CompletableFuture cursorRecoveryFuture = new CompletableFuture<>();
uninitializedCursors.put(cursorName, cursorRecoveryFuture);
@@ -1006,7 +1007,7 @@ public synchronized void asyncOpenCursor(final String cursorName, final InitialP
if (log.isDebugEnabled()) {
log.debug("[{}] Creating new cursor: {}", name, cursorName);
}
- final ManagedCursorImpl cursor = new ManagedCursorImpl(bookKeeper, this, cursorName);
+ final ManagedCursorImpl cursor = createCursor(bookKeeper, cursorName);
CompletableFuture cursorFuture = new CompletableFuture<>();
uninitializedCursors.put(cursorName, cursorFuture);
Position position = InitialPosition.Earliest == initialPosition ? getFirstPosition() : getLastPosition();
@@ -1038,6 +1039,10 @@ public void operationFailed(ManagedLedgerException exception) {
});
}
+ protected ManagedCursorImpl createCursor(BookKeeper bookKeeper, String cursorName) {
+ return new ManagedCursorImpl(bookKeeper, this, cursorName);
+ }
+
@Override
public synchronized void asyncDeleteCursor(final String consumerName, final DeleteCursorCallback callback,
final Object ctx) {
@@ -1139,16 +1144,17 @@ public ManagedCursor newNonDurableCursor(Position startCursorPosition, String cu
return cachedCursor;
}
- NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, this, cursorName,
- startCursorPosition, initialPosition, isReadCompacted);
- cursor.setActive();
-
- log.info("[{}] Opened new cursor: {}", name, cursor);
+ // The backlog of a non-durable cursor could be incorrect if the cursor is created before `internalTrimLedgers`
+ // and added to the managed ledger after `internalTrimLedgers`.
+ // For more details, see https://github.com/apache/pulsar/pull/23951.
synchronized (this) {
+ NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, this, cursorName,
+ startCursorPosition, initialPosition, isReadCompacted);
+ cursor.setActive();
+ log.info("[{}] Opened new cursor: {}", name, cursor);
addCursor(cursor);
+ return cursor;
}
-
- return cursor;
}
@Override
@@ -2229,13 +2235,11 @@ public void readEntryComplete(Entry entry, Object ctx) {
Object cbCtx = this.cntx;
if (recycle(reOpCount)) {
callback.readEntryComplete(entry, cbCtx);
- return;
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] read entry already completed for {}-{}", name, ledgerId, entryId);
}
entry.release();
- return;
}
}
@@ -2246,7 +2250,6 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
Object cbCtx = this.cntx;
if (recycle(reOpCount)) {
callback.readEntryFailed(exception, cbCtx);
- return;
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] read entry already completed for {}-{}", name, ledgerId, entryId);
@@ -2261,13 +2264,11 @@ public void readEntriesComplete(List returnedEntries, Object ctx) {
Object cbCtx = this.cntx;
if (recycle(reOpCount)) {
callback.readEntriesComplete(returnedEntries, cbCtx);
- return;
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] read entry already completed for {}-{}", name, ledgerId, entryId);
}
returnedEntries.forEach(Entry::release);
- return;
}
}
@@ -2278,12 +2279,10 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
Object cbCtx = this.cntx;
if (recycle(reOpCount)) {
callback.readEntriesFailed(exception, cbCtx);
- return;
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] read entry already completed for {}-{}", name, ledgerId, entryId);
}
- return;
}
}
@@ -3937,7 +3936,8 @@ private void checkManagedLedgerIsOpen() throws ManagedLedgerException {
}
}
- synchronized void setFenced() {
+ @VisibleForTesting
+ public synchronized void setFenced() {
log.info("{} Moving to Fenced state", name);
STATE_UPDATER.set(this, State.Fenced);
}
@@ -4147,7 +4147,7 @@ public Clock getClock() {
protected boolean checkAndCompleteLedgerOpTask(int rc, LedgerHandle lh, Object ctx) {
if (ctx instanceof CompletableFuture) {
// ledger-creation is already timed out and callback is already completed so, delete this ledger and return.
- if (((CompletableFuture) ctx).complete(lh)) {
+ if (((CompletableFuture) ctx).complete(lh) || rc == BKException.Code.TimeoutException) {
return false;
} else {
if (rc == BKException.Code.OK) {
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index d9269ec83b179..611d9d60202cd 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -398,10 +398,13 @@ private static ManagedLedgerInfo updateMLInfoTimestamp(ManagedLedgerInfo info) {
}
private static MetaStoreException getException(Throwable t) {
- if (t.getCause() instanceof MetadataStoreException.BadVersionException) {
- return new ManagedLedgerException.BadVersionException(t.getMessage());
+ Throwable actEx = FutureUtil.unwrapCompletionException(t);
+ if (actEx instanceof MetadataStoreException.BadVersionException badVersionException) {
+ return new ManagedLedgerException.BadVersionException(badVersionException);
+ } else if (actEx instanceof MetaStoreException metaStoreException){
+ return metaStoreException;
} else {
- return new MetaStoreException(t);
+ return new MetaStoreException(actEx);
}
}
@@ -453,8 +456,13 @@ public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws InvalidProto
try {
MLDataFormats.ManagedLedgerInfoMetadata metadata =
MLDataFormats.ManagedLedgerInfoMetadata.parseFrom(metadataBytes);
- return ManagedLedgerInfo.parseFrom(getCompressionCodec(metadata.getCompressionType())
- .decode(byteBuf, metadata.getUncompressedSize()).nioBuffer());
+ ByteBuf uncompressed = getCompressionCodec(metadata.getCompressionType())
+ .decode(byteBuf, metadata.getUncompressedSize());
+ try {
+ return ManagedLedgerInfo.parseFrom(uncompressed.nioBuffer());
+ } finally {
+ uncompressed.release();
+ }
} catch (Exception e) {
log.error("Failed to parse managedLedgerInfo metadata, "
+ "fall back to parse managedLedgerInfo directly.", e);
@@ -475,8 +483,13 @@ public ManagedCursorInfo parseManagedCursorInfo(byte[] data) throws InvalidProto
try {
MLDataFormats.ManagedCursorInfoMetadata metadata =
MLDataFormats.ManagedCursorInfoMetadata.parseFrom(metadataBytes);
- return ManagedCursorInfo.parseFrom(getCompressionCodec(metadata.getCompressionType())
- .decode(byteBuf, metadata.getUncompressedSize()).nioBuffer());
+ ByteBuf uncompressed = getCompressionCodec(metadata.getCompressionType())
+ .decode(byteBuf, metadata.getUncompressedSize());
+ try {
+ return ManagedCursorInfo.parseFrom(uncompressed.nioBuffer());
+ } finally {
+ uncompressed.release();
+ }
} catch (Exception e) {
log.error("Failed to parse ManagedCursorInfo metadata, "
+ "fall back to parse ManagedCursorInfo directly", e);
@@ -500,29 +513,23 @@ private byte[] compressManagedInfo(byte[] info, byte[] metadata, int metadataSer
if (compressionType == null || compressionType.equals(CompressionType.NONE)) {
return info;
}
- ByteBuf metadataByteBuf = null;
- ByteBuf encodeByteBuf = null;
+
+ CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer();
try {
- metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(metadataSerializedSize + 6,
+ ByteBuf metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(metadataSerializedSize + 6,
metadataSerializedSize + 6);
metadataByteBuf.writeShort(MAGIC_MANAGED_INFO_METADATA);
metadataByteBuf.writeInt(metadataSerializedSize);
metadataByteBuf.writeBytes(metadata);
- encodeByteBuf = getCompressionCodec(compressionType)
+ ByteBuf encodeByteBuf = getCompressionCodec(compressionType)
.encode(Unpooled.wrappedBuffer(info));
- CompositeByteBuf compositeByteBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer();
compositeByteBuf.addComponent(true, metadataByteBuf);
compositeByteBuf.addComponent(true, encodeByteBuf);
byte[] dataBytes = new byte[compositeByteBuf.readableBytes()];
compositeByteBuf.readBytes(dataBytes);
return dataBytes;
} finally {
- if (metadataByteBuf != null) {
- metadataByteBuf.release();
- }
- if (encodeByteBuf != null) {
- encodeByteBuf.release();
- }
+ compositeByteBuf.release();
}
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java
index 036ce9223e89d..e0d35ce4e91bb 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java
@@ -139,8 +139,16 @@ public void initiate() {
lastInitTime = System.nanoTime();
if (ml.getManagedLedgerInterceptor() != null) {
long originalDataLen = data.readableBytes();
- payloadProcessorHandle = ml.getManagedLedgerInterceptor()
- .processPayloadBeforeLedgerWrite(this.getCtx(), duplicateBuffer);
+ try {
+ payloadProcessorHandle = ml.getManagedLedgerInterceptor()
+ .processPayloadBeforeLedgerWrite(this.getCtx(), duplicateBuffer);
+ } catch (Exception e) {
+ ml.pendingAddEntries.remove(this);
+ ReferenceCountUtil.safeRelease(duplicateBuffer);
+ log.error("[{}] Error processing payload before ledger write", ml.getName(), e);
+ this.failed(new ManagedLedgerException.ManagedLedgerInterceptException(e));
+ return;
+ }
if (payloadProcessorHandle != null) {
duplicateBuffer = payloadProcessorHandle.getProcessedPayload();
// If data len of entry changes, correct "dataLength" and "currentLedgerSize".
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
index 3fd7e36c433ae..a4928b44bd97d 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java
@@ -137,6 +137,8 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
updateReadPosition(nexReadPosition);
if (lostLedger != null) {
cursor.getManagedLedger().skipNonRecoverableLedger(lostLedger);
+ } else {
+ cursor.skipNonRecoverableEntries(readPosition, nexReadPosition);
}
checkReadCompletion();
} else {
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
index a55e6444b2fd9..76ac3e1be726c 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
@@ -18,13 +18,12 @@
*/
package org.apache.bookkeeper.mledger.impl;
-import static java.util.Objects.requireNonNull;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Range;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import java.util.Map;
import org.apache.pulsar.common.util.collections.LongPairRangeSet;
import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet;
import org.roaringbitmap.RoaringBitSet;
@@ -39,7 +38,6 @@ public class RangeSetWrapper> implements LongPairRangeSe
private final LongPairRangeSet rangeSet;
private final LongPairConsumer rangeConverter;
- private final ManagedLedgerConfig config;
private final boolean enableMultiEntry;
/**
@@ -52,13 +50,19 @@ public class RangeSetWrapper> implements LongPairRangeSe
public RangeSetWrapper(LongPairConsumer rangeConverter,
RangeBoundConsumer rangeBoundConsumer,
ManagedCursorImpl managedCursor) {
- requireNonNull(managedCursor);
- this.config = managedCursor.getManagedLedger().getConfig();
+ this(rangeConverter, rangeBoundConsumer, managedCursor.getConfig().isUnackedRangesOpenCacheSetEnabled(),
+ managedCursor.getConfig().isPersistentUnackedRangesWithMultipleEntriesEnabled());
+ }
+
+ public RangeSetWrapper(LongPairConsumer rangeConverter,
+ RangeBoundConsumer rangeBoundConsumer,
+ boolean unackedRangesOpenCacheSetEnabled,
+ boolean persistentUnackedRangesWithMultipleEntriesEnabled) {
this.rangeConverter = rangeConverter;
- this.rangeSet = config.isUnackedRangesOpenCacheSetEnabled()
+ this.rangeSet = unackedRangesOpenCacheSetEnabled
? new OpenLongPairRangeSet<>(rangeConverter, RoaringBitSet::new)
: new LongPairRangeSet.DefaultRangeSet<>(rangeConverter, rangeBoundConsumer);
- this.enableMultiEntry = config.isPersistentUnackedRangesWithMultipleEntriesEnabled();
+ this.enableMultiEntry = persistentUnackedRangesWithMultipleEntriesEnabled;
}
@Override
@@ -142,6 +146,16 @@ public Range lastRange() {
return rangeSet.lastRange();
}
+ @Override
+ public Map toRanges(int maxRanges) {
+ return rangeSet.toRanges(maxRanges);
+ }
+
+ @Override
+ public void build(Map internalRange) {
+ rangeSet.build(internalRange);
+ }
+
@Override
public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) {
return rangeSet.cardinality(lowerKey, lowerValue, upperKey, upperValue);
@@ -176,4 +190,22 @@ public boolean isDirtyLedgers(long ledgerId) {
public String toString() {
return rangeSet.toString();
}
+
+ @Override
+ public int hashCode() {
+ return rangeSet.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof RangeSetWrapper)) {
+ return false;
+ }
+ if (this == obj) {
+ return true;
+ }
+ @SuppressWarnings("rawtypes")
+ RangeSetWrapper set = (RangeSetWrapper) obj;
+ return this.rangeSet.equals(set.rangeSet);
+ }
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiter.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiter.java
index c87807b86631b..1f4d2c267975c 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiter.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiter.java
@@ -22,12 +22,16 @@
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.metrics.ObservableLongCounter;
import io.prometheus.client.Gauge;
-import lombok.AllArgsConstructor;
-import lombok.ToString;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.opentelemetry.Constants;
import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.InflightReadLimiterUtilization;
import org.apache.pulsar.opentelemetry.annotations.PulsarDeprecatedMetric;
+import org.jctools.queues.SpscArrayQueue;
@Slf4j
public class InflightReadsLimiter implements AutoCloseable {
@@ -58,16 +62,36 @@ public class InflightReadsLimiter implements AutoCloseable {
private final long maxReadsInFlightSize;
private long remainingBytes;
+ private final long acquireTimeoutMillis;
+ private final ScheduledExecutorService timeOutExecutor;
+ private final boolean enabled;
- public InflightReadsLimiter(long maxReadsInFlightSize, OpenTelemetry openTelemetry) {
- if (maxReadsInFlightSize <= 0) {
+ record Handle(long permits, long creationTime, boolean success) {
+ }
+
+ record QueuedHandle(Handle handle, Consumer callback) {
+ }
+
+ private final Queue queuedHandles;
+ private boolean timeoutCheckRunning = false;
+
+ public InflightReadsLimiter(long maxReadsInFlightSize, int maxReadsInFlightAcquireQueueSize,
+ long acquireTimeoutMillis, ScheduledExecutorService timeOutExecutor,
+ OpenTelemetry openTelemetry) {
+ this.maxReadsInFlightSize = maxReadsInFlightSize;
+ this.remainingBytes = maxReadsInFlightSize;
+ this.acquireTimeoutMillis = acquireTimeoutMillis;
+ this.timeOutExecutor = timeOutExecutor;
+ if (maxReadsInFlightSize > 0) {
+ enabled = true;
+ this.queuedHandles = new SpscArrayQueue<>(maxReadsInFlightAcquireQueueSize);
+ } else {
+ enabled = false;
+ this.queuedHandles = null;
// set it to -1 in order to show in the metrics that the metric is not available
PULSAR_ML_READS_BUFFER_SIZE.set(-1);
PULSAR_ML_READS_AVAILABLE_BUFFER_SIZE.set(-1);
}
- this.maxReadsInFlightSize = maxReadsInFlightSize;
- this.remainingBytes = maxReadsInFlightSize;
-
var meter = openTelemetry.getMeter(Constants.BROKER_INSTRUMENTATION_SCOPE_NAME);
inflightReadsLimitCounter = meter.counterBuilder(INFLIGHT_READS_LIMITER_LIMIT_METRIC_NAME)
.setDescription("Maximum number of bytes that can be retained by managed ledger data read from storage "
@@ -102,70 +126,178 @@ public void close() {
inflightReadsUsageCounter.close();
}
- @AllArgsConstructor
- @ToString
- static class Handle {
- final long acquiredPermits;
- final boolean success;
- final int trials;
+ private static final Handle DISABLED = new Handle(0, 0, true);
+ private static final Optional DISABLED_OPTIONAL = Optional.of(DISABLED);
- final long creationTime;
+ /**
+ * Acquires permits from the limiter. If the limiter is disabled, it will immediately return a successful handle.
+ * If permits are available, it will return a handle with the acquired permits. If no permits are available,
+ * it will return an empty optional and the callback will be called when permits become available or when the
+ * acquire timeout is reached. The success field in the handle passed to the callback will be false if the acquire
+ * operation times out. The callback should be non-blocking and run on a desired executor handled within the
+ * callback itself.
+ *
+ * A successful handle will have the success field set to true, and the caller must call release with the handle
+ * when the permits are no longer needed.
+ *
+ * If an unsuccessful handle is returned immediately, it means that the queue limit has been reached and the
+ * callback will not be called. The caller should fail the read operation in this case to apply backpressure.
+ *
+ * @param permits the number of permits to acquire
+ * @param callback the callback to be called when the permits are acquired or timed out
+ * @return an optional handle that contains the permits if acquired, otherwise an empty optional
+ */
+ public Optional acquire(long permits, Consumer callback) {
+ if (isDisabled()) {
+ return DISABLED_OPTIONAL;
+ }
+ return internalAcquire(permits, callback);
}
- private static final Handle DISABLED = new Handle(0, true, 0, -1);
+ private synchronized Optional internalAcquire(long permits, Consumer callback) {
+ Handle handle = new Handle(permits, System.currentTimeMillis(), true);
+ if (remainingBytes >= permits) {
+ remainingBytes -= permits;
+ if (log.isDebugEnabled()) {
+ log.debug("acquired permits: {}, creationTime: {}, remainingBytes:{}", permits, handle.creationTime,
+ remainingBytes);
+ }
+ updateMetrics();
+ return Optional.of(handle);
+ } else if (permits > maxReadsInFlightSize && remainingBytes == maxReadsInFlightSize) {
+ remainingBytes = 0;
+ if (log.isInfoEnabled()) {
+ log.info("Requested permits {} exceeded maxReadsInFlightSize {}, creationTime: {}, remainingBytes:{}. "
+ + "Allowing request with permits set to maxReadsInFlightSize.",
+ permits, maxReadsInFlightSize, handle.creationTime, remainingBytes);
+ }
+ updateMetrics();
+ return Optional.of(new Handle(maxReadsInFlightSize, handle.creationTime, true));
+ } else {
+ if (queuedHandles.offer(new QueuedHandle(handle, callback))) {
+ scheduleTimeOutCheck(acquireTimeoutMillis);
+ return Optional.empty();
+ } else {
+ log.warn("Failed to queue handle for acquiring permits: {}, creationTime: {}, remainingBytes:{}",
+ permits, handle.creationTime, remainingBytes);
+ return Optional.of(new Handle(0, handle.creationTime, false));
+ }
+ }
+ }
- Handle acquire(long permits, Handle current) {
- if (maxReadsInFlightSize <= 0) {
- // feature is disabled
- return DISABLED;
+ private synchronized void scheduleTimeOutCheck(long delayMillis) {
+ if (acquireTimeoutMillis <= 0) {
+ return;
}
- synchronized (this) {
- try {
- if (current == null) {
- if (remainingBytes == 0) {
- return new Handle(0, false, 1, System.currentTimeMillis());
- }
- if (remainingBytes >= permits) {
- remainingBytes -= permits;
- return new Handle(permits, true, 1, System.currentTimeMillis());
- } else {
- long possible = remainingBytes;
- remainingBytes = 0;
- return new Handle(possible, false, 1, System.currentTimeMillis());
- }
+ if (!timeoutCheckRunning) {
+ timeoutCheckRunning = true;
+ timeOutExecutor.schedule(this::timeoutCheck, delayMillis, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ private synchronized void timeoutCheck() {
+ timeoutCheckRunning = false;
+ long delay = 0;
+ while (true) {
+ QueuedHandle queuedHandle = queuedHandles.peek();
+ if (queuedHandle != null) {
+ long age = System.currentTimeMillis() - queuedHandle.handle.creationTime;
+ if (age >= acquireTimeoutMillis) {
+ // remove the peeked handle from the queue
+ queuedHandles.poll();
+ handleTimeout(queuedHandle);
} else {
- if (current.trials >= 4 && current.acquiredPermits > 0) {
- remainingBytes += current.acquiredPermits;
- return new Handle(0, false, 1, current.creationTime);
- }
- if (remainingBytes == 0) {
- return new Handle(current.acquiredPermits, false, current.trials + 1,
- current.creationTime);
- }
- long needed = permits - current.acquiredPermits;
- if (remainingBytes >= needed) {
- remainingBytes -= needed;
- return new Handle(permits, true, current.trials + 1, current.creationTime);
- } else {
- long possible = remainingBytes;
- remainingBytes = 0;
- return new Handle(current.acquiredPermits + possible, false,
- current.trials + 1, current.creationTime);
- }
+ delay = acquireTimeoutMillis - age;
+ break;
}
- } finally {
- updateMetrics();
+ } else {
+ break;
}
}
+ if (delay > 0) {
+ scheduleTimeOutCheck(delay);
+ }
+ }
+
+ private void handleTimeout(QueuedHandle queuedHandle) {
+ if (log.isDebugEnabled()) {
+ log.debug("timed out queued permits: {}, creationTime: {}, remainingBytes:{}",
+ queuedHandle.handle.permits, queuedHandle.handle.creationTime, remainingBytes);
+ }
+ try {
+ queuedHandle.callback.accept(new Handle(0, queuedHandle.handle.creationTime, false));
+ } catch (Exception e) {
+ log.error("Error in callback of timed out queued permits: {}, creationTime: {}, remainingBytes:{}",
+ queuedHandle.handle.permits, queuedHandle.handle.creationTime, remainingBytes, e);
+ }
}
- void release(Handle handle) {
+ /**
+ * Releases permits back to the limiter. If the handle is disabled, this method will be a no-op.
+ *
+ * @param handle the handle containing the permits to release
+ */
+ public void release(Handle handle) {
if (handle == DISABLED) {
return;
}
- synchronized (this) {
- remainingBytes += handle.acquiredPermits;
- updateMetrics();
+ internalRelease(handle);
+ }
+
+ private synchronized void internalRelease(Handle handle) {
+ if (log.isDebugEnabled()) {
+ log.debug("release permits: {}, creationTime: {}, remainingBytes:{}", handle.permits,
+ handle.creationTime, getRemainingBytes());
+ }
+ remainingBytes += handle.permits;
+ while (true) {
+ QueuedHandle queuedHandle = queuedHandles.peek();
+ if (queuedHandle != null) {
+ boolean timedOut = acquireTimeoutMillis > 0
+ && System.currentTimeMillis() - queuedHandle.handle.creationTime > acquireTimeoutMillis;
+ if (timedOut) {
+ // remove the peeked handle from the queue
+ queuedHandles.poll();
+ handleTimeout(queuedHandle);
+ } else if (remainingBytes >= queuedHandle.handle.permits
+ || queuedHandle.handle.permits > maxReadsInFlightSize
+ && remainingBytes == maxReadsInFlightSize) {
+ // remove the peeked handle from the queue
+ queuedHandles.poll();
+ handleQueuedHandle(queuedHandle);
+ } else {
+ break;
+ }
+ } else {
+ break;
+ }
+ }
+ updateMetrics();
+ }
+
+ private void handleQueuedHandle(QueuedHandle queuedHandle) {
+ long permits = queuedHandle.handle.permits;
+ Handle handleForCallback = queuedHandle.handle;
+ if (permits > maxReadsInFlightSize && remainingBytes == maxReadsInFlightSize) {
+ remainingBytes = 0;
+ if (log.isInfoEnabled()) {
+ log.info("Requested permits {} exceeded maxReadsInFlightSize {}, creationTime: {}, remainingBytes:{}. "
+ + "Allowing request with permits set to maxReadsInFlightSize.",
+ permits, maxReadsInFlightSize, queuedHandle.handle.creationTime, remainingBytes);
+ }
+ handleForCallback = new Handle(maxReadsInFlightSize, queuedHandle.handle.creationTime, true);
+ } else {
+ remainingBytes -= permits;
+ if (log.isDebugEnabled()) {
+ log.debug("acquired queued permits: {}, creationTime: {}, remainingBytes:{}",
+ permits, queuedHandle.handle.creationTime, remainingBytes);
+ }
+ }
+ try {
+ queuedHandle.callback.accept(handleForCallback);
+ } catch (Exception e) {
+ log.error("Error in callback of acquired queued permits: {}, creationTime: {}, remainingBytes:{}",
+ handleForCallback.permits, handleForCallback.creationTime, remainingBytes, e);
}
}
@@ -175,8 +307,6 @@ private synchronized void updateMetrics() {
}
public boolean isDisabled() {
- return maxReadsInFlightSize <= 0;
+ return !enabled;
}
-
-
-}
+}
\ No newline at end of file
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManager.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManager.java
index 8b2f3e25f1cbb..dd9c92e0bd295 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManager.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManager.java
@@ -25,9 +25,8 @@
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
-import lombok.AllArgsConstructor;
-import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -95,15 +94,11 @@ public PendingReadsManager(RangeEntryCacheImpl rangeEntryCache) {
this.rangeEntryCache = rangeEntryCache;
}
- @Value
- private static class PendingReadKey {
- private final long startEntry;
- private final long endEntry;
+ private record PendingReadKey(long startEntry, long endEntry) {
long size() {
return endEntry - startEntry + 1;
}
-
boolean includes(PendingReadKey other) {
return startEntry <= other.startEntry && other.endEntry <= endEntry;
}
@@ -135,25 +130,18 @@ PendingReadKey reminderOnRight(PendingReadKey other) {
}
- @AllArgsConstructor
- private static final class ReadEntriesCallbackWithContext {
- final AsyncCallbacks.ReadEntriesCallback callback;
- final Object ctx;
- final long startEntry;
- final long endEntry;
+ private record ReadEntriesCallbackWithContext(AsyncCallbacks.ReadEntriesCallback callback, Object ctx,
+ long startEntry, long endEntry) {
}
- @AllArgsConstructor
- private static final class FindPendingReadOutcome {
- final PendingRead pendingRead;
- final PendingReadKey missingOnLeft;
- final PendingReadKey missingOnRight;
+ private record FindPendingReadOutcome(PendingRead pendingRead,
+ PendingReadKey missingOnLeft, PendingReadKey missingOnRight) {
boolean needsAdditionalReads() {
return missingOnLeft != null || missingOnRight != null;
}
}
- private FindPendingReadOutcome findPendingRead(PendingReadKey key, Map ledgerCache, AtomicBoolean created) {
synchronized (ledgerCache) {
PendingRead existing = ledgerCache.get(key);
@@ -222,18 +210,96 @@ private FindPendingReadOutcome findPendingRead(PendingReadKey key, Map ledgerCache;
- final List callbacks = new ArrayList<>(1);
- boolean completed = false;
+ final ConcurrentMap ledgerCache;
+ final List listeners = new ArrayList<>(1);
+ PendingReadState state = PendingReadState.INITIALISED;
+
+ enum PendingReadState {
+ INITIALISED,
+ ATTACHED,
+ COMPLETED
+ }
public PendingRead(PendingReadKey key,
- Map ledgerCache) {
+ ConcurrentMap ledgerCache) {
this.key = key;
this.ledgerCache = ledgerCache;
}
- private List keepEntries(List list, long startEntry, long endEntry) {
- List result = new ArrayList<>((int) (endEntry - startEntry));
+ public synchronized void attach(CompletableFuture> handle) {
+ if (state != PendingReadState.INITIALISED) {
+ // this shouldn't ever happen. this is here to prevent misuse in future changes
+ throw new IllegalStateException("Unexpected state " + state + " for PendingRead for key " + key);
+ }
+ state = PendingReadState.ATTACHED;
+ handle.whenComplete((entriesToReturn, error) -> {
+ // execute in the completing thread and return a copy of the listeners
+ List callbacks = completeAndRemoveFromCache();
+ // execute the callbacks in the managed ledger executor
+ rangeEntryCache.getManagedLedger().getExecutor().execute(() -> {
+ if (error != null) {
+ readEntriesFailed(callbacks, error);
+ } else {
+ readEntriesComplete(callbacks, entriesToReturn);
+ }
+ });
+ });
+ }
+
+ synchronized boolean addListener(AsyncCallbacks.ReadEntriesCallback callback,
+ Object ctx, long startEntry, long endEntry) {
+ if (state == PendingReadState.COMPLETED) {
+ return false;
+ }
+ listeners.add(new ReadEntriesCallbackWithContext(callback, ctx, startEntry, endEntry));
+ return true;
+ }
+
+ private synchronized List completeAndRemoveFromCache() {
+ state = PendingReadState.COMPLETED;
+ // When the read has completed, remove the instance from the ledgerCache map
+ // so that new reads will go to a new instance.
+ // this is required because we are going to do refcount management
+ // on the results of the callback
+ ledgerCache.remove(key, this);
+ // return a copy of the listeners
+ return List.copyOf(listeners);
+ }
+
+ // this method isn't synchronized since that could lead to deadlocks
+ private void readEntriesComplete(List callbacks,
+ List entriesToReturn) {
+ if (callbacks.size() == 1) {
+ ReadEntriesCallbackWithContext first = callbacks.get(0);
+ if (first.startEntry == key.startEntry
+ && first.endEntry == key.endEntry) {
+ // perfect match, no copy, this is the most common case
+ first.callback.readEntriesComplete((List) entriesToReturn, first.ctx);
+ } else {
+ first.callback.readEntriesComplete(
+ keepEntries(entriesToReturn, first.startEntry, first.endEntry), first.ctx);
+ }
+ } else {
+ for (ReadEntriesCallbackWithContext callback : callbacks) {
+ callback.callback.readEntriesComplete(
+ copyEntries(entriesToReturn, callback.startEntry, callback.endEntry), callback.ctx);
+ }
+ for (EntryImpl entry : entriesToReturn) {
+ entry.release();
+ }
+ }
+ }
+
+ // this method isn't synchronized since that could lead to deadlocks
+ private void readEntriesFailed(List callbacks, Throwable error) {
+ for (ReadEntriesCallbackWithContext callback : callbacks) {
+ ManagedLedgerException mlException = createManagedLedgerException(error);
+ callback.callback.readEntriesFailed(mlException, callback.ctx);
+ }
+ }
+
+ private static List keepEntries(List list, long startEntry, long endEntry) {
+ List result = new ArrayList<>((int) (endEntry - startEntry + 1));
for (EntryImpl entry : list) {
long entryId = entry.getEntryId();
if (startEntry <= entryId && entryId <= endEntry) {
@@ -245,71 +311,16 @@ private List keepEntries(List list, long startEntry, long
return result;
}
- public void attach(CompletableFuture> handle) {
- // when the future is done remove this from the map
- // new reads will go to a new instance
- // this is required because we are going to do refcount management
- // on the results of the callback
- handle.whenComplete((___, error) -> {
- synchronized (PendingRead.this) {
- completed = true;
- synchronized (ledgerCache) {
- ledgerCache.remove(key, this);
- }
- }
- });
-
- handle.thenAcceptAsync(entriesToReturn -> {
- synchronized (PendingRead.this) {
- if (callbacks.size() == 1) {
- ReadEntriesCallbackWithContext first = callbacks.get(0);
- if (first.startEntry == key.startEntry
- && first.endEntry == key.endEntry) {
- // perfect match, no copy, this is the most common case
- first.callback.readEntriesComplete((List) entriesToReturn,
- first.ctx);
- } else {
- first.callback.readEntriesComplete(
- (List) keepEntries(entriesToReturn, first.startEntry, first.endEntry),
- first.ctx);
- }
- } else {
- for (ReadEntriesCallbackWithContext callback : callbacks) {
- long callbackStartEntry = callback.startEntry;
- long callbackEndEntry = callback.endEntry;
- List copy = new ArrayList<>((int) (callbackEndEntry - callbackStartEntry + 1));
- for (EntryImpl entry : entriesToReturn) {
- long entryId = entry.getEntryId();
- if (callbackStartEntry <= entryId && entryId <= callbackEndEntry) {
- EntryImpl entryCopy = EntryImpl.create(entry);
- copy.add(entryCopy);
- }
- }
- callback.callback.readEntriesComplete((List) copy, callback.ctx);
- }
- for (EntryImpl entry : entriesToReturn) {
- entry.release();
- }
- }
- }
- }, rangeEntryCache.getManagedLedger().getExecutor()).exceptionally(exception -> {
- synchronized (PendingRead.this) {
- for (ReadEntriesCallbackWithContext callback : callbacks) {
- ManagedLedgerException mlException = createManagedLedgerException(exception);
- callback.callback.readEntriesFailed(mlException, callback.ctx);
- }
+ private static List copyEntries(List entriesToReturn, long startEntry, long endEntry) {
+ List result = new ArrayList<>((int) (endEntry - startEntry + 1));
+ for (EntryImpl entry : entriesToReturn) {
+ long entryId = entry.getEntryId();
+ if (startEntry <= entryId && entryId <= endEntry) {
+ EntryImpl entryCopy = EntryImpl.create(entry);
+ result.add(entryCopy);
}
- return null;
- });
- }
-
- synchronized boolean addListener(AsyncCallbacks.ReadEntriesCallback callback,
- Object ctx, long startEntry, long endEntry) {
- if (completed) {
- return false;
}
- callbacks.add(new ReadEntriesCallbackWithContext(callback, ctx, startEntry, endEntry));
- return true;
+ return result;
}
}
@@ -318,7 +329,7 @@ void readEntries(ReadHandle lh, long firstEntry, long lastEntry, boolean shouldC
final AsyncCallbacks.ReadEntriesCallback callback, Object ctx) {
final PendingReadKey key = new PendingReadKey(firstEntry, lastEntry);
- Map pendingReadsForLedger =
+ ConcurrentMap pendingReadsForLedger =
cachedPendingReads.computeIfAbsent(lh.getId(), (l) -> new ConcurrentHashMap<>());
boolean listenerAdded = false;
@@ -362,7 +373,7 @@ public void readEntriesFailed(ManagedLedgerException exception,
};
rangeEntryCache.asyncReadEntry0(lh,
missingOnRight.startEntry, missingOnRight.endEntry,
- shouldCacheEntry, readFromRightCallback, null);
+ shouldCacheEntry, readFromRightCallback, null, false);
}
@Override
@@ -372,7 +383,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object dummyCtx4
}
};
rangeEntryCache.asyncReadEntry0(lh, missingOnLeft.startEntry, missingOnLeft.endEntry,
- shouldCacheEntry, readFromLeftCallback, null);
+ shouldCacheEntry, readFromLeftCallback, null, false);
} else if (missingOnLeft != null) {
AsyncCallbacks.ReadEntriesCallback readFromLeftCallback =
new AsyncCallbacks.ReadEntriesCallback() {
@@ -395,7 +406,7 @@ public void readEntriesFailed(ManagedLedgerException exception,
}
};
rangeEntryCache.asyncReadEntry0(lh, missingOnLeft.startEntry, missingOnLeft.endEntry,
- shouldCacheEntry, readFromLeftCallback, null);
+ shouldCacheEntry, readFromLeftCallback, null, false);
} else if (missingOnRight != null) {
AsyncCallbacks.ReadEntriesCallback readFromRightCallback =
new AsyncCallbacks.ReadEntriesCallback() {
@@ -418,7 +429,7 @@ public void readEntriesFailed(ManagedLedgerException exception,
}
};
rangeEntryCache.asyncReadEntry0(lh, missingOnRight.startEntry, missingOnRight.endEntry,
- shouldCacheEntry, readFromRightCallback, null);
+ shouldCacheEntry, readFromRightCallback, null, false);
}
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
index cb006a5f0cea9..b81015ea63988 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java
@@ -22,18 +22,19 @@
import static java.util.Objects.requireNonNull;
import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.createManagedLedgerException;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.PooledByteBufAllocator;
+import java.util.ArrayList;
import java.util.Collection;
-import java.util.Iterator;
+import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.LongAdder;
import org.apache.bookkeeper.client.api.BKException;
import org.apache.bookkeeper.client.api.LedgerEntry;
import org.apache.bookkeeper.client.api.ReadHandle;
-import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback;
import org.apache.bookkeeper.mledger.Entry;
@@ -57,7 +58,9 @@ public class RangeEntryCacheImpl implements EntryCache {
/**
* Overhead per-entry to take into account the envelope.
*/
- private static final long BOOKKEEPER_READ_OVERHEAD_PER_ENTRY = 64;
+ public static final long BOOKKEEPER_READ_OVERHEAD_PER_ENTRY = 64;
+ private static final int DEFAULT_ESTIMATED_ENTRY_SIZE = 10 * 1024;
+ private static final boolean DEFAULT_CACHE_INDIVIDUAL_READ_ENTRY = false;
private final RangeEntryCacheManagerImpl manager;
final ManagedLedgerImpl ml;
@@ -66,18 +69,16 @@ public class RangeEntryCacheImpl implements EntryCache {
private final boolean copyEntries;
private final PendingReadsManager pendingReadsManager;
- private volatile long estimatedEntrySize = 10 * 1024;
-
- private final long readEntryTimeoutMillis;
-
private static final double MB = 1024 * 1024;
+ private final LongAdder totalAddedEntriesSize = new LongAdder();
+ private final LongAdder totalAddedEntriesCount = new LongAdder();
+
public RangeEntryCacheImpl(RangeEntryCacheManagerImpl manager, ManagedLedgerImpl ml, boolean copyEntries) {
this.manager = manager;
this.ml = ml;
this.pendingReadsManager = new PendingReadsManager(this);
this.interceptor = ml.getManagedLedgerInterceptor();
- this.readEntryTimeoutMillis = getManagedLedgerConfig().getReadEntryTimeoutSeconds();
this.entries = new RangeCache<>(EntryImpl::getLength, EntryImpl::getTimestamp);
this.copyEntries = copyEntries;
@@ -102,7 +103,7 @@ public String getName() {
}
@VisibleForTesting
- InflightReadsLimiter getPendingReadsLimiter() {
+ public InflightReadsLimiter getPendingReadsLimiter() {
return manager.getInflightReadsLimiter();
}
@@ -118,17 +119,18 @@ InflightReadsLimiter getPendingReadsLimiter() {
@Override
public boolean insert(EntryImpl entry) {
+ int entryLength = entry.getLength();
if (!manager.hasSpaceInCache()) {
if (log.isDebugEnabled()) {
log.debug("[{}] Skipping cache while doing eviction: {} - size: {}", ml.getName(), entry.getPosition(),
- entry.getLength());
+ entryLength);
}
return false;
}
if (log.isDebugEnabled()) {
log.debug("[{}] Adding entry to cache: {} - size: {}", ml.getName(), entry.getPosition(),
- entry.getLength());
+ entryLength);
}
Position position = entry.getPosition();
@@ -150,7 +152,9 @@ public boolean insert(EntryImpl entry) {
EntryImpl cacheEntry = EntryImpl.create(position, cachedData);
cachedData.release();
if (entries.put(position, cacheEntry)) {
- manager.entryAdded(entry.getLength());
+ totalAddedEntriesSize.add(entryLength);
+ totalAddedEntriesCount.increment();
+ manager.entryAdded(entryLength);
return true;
} else {
// entry was not inserted into cache, we need to discard it
@@ -226,7 +230,23 @@ public void invalidateAllEntries(long ledgerId) {
public void asyncReadEntry(ReadHandle lh, Position position, final ReadEntryCallback callback,
final Object ctx) {
try {
- asyncReadEntry0(lh, position, callback, ctx);
+ asyncReadEntriesByPosition(lh, position, position, 1,
+ DEFAULT_CACHE_INDIVIDUAL_READ_ENTRY,
+ new ReadEntriesCallback() {
+ @Override
+ public void readEntriesComplete(List entries, Object ctx) {
+ if (entries.isEmpty()) {
+ callback.readEntryFailed(new ManagedLedgerException("Could not read given position"), ctx);
+ } else {
+ callback.readEntryComplete(entries.get(0), ctx);
+ }
+ }
+
+ @Override
+ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
+ callback.readEntryFailed(exception, ctx);
+ }
+ }, ctx, true);
} catch (Throwable t) {
log.warn("failed to read entries for {}-{}", lh.getId(), position, t);
// invalidate all entries related to ledger from the cache (it might happen if entry gets corrupt
@@ -237,52 +257,11 @@ public void asyncReadEntry(ReadHandle lh, Position position, final ReadEntryCall
}
}
- private void asyncReadEntry0(ReadHandle lh, Position position, final ReadEntryCallback callback,
- final Object ctx) {
- if (log.isDebugEnabled()) {
- log.debug("[{}] Reading entry ledger {}: {}", ml.getName(), lh.getId(), position.getEntryId());
- }
- EntryImpl entry = entries.get(position);
- if (entry != null) {
- EntryImpl cachedEntry = EntryImpl.create(entry);
- entry.release();
- manager.mlFactoryMBean.recordCacheHit(cachedEntry.getLength());
- callback.readEntryComplete(cachedEntry, ctx);
- } else {
- ReadEntryUtils.readAsync(ml, lh, position.getEntryId(), position.getEntryId()).thenAcceptAsync(
- ledgerEntries -> {
- try {
- Iterator iterator = ledgerEntries.iterator();
- if (iterator.hasNext()) {
- LedgerEntry ledgerEntry = iterator.next();
- EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor);
-
- ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength());
- manager.mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength());
- ml.getMbean().addReadEntriesSample(1, returnEntry.getLength());
- callback.readEntryComplete(returnEntry, ctx);
- } else {
- // got an empty sequence
- callback.readEntryFailed(new ManagedLedgerException("Could not read given position"),
- ctx);
- }
- } finally {
- ledgerEntries.close();
- }
- }, ml.getExecutor()).exceptionally(exception -> {
- ml.invalidateLedgerHandle(lh);
- pendingReadsManager.invalidateLedger(lh.getId());
- callback.readEntryFailed(createManagedLedgerException(exception), ctx);
- return null;
- });
- }
- }
-
@Override
public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boolean shouldCacheEntry,
final ReadEntriesCallback callback, Object ctx) {
try {
- asyncReadEntry0(lh, firstEntry, lastEntry, shouldCacheEntry, callback, ctx);
+ asyncReadEntry0(lh, firstEntry, lastEntry, shouldCacheEntry, callback, ctx, true);
} catch (Throwable t) {
log.warn("failed to read entries for {}--{}-{}", lh.getId(), firstEntry, lastEntry, t);
// invalidate all entries related to ledger from the cache (it might happen if entry gets corrupt
@@ -295,34 +274,123 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole
@SuppressWarnings({ "unchecked", "rawtypes" })
void asyncReadEntry0(ReadHandle lh, long firstEntry, long lastEntry, boolean shouldCacheEntry,
- final ReadEntriesCallback callback, Object ctx) {
- asyncReadEntry0WithLimits(lh, firstEntry, lastEntry, shouldCacheEntry, callback, ctx, null);
+ final ReadEntriesCallback callback, Object ctx, boolean acquirePermits) {
+ final long ledgerId = lh.getId();
+ final int numberOfEntries = (int) (lastEntry - firstEntry) + 1;
+ final Position firstPosition = PositionFactory.create(ledgerId, firstEntry);
+ final Position lastPosition = PositionFactory.create(ledgerId, lastEntry);
+ asyncReadEntriesByPosition(lh, firstPosition, lastPosition, numberOfEntries, shouldCacheEntry, callback, ctx,
+ acquirePermits);
}
- void asyncReadEntry0WithLimits(ReadHandle lh, long firstEntry, long lastEntry, boolean shouldCacheEntry,
- final ReadEntriesCallback originalCallback, Object ctx, InflightReadsLimiter.Handle handle) {
+ void asyncReadEntriesByPosition(ReadHandle lh, Position firstPosition, Position lastPosition, int numberOfEntries,
+ boolean shouldCacheEntry, final ReadEntriesCallback originalCallback,
+ Object ctx, boolean acquirePermits) {
+ checkArgument(firstPosition.getLedgerId() == lastPosition.getLedgerId(),
+ "Invalid range. Entries %s and %s should be in the same ledger.",
+ firstPosition, lastPosition);
+ checkArgument(firstPosition.getLedgerId() == lh.getId(),
+ "Invalid ReadHandle. The ledger %s of the range positions should match the handle's ledger %s.",
+ firstPosition.getLedgerId(), lh.getId());
- final AsyncCallbacks.ReadEntriesCallback callback =
- handlePendingReadsLimits(lh, firstEntry, lastEntry, shouldCacheEntry,
- originalCallback, ctx, handle);
- if (callback == null) {
- return;
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Reading {} entries in range {} to {}", ml.getName(), numberOfEntries, firstPosition,
+ lastPosition);
}
- final long ledgerId = lh.getId();
- final int entriesToRead = (int) (lastEntry - firstEntry) + 1;
- final Position firstPosition = PositionFactory.create(lh.getId(), firstEntry);
- final Position lastPosition = PositionFactory.create(lh.getId(), lastEntry);
+ InflightReadsLimiter pendingReadsLimiter = getPendingReadsLimiter();
+ if (!acquirePermits || pendingReadsLimiter.isDisabled()) {
+ doAsyncReadEntriesByPosition(lh, firstPosition, lastPosition, numberOfEntries, shouldCacheEntry,
+ originalCallback, ctx);
+ } else {
+ long estimatedEntrySize = getEstimatedEntrySize();
+ long estimatedReadSize = numberOfEntries * estimatedEntrySize;
+ if (log.isDebugEnabled()) {
+ log.debug("Estimated read size: {} bytes for {} entries with {} estimated entry size",
+ estimatedReadSize,
+ numberOfEntries, estimatedEntrySize);
+ }
+ Optional optionalHandle =
+ pendingReadsLimiter.acquire(estimatedReadSize, handle -> {
+ // permits were not immediately available, callback will be executed when permits are acquired
+ // or timeout
+ ml.getExecutor().execute(() -> {
+ doAsyncReadEntriesWithAcquiredPermits(lh, firstPosition, lastPosition, numberOfEntries,
+ shouldCacheEntry, originalCallback, ctx, handle, estimatedReadSize);
+ });
+ });
+ // permits were immediately available and acquired
+ if (optionalHandle.isPresent()) {
+ doAsyncReadEntriesWithAcquiredPermits(lh, firstPosition, lastPosition, numberOfEntries,
+ shouldCacheEntry, originalCallback, ctx, optionalHandle.get(), estimatedReadSize);
+ }
+ }
+ }
- if (log.isDebugEnabled()) {
- log.debug("[{}] Reading entries range ledger {}: {} to {}", ml.getName(), ledgerId, firstEntry, lastEntry);
+ void doAsyncReadEntriesWithAcquiredPermits(ReadHandle lh, Position firstPosition, Position lastPosition,
+ int numberOfEntries, boolean shouldCacheEntry,
+ final ReadEntriesCallback originalCallback, Object ctx,
+ InflightReadsLimiter.Handle handle, long estimatedReadSize) {
+ if (!handle.success()) {
+ String message = String.format(
+ "Couldn't acquire enough permits on the max reads in flight limiter to read from ledger "
+ + "%d, %s, estimated read size %d bytes for %d entries (check "
+ + "managedLedgerMaxReadsInFlightSizeInMB, "
+ + "managedLedgerMaxReadsInFlightPermitsAcquireTimeoutMillis and "
+ + "managedLedgerMaxReadsInFlightPermitsAcquireQueueSize)", lh.getId(), getName(),
+ estimatedReadSize, numberOfEntries);
+ log.error(message);
+ originalCallback.readEntriesFailed(new ManagedLedgerException.TooManyRequestsException(message), ctx);
+ return;
}
+ InflightReadsLimiter pendingReadsLimiter = getPendingReadsLimiter();
+ ReadEntriesCallback wrappedCallback = new ReadEntriesCallback() {
+ @Override
+ public void readEntriesComplete(List entries, Object ctx2) {
+ if (!entries.isEmpty()) {
+ // release permits only when entries have been handled
+ AtomicInteger remainingCount = new AtomicInteger(entries.size());
+ for (Entry entry : entries) {
+ ((EntryImpl) entry).onDeallocate(() -> {
+ if (remainingCount.decrementAndGet() <= 0) {
+ pendingReadsLimiter.release(handle);
+ }
+ });
+ }
+ } else {
+ pendingReadsLimiter.release(handle);
+ }
+ originalCallback.readEntriesComplete(entries, ctx2);
+ }
- Collection cachedEntries = entries.getRange(firstPosition, lastPosition);
+ @Override
+ public void readEntriesFailed(ManagedLedgerException exception, Object ctx2) {
+ pendingReadsLimiter.release(handle);
+ originalCallback.readEntriesFailed(exception, ctx2);
+ }
+ };
+ doAsyncReadEntriesByPosition(lh, firstPosition, lastPosition, numberOfEntries, shouldCacheEntry,
+ wrappedCallback, ctx);
+ }
+
+ void doAsyncReadEntriesByPosition(ReadHandle lh, Position firstPosition, Position lastPosition, int numberOfEntries,
+ boolean shouldCacheEntry, final ReadEntriesCallback callback,
+ Object ctx) {
+ Collection cachedEntries;
+ if (firstPosition.compareTo(lastPosition) == 0) {
+ EntryImpl cachedEntry = entries.get(firstPosition);
+ if (cachedEntry == null) {
+ cachedEntries = Collections.emptyList();
+ } else {
+ cachedEntries = Collections.singleton(cachedEntry);
+ }
+ } else {
+ cachedEntries = entries.getRange(firstPosition, lastPosition);
+ }
- if (cachedEntries.size() == entriesToRead) {
+ if (cachedEntries.size() == numberOfEntries) {
long totalCachedSize = 0;
- final List entriesToReturn = Lists.newArrayListWithExpectedSize(entriesToRead);
+ final List entriesToReturn = new ArrayList<>(numberOfEntries);
// All entries found in cache
for (EntryImpl entry : cachedEntries) {
@@ -333,11 +401,11 @@ void asyncReadEntry0WithLimits(ReadHandle lh, long firstEntry, long lastEntry, b
manager.mlFactoryMBean.recordCacheHits(entriesToReturn.size(), totalCachedSize);
if (log.isDebugEnabled()) {
- log.debug("[{}] Ledger {} -- Found in cache entries: {}-{}", ml.getName(), ledgerId, firstEntry,
- lastEntry);
+ log.debug("[{}] Cache hit for {} entries in range {} to {}", ml.getName(), numberOfEntries,
+ firstPosition, lastPosition);
}
- callback.readEntriesComplete((List) entriesToReturn, ctx);
+ callback.readEntriesComplete(entriesToReturn, ctx);
} else {
if (!cachedEntries.isEmpty()) {
@@ -345,77 +413,24 @@ void asyncReadEntry0WithLimits(ReadHandle lh, long firstEntry, long lastEntry, b
}
// Read all the entries from bookkeeper
- pendingReadsManager.readEntries(lh, firstEntry, lastEntry,
+ pendingReadsManager.readEntries(lh, firstPosition.getEntryId(), lastPosition.getEntryId(),
shouldCacheEntry, callback, ctx);
-
}
}
- private AsyncCallbacks.ReadEntriesCallback handlePendingReadsLimits(ReadHandle lh,
- long firstEntry, long lastEntry,
- boolean shouldCacheEntry,
- AsyncCallbacks.ReadEntriesCallback originalCallback,
- Object ctx, InflightReadsLimiter.Handle handle) {
- InflightReadsLimiter pendingReadsLimiter = getPendingReadsLimiter();
- if (pendingReadsLimiter.isDisabled()) {
- return originalCallback;
+ @VisibleForTesting
+ public long getEstimatedEntrySize() {
+ long estimatedEntrySize = getAvgEntrySize();
+ if (estimatedEntrySize == 0) {
+ estimatedEntrySize = DEFAULT_ESTIMATED_ENTRY_SIZE;
}
- long estimatedReadSize = (1 + lastEntry - firstEntry)
- * (estimatedEntrySize + BOOKKEEPER_READ_OVERHEAD_PER_ENTRY);
- final AsyncCallbacks.ReadEntriesCallback callback;
- InflightReadsLimiter.Handle newHandle = pendingReadsLimiter.acquire(estimatedReadSize, handle);
- if (!newHandle.success) {
- long now = System.currentTimeMillis();
- if (now - newHandle.creationTime > readEntryTimeoutMillis) {
- String message = "Time-out elapsed while acquiring enough permits "
- + "on the memory limiter to read from ledger "
- + lh.getId()
- + ", " + getName()
- + ", estimated read size " + estimatedReadSize + " bytes"
- + " for " + (1 + lastEntry - firstEntry)
- + " entries (check managedLedgerMaxReadsInFlightSizeInMB)";
- log.error(message);
- pendingReadsLimiter.release(newHandle);
- originalCallback.readEntriesFailed(
- new ManagedLedgerException.TooManyRequestsException(message), ctx);
- return null;
- }
- ml.getExecutor().execute(() -> {
- asyncReadEntry0WithLimits(lh, firstEntry, lastEntry, shouldCacheEntry,
- originalCallback, ctx, newHandle);
- });
- return null;
- } else {
- callback = new AsyncCallbacks.ReadEntriesCallback() {
-
- @Override
- public void readEntriesComplete(List entries, Object ctx) {
- if (!entries.isEmpty()) {
- long size = entries.get(0).getLength();
- estimatedEntrySize = size;
-
- AtomicInteger remainingCount = new AtomicInteger(entries.size());
- for (Entry entry : entries) {
- ((EntryImpl) entry).onDeallocate(() -> {
- if (remainingCount.decrementAndGet() <= 0) {
- pendingReadsLimiter.release(newHandle);
- }
- });
- }
- } else {
- pendingReadsLimiter.release(newHandle);
- }
- originalCallback.readEntriesComplete(entries, ctx);
- }
+ return estimatedEntrySize + BOOKKEEPER_READ_OVERHEAD_PER_ENTRY;
+ }
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- pendingReadsLimiter.release(newHandle);
- originalCallback.readEntriesFailed(exception, ctx);
- }
- };
- }
- return callback;
+ private long getAvgEntrySize() {
+ long totalAddedEntriesCount = this.totalAddedEntriesCount.sum();
+ long totalAddedEntriesSize = this.totalAddedEntriesSize.sum();
+ return totalAddedEntriesCount != 0 ? totalAddedEntriesSize / totalAddedEntriesCount : 0;
}
/**
@@ -438,8 +453,7 @@ CompletableFuture> readFromStorage(ReadHandle lh,
try {
// We got the entries, we need to transform them to a List<> type
long totalSize = 0;
- final List entriesToReturn =
- Lists.newArrayListWithExpectedSize(entriesToRead);
+ final List entriesToReturn = new ArrayList<>(entriesToRead);
for (LedgerEntry e : ledgerEntries) {
EntryImpl entry = RangeEntryCacheManagerImpl.create(e, interceptor);
entriesToReturn.add(entry);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
index 34be25df1f476..61d52aa3919ae 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheManagerImpl.java
@@ -28,7 +28,9 @@
import java.util.concurrent.atomic.AtomicLong;
import org.apache.bookkeeper.client.api.LedgerEntry;
import org.apache.bookkeeper.client.impl.LedgerEntryImpl;
+import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
import org.apache.bookkeeper.mledger.impl.EntryImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryMBeanImpl;
@@ -57,12 +59,16 @@ public class RangeEntryCacheManagerImpl implements EntryCacheManager {
private static final double evictionTriggerThresholdPercent = 0.98;
- public RangeEntryCacheManagerImpl(ManagedLedgerFactoryImpl factory, OpenTelemetry openTelemetry) {
- this.maxSize = factory.getConfig().getMaxCacheSize();
- this.inflightReadsLimiter = new InflightReadsLimiter(
- factory.getConfig().getManagedLedgerMaxReadsInFlightSize(), openTelemetry);
+ public RangeEntryCacheManagerImpl(ManagedLedgerFactoryImpl factory, OrderedScheduler scheduledExecutor,
+ OpenTelemetry openTelemetry) {
+ ManagedLedgerFactoryConfig config = factory.getConfig();
+ this.maxSize = config.getMaxCacheSize();
+ this.inflightReadsLimiter = new InflightReadsLimiter(config.getManagedLedgerMaxReadsInFlightSize(),
+ config.getManagedLedgerMaxReadsInFlightPermitsAcquireQueueSize(),
+ config.getManagedLedgerMaxReadsInFlightPermitsAcquireTimeoutMillis(),
+ scheduledExecutor, openTelemetry);
this.evictionTriggerThreshold = (long) (maxSize * evictionTriggerThresholdPercent);
- this.cacheEvictionWatermark = factory.getConfig().getCacheEvictionWatermark();
+ this.cacheEvictionWatermark = config.getCacheEvictionWatermark();
this.evictionPolicy = new EntryCacheDefaultEvictionPolicy();
this.mlFactory = factory;
this.mlFactoryMBean = factory.getMbean();
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java
index 7af3680d880b2..d160189188a18 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java
@@ -125,7 +125,7 @@ public static OffloaderDefinition getOffloaderDefinition(String narPath, String
public static Offloaders searchForOffloaders(String offloadersPath, String narExtractionDirectory)
throws IOException {
- Path path = Paths.get(offloadersPath).toAbsolutePath();
+ Path path = Paths.get(offloadersPath).toAbsolutePath().normalize();
log.info("Searching for offloaders in {}", path);
Offloaders offloaders = new Offloaders();
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
index 2f2b161a30684..c1de09f10a6b0 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
@@ -106,7 +106,39 @@ K getKey() {
return localKey;
}
+ /**
+ * Get the value associated with the key. Returns null if the key does not match the key.
+ *
+ * @param key the key to match
+ * @return the value associated with the key, or null if the value has already been recycled or the key does not
+ * match
+ */
V getValue(K key) {
+ return getValueInternal(key, false);
+ }
+
+ /**
+ * Get the value associated with the Map.Entry's key and value. Exact instance of the key is required to match.
+ * @param entry the entry which contains the key and {@link EntryWrapper} value to get the value from
+ * @return the value associated with the key, or null if the value has already been recycled or the key does not
+ * exactly match the same instance
+ */
+ static V getValueMatchingMapEntry(Map.Entry> entry) {
+ return entry.getValue().getValueInternal(entry.getKey(), true);
+ }
+
+ /**
+ * Get the value associated with the key. Returns null if the key does not match the key associated with the
+ * value.
+ *
+ * @param key the key to match
+ * @param requireSameKeyInstance when true, the matching will be restricted to exactly the same instance of the
+ * key as the one stored in the wrapper. This is used to avoid any races
+ * when retrieving or removing the entries from the cache when the key and value
+ * instances are available.
+ * @return the value associated with the key, or null if the key does not match
+ */
+ private V getValueInternal(K key, boolean requireSameKeyInstance) {
long stamp = lock.tryOptimisticRead();
K localKey = this.key;
V localValue = this.value;
@@ -116,7 +148,11 @@ V getValue(K key) {
localValue = this.value;
lock.unlockRead(stamp);
}
- if (localKey != key) {
+ // check that the given key matches the key associated with the value in the entry
+ // this is used to detect if the entry has already been recycled and contains another key
+ // when requireSameKeyInstance is true, the key must be exactly the same instance as the one stored in the
+ // entry to match
+ if (localKey != key && (requireSameKeyInstance || localKey == null || !localKey.equals(key))) {
return null;
}
return localValue;
@@ -236,34 +272,51 @@ public boolean exists(Key key) {
* The caller is responsible for releasing the reference.
*/
public Value get(Key key) {
- return getValue(key, entries.get(key));
+ return getValueFromWrapper(key, entries.get(key));
}
- private Value getValue(Key key, EntryWrapper valueWrapper) {
+ private Value getValueFromWrapper(Key key, EntryWrapper valueWrapper) {
if (valueWrapper == null) {
return null;
} else {
Value value = valueWrapper.getValue(key);
- if (value == null) {
- // the wrapper has been recycled and contains another key
- return null;
- }
- try {
- value.retain();
- } catch (IllegalReferenceCountException e) {
- // Value was already deallocated
- return null;
- }
- // check that the value matches the key and that there's at least 2 references to it since
- // the cache should be holding one reference and a new reference was just added in this method
- if (value.refCnt() > 1 && value.matchesKey(key)) {
- return value;
- } else {
- // Value or IdentityWrapper was recycled and already contains another value
- // release the reference added in this method
- value.release();
- return null;
- }
+ return getRetainedValueMatchingKey(key, value);
+ }
+ }
+
+ /**
+ * @apiNote the returned value must be released if it's not null
+ */
+ private Value getValueMatchingEntry(Map.Entry> entry) {
+ Value valueMatchingEntry = EntryWrapper.getValueMatchingMapEntry(entry);
+ return getRetainedValueMatchingKey(entry.getKey(), valueMatchingEntry);
+ }
+
+ // validates that the value matches the key and that the value has not been recycled
+ // which are possible due to the lack of exclusive locks in the cache and the use of reference counted objects
+ /**
+ * @apiNote the returned value must be released if it's not null
+ */
+ private Value getRetainedValueMatchingKey(Key key, Value value) {
+ if (value == null) {
+ // the wrapper has been recycled and contains another key
+ return null;
+ }
+ try {
+ value.retain();
+ } catch (IllegalReferenceCountException e) {
+ // Value was already deallocated
+ return null;
+ }
+ // check that the value matches the key and that there's at least 2 references to it since
+ // the cache should be holding one reference and a new reference was just added in this method
+ if (value.refCnt() > 1 && value.matchesKey(key)) {
+ return value;
+ } else {
+ // Value or IdentityWrapper was recycled and already contains another value
+ // release the reference added in this method
+ value.release();
+ return null;
}
}
@@ -280,7 +333,7 @@ public Collection getRange(Key first, Key last) {
// Return the values of the entries found in cache
for (Map.Entry> entry : entries.subMap(first, true, last, true).entrySet()) {
- Value value = getValue(entry.getKey(), entry.getValue());
+ Value value = getValueMatchingEntry(entry);
if (value != null) {
values.add(value);
}
@@ -297,10 +350,13 @@ public Collection getRange(Key first, Key last) {
* @return an pair of ints, containing the number of removed entries and the total size
*/
public Pair removeRange(Key first, Key last, boolean lastInclusive) {
+ if (log.isDebugEnabled()) {
+ log.debug("Removing entries in range [{}, {}], lastInclusive: {}", first, last, lastInclusive);
+ }
RemovalCounters counters = RemovalCounters.create();
Map> subMap = entries.subMap(first, true, last, lastInclusive);
for (Map.Entry> entry : subMap.entrySet()) {
- removeEntry(entry, counters, true);
+ removeEntry(entry, counters);
}
return handleRemovalResult(counters);
}
@@ -311,84 +367,48 @@ enum RemoveEntryResult {
BREAK_LOOP;
}
- private RemoveEntryResult removeEntry(Map.Entry> entry, RemovalCounters counters,
- boolean skipInvalid) {
- return removeEntry(entry, counters, skipInvalid, x -> true);
+ private RemoveEntryResult removeEntry(Map.Entry> entry, RemovalCounters counters) {
+ return removeEntry(entry, counters, x -> true);
}
private RemoveEntryResult removeEntry(Map.Entry> entry, RemovalCounters counters,
- boolean skipInvalid, Predicate removeCondition) {
+ Predicate removeCondition) {
Key key = entry.getKey();
EntryWrapper entryWrapper = entry.getValue();
- Value value = entryWrapper.getValue(key);
+ Value value = getValueMatchingEntry(entry);
if (value == null) {
- // the wrapper has already been recycled and contains another key
- if (!skipInvalid) {
- EntryWrapper removed = entries.remove(key);
- if (removed != null) {
- // log and remove the entry without releasing the value
- log.info("Key {} does not match the entry's value wrapper's key {}, removed entry by key without "
- + "releasing the value", key, entryWrapper.getKey());
- counters.entryRemoved(removed.getSize());
- return RemoveEntryResult.ENTRY_REMOVED;
- }
- }
- return RemoveEntryResult.CONTINUE_LOOP;
- }
- try {
- // add extra retain to avoid value being released while we are removing it
- value.retain();
- } catch (IllegalReferenceCountException e) {
- // Value was already released
- if (!skipInvalid) {
- // remove the specific entry without releasing the value
- if (entries.remove(key, entryWrapper)) {
- log.info("Value was already released for key {}, removed entry without releasing the value", key);
- counters.entryRemoved(entryWrapper.getSize());
- return RemoveEntryResult.ENTRY_REMOVED;
- }
- }
+ // the wrapper has already been recycled or contains another key
+ entries.remove(key, entryWrapper);
return RemoveEntryResult.CONTINUE_LOOP;
}
- if (!value.matchesKey(key)) {
- // this is unexpected since the IdentityWrapper.getValue(key) already checked that the value matches the key
- log.warn("Unexpected race condition. Value {} does not match the key {}. Removing entry.", value, key);
- }
try {
if (!removeCondition.test(value)) {
return RemoveEntryResult.BREAK_LOOP;
}
- if (!skipInvalid) {
- // remove the specific entry
- boolean entryRemoved = entries.remove(key, entryWrapper);
- if (entryRemoved) {
- counters.entryRemoved(entryWrapper.getSize());
- // check that the value hasn't been recycled in between
- // there should be at least 2 references since this method adds one and the cache should have
- // one reference. it is valid that the value contains references even after the key has been
- // removed from the cache
- if (value.refCnt() > 1) {
- entryWrapper.recycle();
- // remove the cache reference
- value.release();
- } else {
- log.info("Unexpected refCnt {} for key {}, removed entry without releasing the value",
- value.refCnt(), key);
- }
- }
- } else if (skipInvalid && value.refCnt() > 1 && entries.remove(key, entryWrapper)) {
- // when skipInvalid is true, we don't remove the entry if it doesn't match matches the key
- // or the refCnt is invalid
+ // remove the specific entry
+ boolean entryRemoved = entries.remove(key, entryWrapper);
+ if (entryRemoved) {
counters.entryRemoved(entryWrapper.getSize());
- entryWrapper.recycle();
- // remove the cache reference
- value.release();
+ // check that the value hasn't been recycled in between
+ // there should be at least 2 references since this method adds one and the cache should have
+ // one reference. it is valid that the value contains references even after the key has been
+ // removed from the cache
+ if (value.refCnt() > 1) {
+ entryWrapper.recycle();
+ // remove the cache reference
+ value.release();
+ } else {
+ log.info("Unexpected refCnt {} for key {}, removed entry without releasing the value",
+ value.refCnt(), key);
+ }
+ return RemoveEntryResult.ENTRY_REMOVED;
+ } else {
+ return RemoveEntryResult.CONTINUE_LOOP;
}
} finally {
// remove the extra retain
value.release();
}
- return RemoveEntryResult.ENTRY_REMOVED;
}
private Pair handleRemovalResult(RemovalCounters counters) {
@@ -404,6 +424,9 @@ private Pair handleRemovalResult(RemovalCounters counters) {
* @return a pair containing the number of entries evicted and their total size
*/
public Pair evictLeastAccessedEntries(long minSize) {
+ if (log.isDebugEnabled()) {
+ log.debug("Evicting entries to reach a minimum size of {}", minSize);
+ }
checkArgument(minSize > 0);
RemovalCounters counters = RemovalCounters.create();
while (counters.removedSize < minSize && !Thread.currentThread().isInterrupted()) {
@@ -411,7 +434,7 @@ public Pair evictLeastAccessedEntries(long minSize) {
if (entry == null) {
break;
}
- removeEntry(entry, counters, false);
+ removeEntry(entry, counters);
}
return handleRemovalResult(counters);
}
@@ -422,13 +445,16 @@ public Pair evictLeastAccessedEntries(long minSize) {
* @return the tota
*/
public Pair evictLEntriesBeforeTimestamp(long maxTimestamp) {
+ if (log.isDebugEnabled()) {
+ log.debug("Evicting entries with timestamp <= {}", maxTimestamp);
+ }
RemovalCounters counters = RemovalCounters.create();
while (!Thread.currentThread().isInterrupted()) {
Map.Entry> entry = entries.firstEntry();
if (entry == null) {
break;
}
- if (removeEntry(entry, counters, false, value -> timestampExtractor.getTimestamp(value) <= maxTimestamp)
+ if (removeEntry(entry, counters, value -> timestampExtractor.getTimestamp(value) <= maxTimestamp)
== RemoveEntryResult.BREAK_LOOP) {
break;
}
@@ -453,13 +479,16 @@ public long getSize() {
* @return size of removed entries
*/
public Pair clear() {
+ if (log.isDebugEnabled()) {
+ log.debug("Clearing the cache with {} entries and size {}", entries.size(), size.get());
+ }
RemovalCounters counters = RemovalCounters.create();
while (!Thread.currentThread().isInterrupted()) {
Map.Entry> entry = entries.firstEntry();
if (entry == null) {
break;
}
- removeEntry(entry, counters, false);
+ removeEntry(entry, counters);
}
return handleRemovalResult(counters);
}
diff --git a/managed-ledger/src/main/proto/MLDataFormats.proto b/managed-ledger/src/main/proto/MLDataFormats.proto
index fdffed6762db7..f196649df0fdf 100644
--- a/managed-ledger/src/main/proto/MLDataFormats.proto
+++ b/managed-ledger/src/main/proto/MLDataFormats.proto
@@ -82,6 +82,7 @@ message PositionInfo {
// Store which index in the batch message has been deleted
repeated BatchedEntryDeletionIndexInfo batchedEntryDeletionIndexInfo = 5;
+ repeated LongListMap individualDeletedMessageRanges = 6;
}
message NestedPositionInfo {
@@ -89,6 +90,11 @@ message NestedPositionInfo {
required int64 entryId = 2;
}
+message LongListMap {
+ required int64 key = 1;
+ repeated int64 values = 2;
+}
+
message MessageRange {
required NestedPositionInfo lowerEndpoint = 1;
required NestedPositionInfo upperEndpoint = 2;
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/InflightReadsLimiterIntegrationTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/InflightReadsLimiterIntegrationTest.java
new file mode 100644
index 0000000000000..48f0cf08ddff4
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/InflightReadsLimiterIntegrationTest.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import io.netty.util.concurrent.DefaultThreadFactory;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.api.LedgerEntries;
+import org.apache.bookkeeper.mledger.AsyncCallbacks;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
+import org.apache.bookkeeper.mledger.impl.cache.InflightReadsLimiter;
+import org.apache.bookkeeper.mledger.impl.cache.RangeEntryCacheImpl;
+import org.apache.bookkeeper.mledger.impl.cache.RangeEntryCacheManagerImpl;
+import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
+import org.awaitility.Awaitility;
+import org.mockito.Mockito;
+import org.mockito.stubbing.Answer;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Slf4j
+public class InflightReadsLimiterIntegrationTest extends MockedBookKeeperTestCase {
+
+ @DataProvider
+ public Object[][] readMissingCases() {
+ return new Object[][]{
+ {"missRight"},
+ {"missLeft"},
+ {"bothMiss"}
+ };
+ }
+
+ @Test(dataProvider = "readMissingCases")
+ public void testPreciseLimitation(String missingCase) throws Exception {
+ final long start1 = 50;
+ final long start2 = "missLeft".endsWith(missingCase) || "bothMiss".equals(missingCase) ? 30 : 50;
+ final long end1 = 99;
+ final long end2 = "missRight".endsWith(missingCase) || "bothMiss".equals(missingCase) ? 109 : 99;
+ final HashSet secondReadEntries = new HashSet<>();
+ if (start2 < start1) {
+ secondReadEntries.add(start2);
+ }
+ if (end2 > end1) {
+ secondReadEntries.add(end1 + 1);
+ }
+ final int readCount1 = (int) (end1 - start1 + 1);
+ final int readCount2 = (int) (end2 - start2 + 1);
+
+ final DefaultThreadFactory threadFactory = new DefaultThreadFactory(UUID.randomUUID().toString());
+ final ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setMaxEntriesPerLedger(100000);
+ ManagedLedgerFactoryConfig factoryConfig = new ManagedLedgerFactoryConfig();
+ factoryConfig.setCacheEvictionIntervalMs(3600 * 1000);
+ factoryConfig.setManagedLedgerMaxReadsInFlightSize(1000_000);
+ final ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConfig);
+ final ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("my_test_ledger", config);
+ final RangeEntryCacheImpl entryCache = (RangeEntryCacheImpl) ml.entryCache;
+ final RangeEntryCacheManagerImpl rangeEntryCacheManager =
+ (RangeEntryCacheManagerImpl) factory.getEntryCacheManager();
+ final InflightReadsLimiter limiter = rangeEntryCacheManager.getInflightReadsLimiter();
+ final long totalCapacity =limiter.getRemainingBytes();
+ // final ManagedCursorImpl c1 = (ManagedCursorImpl) ml.openCursor("c1");
+ for (byte i = 1; i < 127; i++) {
+ log.info("add entry: " + i);
+ ml.addEntry(new byte[]{i});
+ }
+ // Evict cached entries.
+ entryCache.evictEntries(ml.currentLedgerSize);
+ Assert.assertEquals(entryCache.getSize(), 0);
+
+ CountDownLatch readCompleteSignal1 = new CountDownLatch(1);
+ CountDownLatch readCompleteSignal2 = new CountDownLatch(1);
+ CountDownLatch firstReadingStarted = new CountDownLatch(1);
+ LedgerHandle currentLedger = ml.currentLedger;
+ LedgerHandle spyCurrentLedger = Mockito.spy(currentLedger);
+ ml.currentLedger = spyCurrentLedger;
+ Answer answer = invocation -> {
+ long firstEntry = (long) invocation.getArguments()[0];
+ log.info("reading entry: {}", firstEntry);
+ if (firstEntry == start1) {
+ // Wait 3s to make
+ firstReadingStarted.countDown();
+ readCompleteSignal1.await();
+ Object res = invocation.callRealMethod();
+ return res;
+ } else if(secondReadEntries.contains(firstEntry)) {
+ final CompletableFuture res = new CompletableFuture<>();
+ threadFactory.newThread(() -> {
+ try {
+ readCompleteSignal2.await();
+ CompletableFuture future =
+ (CompletableFuture) invocation.callRealMethod();
+ future.thenAccept(v -> {
+ res.complete(v);
+ }).exceptionally(ex -> {
+ res.completeExceptionally(ex);
+ return null;
+ });
+ } catch (Throwable ex) {
+ res.completeExceptionally(ex);
+ }
+ }).start();
+ return res;
+ } else {
+ return invocation.callRealMethod();
+ }
+ };
+ doAnswer(answer).when(spyCurrentLedger).readAsync(anyLong(), anyLong());
+ doAnswer(answer).when(spyCurrentLedger).readUnconfirmedAsync(anyLong(), anyLong());
+
+ // Initialize "entryCache.estimatedEntrySize" to the correct value.
+ Object ctx = new Object();
+ SimpleReadEntriesCallback cb0 = new SimpleReadEntriesCallback();
+ entryCache.asyncReadEntry(spyCurrentLedger, 125, 125, true, cb0, ctx);
+ cb0.entries.join();
+ Long sizePerEntry1 = entryCache.getEstimatedEntrySize();
+ Assert.assertEquals(sizePerEntry1, 1 + RangeEntryCacheImpl.BOOKKEEPER_READ_OVERHEAD_PER_ENTRY);
+ Awaitility.await().untilAsserted(() -> {
+ long remainingBytes =limiter.getRemainingBytes();
+ Assert.assertEquals(remainingBytes, totalCapacity);
+ });
+ log.info("remainingBytes 0: {}", limiter.getRemainingBytes());
+
+ // Concurrency reading.
+
+ SimpleReadEntriesCallback cb1 = new SimpleReadEntriesCallback();
+ SimpleReadEntriesCallback cb2 = new SimpleReadEntriesCallback();
+ threadFactory.newThread(() -> {
+ entryCache.asyncReadEntry(spyCurrentLedger, start1, end1, true, cb1, ctx);
+ }).start();
+ threadFactory.newThread(() -> {
+ try {
+ firstReadingStarted.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ entryCache.asyncReadEntry(spyCurrentLedger, start2, end2, true, cb2, ctx);
+ }).start();
+
+ long bytesAcquired1 = calculateBytesSizeBeforeFirstReading(readCount1 + readCount2, 1);
+ long remainingBytesExpected1 = totalCapacity - bytesAcquired1;
+ log.info("acquired : {}", bytesAcquired1);
+ log.info("remainingBytesExpected 0 : {}", remainingBytesExpected1);
+ Awaitility.await().untilAsserted(() -> {
+ log.info("remainingBytes 0: {}", limiter.getRemainingBytes());
+ Assert.assertEquals(limiter.getRemainingBytes(), remainingBytesExpected1);
+ });
+
+ // Complete the read1.
+ Thread.sleep(3000);
+ readCompleteSignal1.countDown();
+ cb1.entries.join();
+ Long sizePerEntry2 = entryCache.getEstimatedEntrySize();
+ Assert.assertEquals(sizePerEntry2, 1 + RangeEntryCacheImpl.BOOKKEEPER_READ_OVERHEAD_PER_ENTRY);
+ long bytesAcquired2 = calculateBytesSizeBeforeFirstReading(readCount2, 1);
+ long remainingBytesExpected2 = totalCapacity - bytesAcquired2;
+ log.info("acquired : {}", bytesAcquired2);
+ log.info("remainingBytesExpected 1: {}", remainingBytesExpected2);
+ Awaitility.await().untilAsserted(() -> {
+ log.info("remainingBytes 1: {}", limiter.getRemainingBytes());
+ Assert.assertEquals(limiter.getRemainingBytes(), remainingBytesExpected2);
+ });
+
+ readCompleteSignal2.countDown();
+ cb2.entries.join();
+ Long sizePerEntry3 = entryCache.getEstimatedEntrySize();
+ Assert.assertEquals(sizePerEntry3, 1 + RangeEntryCacheImpl.BOOKKEEPER_READ_OVERHEAD_PER_ENTRY);
+ Awaitility.await().untilAsserted(() -> {
+ long remainingBytes = limiter.getRemainingBytes();
+ log.info("remainingBytes 2: {}", remainingBytes);
+ Assert.assertEquals(remainingBytes, totalCapacity);
+ });
+ // cleanup
+ ml.delete();
+ factory.shutdown();
+ }
+
+ private long calculateBytesSizeBeforeFirstReading(int entriesCount, int perEntrySize) {
+ return entriesCount * (perEntrySize + RangeEntryCacheImpl.BOOKKEEPER_READ_OVERHEAD_PER_ENTRY);
+ }
+
+ class SimpleReadEntriesCallback implements AsyncCallbacks.ReadEntriesCallback {
+
+ CompletableFuture> entries = new CompletableFuture<>();
+
+ @Override
+ public void readEntriesComplete(List entriesRead, Object ctx) {
+ List list = new ArrayList<>(entriesRead.size());
+ for (Entry entry : entriesRead) {
+ byte b = entry.getDataBuffer().readByte();
+ list.add(b);
+ entry.release();
+ }
+ this.entries.complete(list);
+ }
+
+ @Override
+ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
+ this.entries.completeExceptionally(exception);
+ }
+ }
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index 8913c4013b4ab..d3ea98131ad8f 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -70,11 +70,14 @@
import java.util.function.Supplier;
import java.util.stream.Collectors;
import lombok.Cleanup;
+import org.apache.bookkeeper.client.AsyncCallback.OpenCallback;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.api.ReadHandle;
+import org.apache.bookkeeper.client.PulsarMockBookKeeper;
+import org.apache.bookkeeper.common.util.OrderedExecutor;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
@@ -98,6 +101,7 @@
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo;
import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.pulsar.common.api.proto.CommandSubscribe;
import org.apache.pulsar.common.api.proto.IntRange;
import org.apache.pulsar.common.util.FutureUtil;
@@ -1251,6 +1255,34 @@ void cursorPersistence() throws Exception {
assertEquals(c2.getMarkDeletedPosition(), p2);
}
+ @Test(timeOut = 20000)
+ public void cursorPersistenceWithLedgerForceRecovery() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setLedgerForceRecovery(true);
+
+ ManagedLedger ledger = factory.open("my_test_ledger", config);
+ ManagedCursor c1 = ledger.openCursor("c1");
+ ledger.addEntry("dummy-entry-1".getBytes(Encoding));
+ ledger.addEntry("dummy-entry-2".getBytes(Encoding));
+ ledger.addEntry("dummy-entry-3".getBytes(Encoding));
+
+ List entries = c1.readEntries(2);
+ Position p1 = entries.get(1).getPosition();
+ c1.markDelete(p1);
+ entries.forEach(Entry::release);
+
+ entries = c1.readEntries(1);
+ entries.forEach(Entry::release);
+
+ // Reopen
+ @Cleanup("shutdown")
+ ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(metadataStore, bkc);
+ ledger = factory2.open("my_test_ledger", config);
+ c1 = ledger.openCursor("c1");
+
+ assertEquals(c1.getMarkDeletedPosition(), p1);
+ }
+
@Test(timeOut = 20000)
void cursorPersistence2() throws Exception {
ManagedLedger ledger = factory.open("my_test_ledger",
@@ -3223,7 +3255,7 @@ public void testOutOfOrderDeletePersistenceIntoLedgerWithClose() throws Exceptio
managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore(10);
ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName);
+ final ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName);
List addedPositions = new ArrayList<>();
for (int i = 0; i < totalAddEntries; i++) {
@@ -3269,7 +3301,8 @@ public void operationFailed(MetaStoreException e) {
LedgerEntry entry = seq.nextElement();
PositionInfo positionInfo;
positionInfo = PositionInfo.parseFrom(entry.getEntry());
- individualDeletedMessagesCount.set(positionInfo.getIndividualDeletedMessagesCount());
+ c1.recoverIndividualDeletedMessages(positionInfo);
+ individualDeletedMessagesCount.set(c1.getIndividuallyDeletedMessagesSet().asRanges().size());
} catch (Exception e) {
}
latch.countDown();
@@ -3286,12 +3319,12 @@ public void operationFailed(MetaStoreException e) {
@Cleanup("shutdown")
ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(metadataStore, bkc);
ledger = (ManagedLedgerImpl) factory2.open(ledgerName, managedLedgerConfig);
- c1 = (ManagedCursorImpl) ledger.openCursor("c1");
+ ManagedCursorImpl reopenCursor = (ManagedCursorImpl) ledger.openCursor("c1");
// verify cursor has been recovered
- assertEquals(c1.getNumberOfEntriesInBacklog(false), totalAddEntries / 2);
+ assertEquals(reopenCursor.getNumberOfEntriesInBacklog(false), totalAddEntries / 2);
// try to read entries which should only read non-deleted positions
- List entries = c1.readEntries(totalAddEntries);
+ List entries = reopenCursor.readEntries(totalAddEntries);
assertEquals(entries.size(), totalAddEntries / 2);
}
@@ -4537,7 +4570,6 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
ledger.close();
}
-
@Test
public void testReadEntriesWithSkipDeletedEntries() throws Exception {
@Cleanup
@@ -4734,6 +4766,53 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
&& cursorReadPosition.getEntryId() == expectReadPosition.getEntryId());
}
+ @Test
+ public void testSkipNonRecoverableEntries() throws ManagedLedgerException, InterruptedException {
+ ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
+ int maxMessagePerLedger = 10;
+ managedLedgerConfig.setMaxEntriesPerLedger(maxMessagePerLedger);
+ ManagedLedger ledger = factory.open("testSkipNonRecoverableEntries", managedLedgerConfig);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("my-cursor");
+
+ Position lacPosition = ledger.getLastConfirmedEntry();
+ long ledgerId = lacPosition.getLedgerId();
+ assertEquals(PositionFactory.create(ledgerId, -1), cursor.getMarkDeletedPosition());
+
+ // Mock add 10 entry
+ for (int i = 0; i < 10; i++) {
+ ledger.addEntry(String.valueOf(i).getBytes());
+ }
+
+ // read 2 entry and delete these entries, MarkDeletedPosition move forward
+ List entries = cursor.readEntries(2);
+ for (Entry entry : entries) {
+ cursor.delete(entry.getPosition());
+ }
+ assertEquals(PositionFactory.create(ledgerId, 1), cursor.getMarkDeletedPosition());
+
+ // read the next 6 entry and not delete, MarkDeletedPosition not move forward
+ entries = cursor.readEntries(6);
+ assertEquals(PositionFactory.create(ledgerId, 1), cursor.getMarkDeletedPosition());
+
+ // delete last read entry, MarkDeletedPosition not move forward
+ Entry lastEntry = entries.get(entries.size() - 1);
+ cursor.delete(lastEntry.getPosition());
+ assertEquals(PositionFactory.create(ledgerId, 1), cursor.getMarkDeletedPosition());
+
+ // call skip entries, MarkDeletedPosition move forward
+ cursor.skipNonRecoverableEntries(cursor.getMarkDeletedPosition(),
+ PositionFactory.create(ledgerId, lastEntry.getEntryId()));
+ assertEquals(PositionFactory.create(ledgerId, lastEntry.getEntryId()), cursor.getMarkDeletedPosition());
+
+ // repeat call skip entries, MarkDeletedPosition not change
+ cursor.skipNonRecoverableEntries(cursor.getMarkDeletedPosition(),
+ PositionFactory.create(ledgerId, lastEntry.getEntryId()));
+ assertEquals(PositionFactory.create(ledgerId, lastEntry.getEntryId()), cursor.getMarkDeletedPosition());
+
+ cursor.close();
+ ledger.close();
+ }
+
@Test
public void testRecoverCursorWithTerminateManagedLedger() throws Exception {
String mlName = "my_test_ledger";
@@ -4794,5 +4873,349 @@ public void operationFailed(ManagedLedgerException exception) {
assertEquals(cursor.getReadPosition(), markDeletedPosition.getNext());
}
+ @Test
+ public void testFindNewestMatching_SearchAllAvailableEntries_ByStartAndEnd() throws Exception {
+ ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
+ managedLedgerConfig.setMaxEntriesPerLedger(2);
+ managedLedgerConfig.setMinimumRolloverTime(0, TimeUnit.MILLISECONDS);
+ @Cleanup
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testFindNewestMatching_SearchAllAvailableEntries_ByStartAndEnd", managedLedgerConfig);
+ @Cleanup
+ ManagedCursor managedCursor = ledger.openCursor("test");
+
+ Position position = ledger.addEntry("test".getBytes(Encoding));
+ Position position1 = ledger.addEntry("test1".getBytes(Encoding));
+ Position position2 = ledger.addEntry("test2".getBytes(Encoding));
+ Position position3 = ledger.addEntry("test3".getBytes(Encoding));
+
+ Predicate condition = entry -> {
+ try {
+ Position p = entry.getPosition();
+ return p.compareTo(position1) <= 0;
+ } finally {
+ entry.release();
+ }
+ };
+
+ // find the newest entry with start and end position
+ AtomicBoolean failed = new AtomicBoolean(false);
+ CountDownLatch latch = new CountDownLatch(1);
+ AtomicReference positionRef = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries, condition, position, position2, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef.set(position);
+ latch.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed.set(true);
+ latch.countDown();
+ }
+ }, null, true);
+
+ latch.await();
+ assertFalse(failed.get());
+ assertNotNull(positionRef.get());
+ assertEquals(positionRef.get(), position1);
+
+ // find the newest entry with start
+ AtomicBoolean failed1 = new AtomicBoolean(false);
+ CountDownLatch latch1 = new CountDownLatch(1);
+ AtomicReference positionRef1 = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries, condition, position, null, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef1.set(position);
+ latch1.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed1.set(true);
+ latch1.countDown();
+ }
+ }, null, true);
+ latch1.await();
+ assertFalse(failed1.get());
+ assertNotNull(positionRef1.get());
+ assertEquals(positionRef1.get(), position1);
+
+ // find the newest entry with end
+ AtomicBoolean failed2 = new AtomicBoolean(false);
+ CountDownLatch latch2 = new CountDownLatch(1);
+ AtomicReference positionRef2 = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries, condition, null, position2, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef2.set(position);
+ latch2.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed2.set(true);
+ latch2.countDown();
+ }
+ }, null, true);
+ latch2.await();
+ assertFalse(failed2.get());
+ assertNotNull(positionRef2.get());
+ assertEquals(positionRef2.get(), position1);
+
+ // find the newest entry without start and end position
+ AtomicBoolean failed3 = new AtomicBoolean(false);
+ CountDownLatch latch3 = new CountDownLatch(1);
+ AtomicReference positionRef3 = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries, condition, null, null, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef3.set(position);
+ latch3.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed3.set(true);
+ latch3.countDown();
+ }
+ }, null, true);
+ latch3.await();
+ assertFalse(failed3.get());
+ assertNotNull(positionRef3.get());
+ assertEquals(positionRef3.get(), position1);
+
+ // find position3
+ AtomicBoolean failed4 = new AtomicBoolean(false);
+ CountDownLatch latch4 = new CountDownLatch(1);
+ AtomicReference positionRef4 = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries, entry -> {
+ try {
+ Position p = entry.getPosition();
+ return p.compareTo(position3) <= 0;
+ } finally {
+ entry.release();
+ }
+ }, position3, position3, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef4.set(position);
+ latch4.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed4.set(true);
+ latch4.countDown();
+ }
+ }, null, true);
+ latch4.await();
+ assertFalse(failed4.get());
+ assertNotNull(positionRef4.get());
+ assertEquals(positionRef4.get(), position3);
+ }
+
+
+ @Test
+ public void testFindNewestMatching_SearchActiveEntries_ByStartAndEnd() throws Exception {
+ ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
+ managedLedgerConfig.setMaxEntriesPerLedger(2);
+ managedLedgerConfig.setMinimumRolloverTime(0, TimeUnit.MILLISECONDS);
+ @Cleanup
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testFindNewestMatching_SearchActiveEntries_ByStartAndEnd", managedLedgerConfig);
+ @Cleanup
+ ManagedCursorImpl managedCursor = (ManagedCursorImpl) ledger.openCursor("test");
+
+ Position position = ledger.addEntry("test".getBytes(Encoding));
+ Position position1 = ledger.addEntry("test1".getBytes(Encoding));
+ Position position2 = ledger.addEntry("test2".getBytes(Encoding));
+ Position position3 = ledger.addEntry("test3".getBytes(Encoding));
+ Position position4 = ledger.addEntry("test4".getBytes(Encoding));
+ managedCursor.markDelete(position1);
+ assertEquals(managedCursor.getNumberOfEntries(), 3);
+
+ Predicate condition = entry -> {
+ try {
+ Position p = entry.getPosition();
+ return p.compareTo(position3) <= 0;
+ } finally {
+ entry.release();
+ }
+ };
+
+ // find the newest entry with start and end position
+ AtomicBoolean failed = new AtomicBoolean(false);
+ CountDownLatch latch = new CountDownLatch(1);
+ AtomicReference positionRef = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, condition, position2, position4, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef.set(position);
+ latch.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed.set(true);
+ latch.countDown();
+ }
+ }, null, true);
+ latch.await();
+ assertFalse(failed.get());
+ assertNotNull(positionRef.get());
+ assertEquals(positionRef.get(), position3);
+
+ // find the newest entry with start
+ AtomicBoolean failed1 = new AtomicBoolean(false);
+ CountDownLatch latch1 = new CountDownLatch(1);
+ AtomicReference positionRef1 = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, condition, position2, null, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef1.set(position);
+ latch1.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed1.set(true);
+ latch1.countDown();
+ }
+ }, null, true);
+
+ latch1.await();
+ assertFalse(failed1.get());
+ assertNotNull(positionRef1.get());
+ assertEquals(positionRef1.get(), position3);
+
+ // find the newest entry with end
+ AtomicBoolean failed2 = new AtomicBoolean(false);
+ CountDownLatch latch2 = new CountDownLatch(1);
+ AtomicReference positionRef2 = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, condition, null, position4, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef2.set(position);
+ latch2.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed2.set(true);
+ latch2.countDown();
+ }
+ }, null, true);
+
+ latch2.await();
+ assertFalse(failed2.get());
+ assertNotNull(positionRef2.get());
+ assertEquals(positionRef2.get(), position3);
+
+ // find the newest entry without start and end position
+ AtomicBoolean failed3 = new AtomicBoolean(false);
+ CountDownLatch latch3 = new CountDownLatch(1);
+ AtomicReference positionRef3 = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, condition, null, null, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef3.set(position);
+ latch3.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed3.set(true);
+ latch3.countDown();
+ }
+ }, null, true);
+ latch3.await();
+ assertFalse(failed3.get());
+ assertNotNull(positionRef3.get());
+ assertEquals(positionRef3.get(), position3);
+
+ // find position4
+ AtomicBoolean failed4 = new AtomicBoolean(false);
+ CountDownLatch latch4 = new CountDownLatch(1);
+ AtomicReference positionRef4 = new AtomicReference<>();
+ managedCursor.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchActiveEntries, entry -> {
+ try {
+ Position p = entry.getPosition();
+ return p.compareTo(position4) <= 0;
+ } finally {
+ entry.release();
+ }
+ }, position4, position4, new AsyncCallbacks.FindEntryCallback() {
+ @Override
+ public void findEntryComplete(Position position, Object ctx) {
+ positionRef4.set(position);
+ latch4.countDown();
+ }
+
+ @Override
+ public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, Object ctx) {
+ failed4.set(true);
+ latch4.countDown();
+ }
+ }, null, true);
+ latch4.await();
+ assertFalse(failed4.get());
+ assertNotNull(positionRef4.get());
+ assertEquals(positionRef4.get(), position4);
+ }
+
+ @Test
+ void testForceCursorRecovery() throws Exception {
+ TestPulsarMockBookKeeper bk = new TestPulsarMockBookKeeper(executor);
+ factory = new ManagedLedgerFactoryImpl(metadataStore, bk);
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setLedgerForceRecovery(true);
+ ManagedLedger ledger = factory.open("my_test_ledger", config);
+ ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
+ ledger.addEntry("entry-1".getBytes(Encoding));
+ long invalidLedger = -1L;
+ bk.setErrorCodeMap(invalidLedger, BKException.Code.BookieHandleNotAvailableException);
+ ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(invalidLedger).build();
+ CountDownLatch latch = new CountDownLatch(1);
+ MutableBoolean recovered = new MutableBoolean(false);
+ VoidCallback callback = new VoidCallback() {
+ @Override
+ public void operationComplete() {
+ recovered.setValue(true);
+ latch.countDown();
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException exception) {
+ recovered.setValue(false);
+ latch.countDown();
+ }
+ };
+ c1.recoverFromLedger(info, callback);
+ latch.await();
+ assertTrue(recovered.booleanValue());
+ }
+
+ class TestPulsarMockBookKeeper extends PulsarMockBookKeeper {
+ Map ledgerErrors = new HashMap<>();
+
+ public TestPulsarMockBookKeeper(OrderedExecutor orderedExecutor) throws Exception {
+ super(orderedExecutor);
+ }
+
+ public void setErrorCodeMap(long ledgerId, int rc) {
+ ledgerErrors.put(ledgerId, rc);
+ }
+
+ public void asyncOpenLedger(final long lId, final DigestType digestType, final byte[] passwd,
+ final OpenCallback cb, final Object ctx) {
+ if (ledgerErrors.containsKey(lId)) {
+ cb.openComplete(ledgerErrors.get(lId), null, ctx);
+ } else {
+ super.asyncOpenLedger(lId, digestType, passwd, cb, ctx);
+ }
+ }
+ }
+
private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class);
}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
index cd1dcf05c3708..574ed2f325136 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
@@ -20,24 +20,32 @@
import static org.apache.pulsar.common.util.PortManager.releaseLockedPort;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.netty.buffer.ByteBuf;
-
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.BookKeeperTestClient;
+import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
import org.apache.bookkeeper.mledger.Entry;
@@ -49,15 +57,22 @@
import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
import org.apache.bookkeeper.mledger.impl.cache.EntryCacheManager;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats;
import org.apache.bookkeeper.mledger.util.ThrowableToStringUtil;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
+import org.apache.pulsar.common.util.FutureUtil;
import org.awaitility.Awaitility;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+@Slf4j
public class ManagedLedgerBkTest extends BookKeeperClusterTestCase {
+ private final ObjectMapper jackson = new ObjectMapper();
+
public ManagedLedgerBkTest() {
super(2);
}
@@ -229,6 +244,108 @@ public void verifyConcurrentUsage() throws Exception {
assertEquals(factory.getMbean().getNumberOfCacheEvictions(), 0);
}
+ @Test
+ public void verifyAsyncReadEntryUsingCache() throws Exception {
+ ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
+
+ config.setMaxCacheSize(100 * 1024 * 1024);
+ config.setCacheEvictionTimeThresholdMillis(10000);
+ config.setCacheEvictionIntervalMs(10000);
+
+ @Cleanup("shutdown")
+ ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, config);
+
+ ManagedLedgerConfig conf = new ManagedLedgerConfig();
+ conf.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2)
+ .setRetentionSizeInMB(-1).setRetentionTime(-1, TimeUnit.MILLISECONDS);
+ final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-ledger" + testName, conf);
+
+ int NumProducers = 5;
+ int NumConsumers = 10;
+
+ final AtomicBoolean done = new AtomicBoolean();
+ final CyclicBarrier barrier = new CyclicBarrier(NumProducers + NumConsumers + 1);
+
+ List> futures = new ArrayList();
+ List positions = new CopyOnWriteArrayList<>();
+
+ for (int i = 0; i < NumProducers; i++) {
+ futures.add(executor.submit(() -> {
+ try {
+ // wait for all threads to be ready to start at once
+ barrier.await();
+ while (!done.get()) {
+ Position position = ledger.addEntry("entry".getBytes());
+ positions.add(position);
+ Thread.sleep(1);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw FutureUtil.wrapToCompletionException(e);
+ }
+ }));
+ }
+
+ // create a dummy cursor since caching happens only when there are active consumers
+ ManagedCursor cursor = ledger.openCursor("dummy");
+
+ for (int i = 0; i < NumConsumers; i++) {
+ futures.add(executor.submit(() -> {
+ try {
+ // wait for all threads to be ready to start at once
+ barrier.await();
+ while (!done.get()) {
+ if (positions.isEmpty()) {
+ Thread.sleep(1);
+ continue;
+ }
+ // Simulate a replay queue read pattern where individual entries are read
+ Position randomPosition = positions.get(ThreadLocalRandom.current().nextInt(positions.size()));
+ // Clone the original instance so that another instance is used in the asyncReadEntry call
+ // This is to test that keys are compared by .equals and not by reference under the covers
+ randomPosition = PositionFactory.create(randomPosition);
+ CompletableFuture future = new CompletableFuture<>();
+ ledger.asyncReadEntry(randomPosition, new AsyncCallbacks.ReadEntryCallback() {
+ @Override
+ public void readEntryComplete(Entry entry, Object ctx) {
+ entry.release();
+ future.complete(null);
+ }
+
+ @Override
+ public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
+ future.completeExceptionally(exception);
+ }
+ }, null);
+ future.get();
+ Thread.sleep(2);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw FutureUtil.wrapToCompletionException(e);
+ }
+ }));
+ }
+
+ // trigger all worker threads at once to continue from the barrier
+ barrier.await();
+
+ int testDurationSeconds = 3;
+ Thread.sleep(testDurationSeconds * 1000);
+
+ done.set(true);
+ for (Future future : futures) {
+ future.get();
+ }
+
+ factory.getMbean().refreshStats(testDurationSeconds, TimeUnit.SECONDS);
+
+ assertTrue(factory.getMbean().getCacheHitsRate() > 0.0);
+ assertEquals(factory.getMbean().getCacheMissesRate(), 0.0);
+ assertTrue(factory.getMbean().getCacheHitsThroughput() > 0.0);
+ assertEquals(factory.getMbean().getNumberOfCacheEvictions(), 0);
+ }
+
@Test
public void testSimple() throws Exception {
@Cleanup("shutdown")
@@ -587,4 +704,114 @@ public void testPeriodicRollover() throws Exception {
Awaitility.await().until(() -> cursorImpl.getCursorLedger() != currentLedgerId);
}
+ @DataProvider(name = "unackedRangesOpenCacheSetEnabledPair")
+ public Object[][] unackedRangesOpenCacheSetEnabledPair() {
+ return new Object[][]{
+ {false, true},
+ {true, false},
+ {true, true},
+ {false, false}
+ };
+ }
+
+ /**
+ * This test validates that cursor serializes and deserializes individual-ack list from the bk-ledger.
+ * @throws Exception
+ */
+ @Test(dataProvider = "unackedRangesOpenCacheSetEnabledPair")
+ public void testUnackmessagesAndRecoveryCompatibility(boolean enabled1, boolean enabled2) throws Exception {
+ final String mlName = "ml" + UUID.randomUUID().toString().replaceAll("-", "");
+ final String cursorName = "c1";
+ ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
+ ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf);
+ final ManagedLedgerConfig config1 = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1)
+ .setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1)
+ .setMaxUnackedRangesToPersistInMetadataStore(1).setMaxEntriesPerLedger(5).setMetadataAckQuorumSize(1)
+ .setUnackedRangesOpenCacheSetEnabled(enabled1);
+ final ManagedLedgerConfig config2 = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1)
+ .setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1)
+ .setMaxUnackedRangesToPersistInMetadataStore(1).setMaxEntriesPerLedger(5).setMetadataAckQuorumSize(1)
+ .setUnackedRangesOpenCacheSetEnabled(enabled2);
+
+ ManagedLedger ledger1 = factory.open(mlName, config1);
+ ManagedCursorImpl cursor1 = (ManagedCursorImpl) ledger1.openCursor(cursorName);
+
+ int totalEntries = 100;
+ for (int i = 0; i < totalEntries; i++) {
+ Position p = ledger1.addEntry("entry".getBytes());
+ if (i % 2 == 0) {
+ cursor1.delete(p);
+ }
+ }
+ log.info("ack ranges: {}", cursor1.getIndividuallyDeletedMessagesSet().size());
+
+ // reopen and recover cursor
+ ledger1.close();
+ ManagedLedger ledger2 = factory.open(mlName, config2);
+ ManagedCursorImpl cursor2 = (ManagedCursorImpl) ledger2.openCursor(cursorName);
+
+ log.info("before: {}", cursor1.getIndividuallyDeletedMessagesSet().asRanges());
+ log.info("after : {}", cursor2.getIndividuallyDeletedMessagesSet().asRanges());
+ assertEquals(cursor1.getIndividuallyDeletedMessagesSet().asRanges(), cursor2.getIndividuallyDeletedMessagesSet().asRanges());
+ assertEquals(cursor1.markDeletePosition, cursor2.markDeletePosition);
+
+ ledger2.close();
+ factory.shutdown();
+ }
+
+ @DataProvider(name = "booleans")
+ public Object[][] booleans() {
+ return new Object[][] {
+ {true},
+ {false},
+ };
+ }
+
+ @Test(dataProvider = "booleans")
+ public void testConfigPersistIndividualAckAsLongArray(boolean enable) throws Exception {
+ final String mlName = "ml" + UUID.randomUUID().toString().replaceAll("-", "");
+ final String cursorName = "c1";
+ ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
+ ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf);
+ final ManagedLedgerConfig config = new ManagedLedgerConfig()
+ .setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1)
+ .setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1).setMetadataAckQuorumSize(1)
+ .setMaxUnackedRangesToPersistInMetadataStore(1)
+ .setUnackedRangesOpenCacheSetEnabled(true).setPersistIndividualAckAsLongArray(enable);
+
+ ManagedLedger ledger1 = factory.open(mlName, config);
+ ManagedCursorImpl cursor1 = (ManagedCursorImpl) ledger1.openCursor(cursorName);
+
+ // Write entries.
+ int totalEntries = 100;
+ List entries = new ArrayList<>();
+ for (int i = 0; i < totalEntries; i++) {
+ Position p = ledger1.addEntry("entry".getBytes());
+ entries.add(p);
+ }
+ // Make ack holes and trigger a mark deletion.
+ for (int i = totalEntries - 1; i >=0 ; i--) {
+ if (i % 2 == 0) {
+ cursor1.delete(entries.get(i));
+ }
+ }
+ cursor1.markDelete(entries.get(9));
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(cursor1.pendingMarkDeleteOps.size(), 0);
+ });
+
+ // Verify: the config affects.
+ long cursorLedgerLac = cursor1.cursorLedger.getLastAddConfirmed();
+ LedgerEntry ledgerEntry = cursor1.cursorLedger.readEntries(cursorLedgerLac, cursorLedgerLac).nextElement();
+ MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry());
+ if (enable) {
+ assertNotEquals(positionInfo.getIndividualDeletedMessageRangesList().size(), 0);
+ } else {
+ assertEquals(positionInfo.getIndividualDeletedMessageRangesList().size(), 0);
+ }
+
+ // cleanup
+ ledger1.close();
+ factory.shutdown();
+ }
}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java
index 7b2f8228ad722..d72bffa27d30a 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java
@@ -31,12 +31,14 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import lombok.Cleanup;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.api.DigestType;
+import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback;
import org.apache.bookkeeper.mledger.Entry;
@@ -509,6 +511,35 @@ public void recoverAfterWriteError() throws Exception {
entries.forEach(Entry::release);
}
+ @Test
+ public void recoverAfterOpenManagedLedgerFail() throws Exception {
+ ManagedLedger ledger = factory.open("recoverAfterOpenManagedLedgerFail");
+ Position position = ledger.addEntry("entry".getBytes());
+ ledger.close();
+ bkc.failAfter(0, BKException.Code.BookieHandleNotAvailableException);
+ try {
+ factory.open("recoverAfterOpenManagedLedgerFail");
+ } catch (Exception e) {
+ // ok
+ }
+
+ ledger = factory.open("recoverAfterOpenManagedLedgerFail");
+ CompletableFuture future = new CompletableFuture<>();
+ ledger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() {
+ @Override
+ public void readEntryComplete(Entry entry, Object ctx) {
+ future.complete(entry.getData());
+ }
+
+ @Override
+ public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
+ future.completeExceptionally(exception);
+ }
+ }, null);
+ byte[] bytes = future.get(30, TimeUnit.SECONDS);
+ assertEquals(new String(bytes), "entry");
+ }
+
@Test
public void recoverLongTimeAfterMultipleWriteErrors() throws Exception {
ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("recoverLongTimeAfterMultipleWriteErrors");
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
index 83a6c771513a9..5ec453a6d4e69 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
@@ -94,6 +94,7 @@
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.PulsarMockBookKeeper;
import org.apache.bookkeeper.client.PulsarMockLedgerHandle;
+import org.apache.bookkeeper.client.PulsarMockReadHandleInterceptor;
import org.apache.bookkeeper.client.api.LedgerEntries;
import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.client.api.ReadHandle;
@@ -3133,17 +3134,26 @@ public void testManagedLedgerWithReadEntryTimeOut() throws Exception {
ManagedLedgerConfig config = new ManagedLedgerConfig().setReadEntryTimeoutSeconds(1);
ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("timeout_ledger_test", config);
- BookKeeper bk = mock(BookKeeper.class);
- doNothing().when(bk).asyncCreateLedger(anyInt(), anyInt(), anyInt(), any(), any(), any(), any(), any());
+ Position position = ledger.addEntry("entry-1".getBytes());
+
+ // ensure that the read isn't cached
+ factory.getEntryCacheManager().clear();
+
+ bkc.setReadHandleInterceptor(new PulsarMockReadHandleInterceptor() {
+ @Override
+ public CompletableFuture interceptReadAsync(long ledgerId, long firstEntry, long lastEntry,
+ LedgerEntries entries) {
+ return CompletableFuture.supplyAsync(() -> {
+ return entries;
+ }, CompletableFuture.delayedExecutor(3, TimeUnit.SECONDS));
+ }
+ });
+
AtomicReference responseException1 = new AtomicReference<>();
String ctxStr = "timeoutCtx";
- CompletableFuture entriesFuture = new CompletableFuture<>();
- ReadHandle ledgerHandle = mock(ReadHandle.class);
- doReturn(entriesFuture).when(ledgerHandle).readAsync(PositionFactory.EARLIEST.getLedgerId(),
- PositionFactory.EARLIEST.getEntryId());
// (1) test read-timeout for: ManagedLedger.asyncReadEntry(..)
- ledger.asyncReadEntry(ledgerHandle, PositionFactory.EARLIEST, new ReadEntryCallback() {
+ ledger.asyncReadEntry(position, new ReadEntryCallback() {
@Override
public void readEntryComplete(Entry entry, Object ctx) {
responseException1.set(null);
@@ -3155,18 +3165,20 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
responseException1.set(exception);
}
}, ctxStr);
- ledger.asyncCreateLedger(bk, config, null, (rc, lh, ctx) -> {}, Collections.emptyMap());
- retryStrategically((test) -> responseException1.get() != null, 5, 1000);
- assertNotNull(responseException1.get());
- assertTrue(responseException1.get().getMessage()
- .startsWith(BKException.getMessage(BKException.Code.TimeoutException)));
- // (2) test read-timeout for: ManagedLedger.asyncReadEntry(..)
- AtomicReference responseException2 = new AtomicReference<>();
- Position readPositionRef = PositionFactory.EARLIEST;
- ManagedCursorImpl cursor = new ManagedCursorImpl(bk, ledger, "cursor1");
- OpReadEntry opReadEntry = OpReadEntry.create(cursor, readPositionRef, 1, new ReadEntriesCallback() {
+ Awaitility.await().untilAsserted(() -> {
+ assertNotNull(responseException1.get());
+ assertTrue(responseException1.get().getMessage()
+ .startsWith(BKException.getMessage(BKException.Code.TimeoutException)));
+ });
+ // ensure that the read isn't cached
+ factory.getEntryCacheManager().clear();
+
+ // (2) test read-timeout for: ManagedCursor.asyncReadEntries(..)
+ AtomicReference responseException2 = new AtomicReference<>();
+ ManagedCursor cursor = ledger.openCursor("cursor1", InitialPosition.Earliest);
+ cursor.asyncReadEntries(1, new ReadEntriesCallback() {
@Override
public void readEntriesComplete(List entries, Object ctx) {
}
@@ -3176,16 +3188,13 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
assertEquals(ctxStr, (String) ctx);
responseException2.set(exception);
}
+ }, ctxStr, PositionFactory.LATEST);
- }, null, PositionFactory.LATEST, null);
- ledger.asyncReadEntry(ledgerHandle, PositionFactory.EARLIEST.getEntryId(), PositionFactory.EARLIEST.getEntryId(),
- opReadEntry, ctxStr);
- retryStrategically((test) -> {
- return responseException2.get() != null;
- }, 5, 1000);
- assertNotNull(responseException2.get());
- assertTrue(responseException2.get().getMessage()
- .startsWith(BKException.getMessage(BKException.Code.TimeoutException)));
+ Awaitility.await().untilAsserted(() -> {
+ assertNotNull(responseException2.get());
+ assertTrue(responseException2.get().getMessage()
+ .startsWith(BKException.getMessage(BKException.Code.TimeoutException)));
+ });
ledger.close();
}
@@ -3723,6 +3732,10 @@ public void testInvalidateReadHandleWhenDeleteLedger() throws Exception {
for (int i = 0; i < entries; i++) {
ledger.addEntry(String.valueOf(i).getBytes(Encoding));
}
+
+ // clear the cache to avoid flakiness
+ factory.getEntryCacheManager().clear();
+
List entryList = cursor.readEntries(3);
assertEquals(entryList.size(), 3);
Awaitility.await().untilAsserted(() -> {
@@ -3791,10 +3804,16 @@ public void testInvalidateReadHandleWhenConsumed() throws Exception {
for (int i = 0; i < entries; i++) {
ledger.addEntry(String.valueOf(i).getBytes(Encoding));
}
- List entryList = cursor.readEntries(3);
- assertEquals(entryList.size(), 3);
- assertEquals(ledger.ledgers.size(), 4);
- assertEquals(ledger.ledgerCache.size(), 3);
+
+ // clear the cache to avoid flakiness
+ factory.getEntryCacheManager().clear();
+
+ final List entryList = cursor.readEntries(3);
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(entryList.size(), 3);
+ assertEquals(ledger.ledgers.size(), 4);
+ assertEquals(ledger.ledgerCache.size(), 3);
+ });
cursor.clearBacklog();
cursor2.clearBacklog();
ledger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE);
@@ -3803,11 +3822,17 @@ public void testInvalidateReadHandleWhenConsumed() throws Exception {
assertEquals(ledger.ledgerCache.size(), 0);
});
+ // clear the cache to avoid flakiness
+ factory.getEntryCacheManager().clear();
+
// Verify the ReadHandle can be reopened.
ManagedCursor cursor3 = ledger.openCursor("test-cursor3", InitialPosition.Earliest);
- entryList = cursor3.readEntries(3);
- assertEquals(entryList.size(), 3);
- assertEquals(ledger.ledgerCache.size(), 3);
+ final List entryList2 = cursor3.readEntries(3);
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(entryList2.size(), 3);
+ assertEquals(ledger.ledgerCache.size(), 3);
+ });
+
cursor3.clearBacklog();
ledger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE);
Awaitility.await().untilAsserted(() -> {
@@ -3815,7 +3840,6 @@ public void testInvalidateReadHandleWhenConsumed() throws Exception {
assertEquals(ledger.ledgerCache.size(), 0);
});
-
cursor.close();
cursor2.close();
cursor3.close();
@@ -4236,6 +4260,31 @@ public void testNonDurableCursorCreateForInactiveLedger() throws Exception {
assertNotNull(ml.newNonDurableCursor(Position));
}
+ @Test(timeOut = 60 * 1000)
+ public void testCreateDataLedgerTimeout() throws Exception {
+ String mlName = UUID.randomUUID().toString();
+ ManagedLedgerFactoryImpl factory = null;
+ ManagedLedger ml = null;
+ try {
+ factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setMetadataOperationsTimeoutSeconds(5);
+ bkc.delay(10 * 1000);
+ ml = factory.open(mlName, config);
+ fail("Should get a timeout ex");
+ } catch (ManagedLedgerException ex) {
+ assertTrue(ex.getMessage().contains("timeout"));
+ } finally {
+ // cleanup.
+ if (ml != null) {
+ ml.delete();
+ }
+ if (factory != null) {
+ factory.shutdown();
+ }
+ }
+ }
+
/***
* When a ML tries to create a ledger, it will create a delay task to check if the ledger create request is timeout.
* But we should guarantee that the delay task should be canceled after the ledger create request responded.
@@ -4361,9 +4410,10 @@ public void testDeleteCurrentLedgerWhenItIsClosed(boolean closeLedgerByAddEntry)
// Detect the current ledger is full by the timed task. (Imitate: the timed task `checkLedgerRollTask` call
// `rollCurrentLedgerIfFull` periodically).
ml.rollCurrentLedgerIfFull();
- // the ledger closing in the `rollCurrentLedgerIfFull` is async, so the wait is needed.
- Awaitility.await().untilAsserted(() -> assertEquals(ml.ledgers.size(), 2));
}
+ // wait the new ledger create
+ Awaitility.await().untilAsserted(() -> assertEquals(ml.ledgers.size(), 2));
+
// Act: Trigger trimming to delete the previous current ledger.
ml.internalTrimLedgers(false, Futures.NULL_PROMISE);
// Verify: A new ledger will be opened after the current ledger is closed and the previous current ledger can be
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiterTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiterTest.java
index 89bdda15afb4b..7475b620f5792 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiterTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/InflightReadsLimiterTest.java
@@ -21,39 +21,51 @@
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
import static org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.InflightReadLimiterUtilization.FREE;
import static org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.InflightReadLimiterUtilization.USED;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
import lombok.Cleanup;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
+import org.assertj.core.api.Assertions;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Slf4j
public class InflightReadsLimiterTest {
+ private static final int ACQUIRE_QUEUE_SIZE = 1000;
+ private static final int ACQUIRE_TIMEOUT_MILLIS = 500;
@DataProvider
private static Object[][] isDisabled() {
- return new Object[][] {
- {0, true},
- {-1, true},
- {1, false},
+ return new Object[][]{
+ {0, true},
+ {-1, true},
+ {1, false},
};
}
+ @DataProvider
+ private static Object[] booleanValues() {
+ return new Object[]{ true, false };
+ }
+
@Test(dataProvider = "isDisabled")
public void testDisabled(long maxReadsInFlightSize, boolean shouldBeDisabled) throws Exception {
var otel = buildOpenTelemetryAndReader();
@Cleanup var openTelemetry = otel.getLeft();
@Cleanup var metricReader = otel.getRight();
- var limiter = new InflightReadsLimiter(maxReadsInFlightSize, openTelemetry);
- assertEquals(limiter.isDisabled(), shouldBeDisabled);
+ var limiter = new InflightReadsLimiter(maxReadsInFlightSize, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS,
+ mock(ScheduledExecutorService.class), openTelemetry);
+ assertThat(limiter.isDisabled()).isEqualTo(shouldBeDisabled);
if (shouldBeDisabled) {
// Verify metrics are not present
@@ -71,141 +83,468 @@ public void testBasicAcquireRelease() throws Exception {
@Cleanup var openTelemetry = otel.getLeft();
@Cleanup var metricReader = otel.getRight();
- InflightReadsLimiter limiter = new InflightReadsLimiter(100, openTelemetry);
- assertEquals(100, limiter.getRemainingBytes());
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(100, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS,
+ mock(ScheduledExecutorService.class), openTelemetry);
+ assertThat(limiter.getRemainingBytes()).isEqualTo(100);
assertLimiterMetrics(metricReader, 100, 0, 100);
- InflightReadsLimiter.Handle handle = limiter.acquire(100, null);
- assertEquals(0, limiter.getRemainingBytes());
- assertTrue(handle.success);
- assertEquals(handle.acquiredPermits, 100);
- assertEquals(1, handle.trials);
+ Optional optionalHandle = limiter.acquire(100, null);
+ assertThat(limiter.getRemainingBytes()).isZero();
+ assertThat(optionalHandle).isPresent();
+ InflightReadsLimiter.Handle handle = optionalHandle.get();
+ assertThat(handle.success()).isTrue();
+ assertThat(handle.permits()).isEqualTo(100);
assertLimiterMetrics(metricReader, 100, 100, 0);
limiter.release(handle);
- assertEquals(100, limiter.getRemainingBytes());
+ assertThat(limiter.getRemainingBytes()).isEqualTo(100);
assertLimiterMetrics(metricReader, 100, 0, 100);
}
-
@Test
public void testNotEnoughPermits() throws Exception {
- InflightReadsLimiter limiter = new InflightReadsLimiter(100, OpenTelemetry.noop());
- assertEquals(100, limiter.getRemainingBytes());
- InflightReadsLimiter.Handle handle = limiter.acquire(100, null);
- assertEquals(0, limiter.getRemainingBytes());
- assertTrue(handle.success);
- assertEquals(handle.acquiredPermits, 100);
- assertEquals(1, handle.trials);
-
- InflightReadsLimiter.Handle handle2 = limiter.acquire(100, null);
- assertEquals(0, limiter.getRemainingBytes());
- assertFalse(handle2.success);
- assertEquals(handle2.acquiredPermits, 0);
- assertEquals(1, handle2.trials);
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(100, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS,
+ mock(ScheduledExecutorService.class), OpenTelemetry.noop());
+ assertThat(limiter.getRemainingBytes()).isEqualTo(100);
+ Optional optionalHandle = limiter.acquire(100, null);
+ assertThat(limiter.getRemainingBytes()).isZero();
+ assertThat(optionalHandle).isPresent();
+ InflightReadsLimiter.Handle handle = optionalHandle.get();
+ assertThat(handle.success()).isTrue();
+ assertThat(handle.permits()).isEqualTo(100);
+
+ AtomicReference handle2Reference = new AtomicReference<>();
+ Optional optionalHandle2 = limiter.acquire(100, handle2Reference::set);
+ assertThat(limiter.getRemainingBytes()).isZero();
+ assertThat(optionalHandle2).isNotPresent();
limiter.release(handle);
- assertEquals(100, limiter.getRemainingBytes());
-
- handle2 = limiter.acquire(100, handle2);
- assertEquals(0, limiter.getRemainingBytes());
- assertTrue(handle2.success);
- assertEquals(handle2.acquiredPermits, 100);
- assertEquals(2, handle2.trials);
-
- limiter.release(handle2);
- assertEquals(100, limiter.getRemainingBytes());
+ assertThat(handle2Reference)
+ .hasValueSatisfying(h ->
+ assertThat(h.success()).isTrue());
+ limiter.release(handle2Reference.get());
+ assertThat(limiter.getRemainingBytes()).isEqualTo(100);
}
@Test
- public void testPartialAcquire() throws Exception {
- InflightReadsLimiter limiter = new InflightReadsLimiter(100, OpenTelemetry.noop());
- assertEquals(100, limiter.getRemainingBytes());
-
- InflightReadsLimiter.Handle handle = limiter.acquire(30, null);
- assertEquals(70, limiter.getRemainingBytes());
- assertTrue(handle.success);
- assertEquals(handle.acquiredPermits, 30);
- assertEquals(1, handle.trials);
-
- InflightReadsLimiter.Handle handle2 = limiter.acquire(100, null);
- assertEquals(0, limiter.getRemainingBytes());
- assertFalse(handle2.success);
- assertEquals(handle2.acquiredPermits, 70);
- assertEquals(1, handle2.trials);
-
- limiter.release(handle);
+ public void testAcquireTimeout() throws Exception {
+ @Cleanup("shutdownNow")
+ ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(100, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS,
+ executor, OpenTelemetry.noop());
+ assertThat(limiter.getRemainingBytes()).isEqualTo(100);
+ limiter.acquire(100, null);
+
+ AtomicReference handle2Reference = new AtomicReference<>();
+ Optional optionalHandle2 = limiter.acquire(100, handle2Reference::set);
+ assertThat(optionalHandle2).isNotPresent();
+
+ Thread.sleep(ACQUIRE_TIMEOUT_MILLIS + 100);
+
+ assertThat(handle2Reference).hasValueSatisfying(h -> assertThat(h.success()).isFalse());
+ }
- handle2 = limiter.acquire(100, handle2);
- assertEquals(0, limiter.getRemainingBytes());
- assertTrue(handle2.success);
- assertEquals(handle2.acquiredPermits, 100);
- assertEquals(2, handle2.trials);
+ @Test
+ public void testMultipleQueuedEntriesWithExceptionInFirstCallback() throws Exception {
+ @Cleanup("shutdownNow")
+ ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(100, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS,
+ executor, OpenTelemetry.noop());
+ assertThat(limiter.getRemainingBytes())
+ .as("Initial remaining bytes should be 100")
+ .isEqualTo(100);
+
+ // Acquire the initial permits
+ Optional handle1 = limiter.acquire(100, null);
+ assertThat(handle1)
+ .as("Initial handle should be present")
+ .isPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be 0 after acquiring 100 permits")
+ .isEqualTo(0);
+
+ // Queue the first handle with a callback that throws an exception
+ AtomicReference handle2Reference = new AtomicReference<>();
+ Optional handle2 = limiter.acquire(50, handle -> {
+ handle2Reference.set(handle);
+ throw new RuntimeException("Callback exception");
+ });
+ assertThat(handle2)
+ .as("Second handle should not be present")
+ .isNotPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 after failed acquisition")
+ .isEqualTo(0);
+
+ // Queue the second handle with a successful callback
+ AtomicReference handle3Reference = new AtomicReference<>();
+ Optional handle3 = limiter.acquire(50, handle3Reference::set);
+ assertThat(handle3)
+ .as("Third handle should not be present as queue is full")
+ .isNotPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0")
+ .isEqualTo(0);
+
+ // Release the initial handle to trigger the queued callbacks
+ limiter.release(handle1.get());
+
+ // Verify the first callback threw an exception but the second callback was handled successfully
+ assertThat(handle2Reference)
+ .as("Handle2 should have been set in the callback despite the exception")
+ .hasValueSatisfying(handle -> assertThat(handle.success())
+ .as("Handle2 should be marked as successful")
+ .isTrue());
+ assertThat(handle3Reference)
+ .as("Handle3 should have been set successfully")
+ .hasValueSatisfying(handle -> assertThat(handle.success())
+ .as("Handle3 should be marked as successful")
+ .isTrue());
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 after first releases are acquired")
+ .isEqualTo(0);
+
+ // Release the second handle
+ limiter.release(handle3Reference.get());
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be 50 after releasing handle3")
+ .isEqualTo(50);
+
+ // Release the third handle
+ limiter.release(handle3Reference.get());
+ assertThat(limiter.getRemainingBytes())
+ .as("All bytes should be released, so remaining bytes should be 100")
+ .isEqualTo(100);
+ }
- limiter.release(handle2);
- assertEquals(100, limiter.getRemainingBytes());
+ @Test
+ public void testMultipleQueuedEntriesWithTimeoutAndExceptionInFirstCallback() throws Exception {
+ @Cleanup("shutdownNow")
+ ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(100, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS,
+ executor, OpenTelemetry.noop());
+ assertThat(limiter.getRemainingBytes())
+ .as("Initial remaining bytes should be 100")
+ .isEqualTo(100);
+
+ // Acquire the initial permits
+ Optional handle1 = limiter.acquire(100, null);
+ assertThat(handle1)
+ .as("The first handle should be present after acquiring 100 permits")
+ .isPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be 0 after acquiring all permits")
+ .isEqualTo(0);
+
+ // Queue the first handle with a callback that times out and throws an exception
+ AtomicReference handle2Reference = new AtomicReference<>();
+ Optional handle2 = limiter.acquire(50, handle -> {
+ handle2Reference.set(handle);
+ throw new RuntimeException("Callback exception on timeout");
+ });
+ assertThat(handle2)
+ .as("The second handle should not be present as the callback throws an exception")
+ .isNotPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 after failed acquisition")
+ .isEqualTo(0);
+
+ // Introduce a delay to differentiate operations between queued entries
+ Thread.sleep(50);
+
+ // Queue the second handle with a successful callback
+ AtomicReference handle3Reference = new AtomicReference<>();
+ Optional handle3 = limiter.acquire(50, handle3Reference::set);
+ assertThat(handle3)
+ .as("The third handle should not be present as permits are still unavailable")
+ .isNotPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 after failed acquisition attempt")
+ .isEqualTo(0);
+
+ // Wait for the timeout to occur
+ Thread.sleep(ACQUIRE_TIMEOUT_MILLIS + 100);
+
+ // Verify the first callback timed out and threw an exception, and the second callback was handled
+ assertThat(handle2Reference)
+ .as("Handle2 should have been set in the callback despite the exception")
+ .hasValueSatisfying(handle -> assertThat(handle.success())
+ .as("Handle2 should be marked as unsuccessful due to a timeout")
+ .isFalse());
+ assertThat(handle3Reference)
+ .as("Handle3 should have been set in the callback after the permits became available")
+ .hasValueSatisfying(handle -> Assertions.assertThat(handle.success())
+ .as("Handle3 should be marked as unsuccessful due to a timeout")
+ .isFalse());
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 as no permits were released")
+ .isEqualTo(0);
+
+ // Release the first handle
+ limiter.release(handle1.get());
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be fully restored to 100 after releasing all permits")
+ .isEqualTo(100);
+ }
+ @Test
+ public void testMultipleQueuedEntriesWithTimeoutsThatAreTimedOutWhenPermitsAreAvailable() throws Exception {
+ // Use a mock executor to simulate scenarios where timed out queued handles are processed when permits become
+ // available
+ ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(100, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS,
+ executor, OpenTelemetry.noop());
+ assertThat(limiter.getRemainingBytes())
+ .as("Initial remaining bytes should be 100")
+ .isEqualTo(100);
+
+ // Acquire the initial permits
+ Optional handle1 = limiter.acquire(100, null);
+ assertThat(handle1)
+ .as("The first handle should be present after acquiring 100 permits")
+ .isPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be 0 after acquiring all permits")
+ .isEqualTo(0);
+
+ // Queue the first handle
+ AtomicReference handle2Reference = new AtomicReference<>();
+ Optional handle2 = limiter.acquire(50, handle2Reference::set);
+ assertThat(handle2)
+ .as("The second handle should not be present as permits are unavailable")
+ .isNotPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 after failed acquisition attempt for handle2")
+ .isEqualTo(0);
+
+ // Queue the second handle
+ AtomicReference handle3Reference = new AtomicReference<>();
+ Optional handle3 = limiter.acquire(50, handle3Reference::set);
+ assertThat(handle3)
+ .as("The third handle should not be present as permits are unavailable")
+ .isNotPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 after failed acquisition attempt for handle3")
+ .isEqualTo(0);
+
+ // Wait for the timeout to occur
+ Thread.sleep(ACQUIRE_TIMEOUT_MILLIS + 100);
+
+ // Queue another handle
+ AtomicReference handle4Reference = new AtomicReference<>();
+ Optional handle4 = limiter.acquire(50, handle4Reference::set);
+ assertThat(handle4)
+ .as("The fourth handle should not be present because permits are unavailable")
+ .isNotPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 after failed acquisition attempt for handle4")
+ .isEqualTo(0);
+
+ // Queue another handle
+ AtomicReference handle5Reference = new AtomicReference<>();
+ Optional handle5 = limiter.acquire(100, handle5Reference::set);
+ assertThat(handle5)
+ .as("The fifth handle should not be present as permits are unavailable")
+ .isNotPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should still be 0 after failed acquisition attempt for handle5")
+ .isEqualTo(0);
+
+ // Release the first handle
+ limiter.release(handle1.get());
+
+ assertThat(handle2Reference)
+ .as("Handle2 should have been set in the callback and marked unsuccessful")
+ .hasValueSatisfying(handle -> assertThat(handle.success()).isFalse());
+
+ assertThat(handle3Reference)
+ .as("Handle3 should have been set in the callback and marked unsuccessful")
+ .hasValueSatisfying(handle -> assertThat(handle.success()).isFalse());
+
+ assertThat(handle4Reference)
+ .as("Handle4 should have been set in the callback and marked successful")
+ .hasValueSatisfying(handle -> assertThat(handle.success()).isTrue());
+
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be 50 after releasing handle4")
+ .isEqualTo(50);
+
+ limiter.release(handle4Reference.get());
+
+ assertThat(handle5Reference)
+ .as("Handle5 should have been set in the callback and marked successful")
+ .hasValueSatisfying(handle -> assertThat(handle.success()).isTrue());
+
+ limiter.release(handle5Reference.get());
+
+ assertThat(limiter.getRemainingBytes())
+ .as("All bytes should be released, so remaining bytes should be back to 100")
+ .isEqualTo(100);
}
@Test
- public void testTooManyTrials() throws Exception {
- InflightReadsLimiter limiter = new InflightReadsLimiter(100, OpenTelemetry.noop());
- assertEquals(100, limiter.getRemainingBytes());
-
- InflightReadsLimiter.Handle handle = limiter.acquire(30, null);
- assertEquals(70, limiter.getRemainingBytes());
- assertTrue(handle.success);
- assertEquals(handle.acquiredPermits, 30);
- assertEquals(1, handle.trials);
-
- InflightReadsLimiter.Handle handle2 = limiter.acquire(100, null);
- assertEquals(0, limiter.getRemainingBytes());
- assertFalse(handle2.success);
- assertEquals(handle2.acquiredPermits, 70);
- assertEquals(1, handle2.trials);
-
- handle2 = limiter.acquire(100, handle2);
- assertEquals(0, limiter.getRemainingBytes());
- assertFalse(handle2.success);
- assertEquals(handle2.acquiredPermits, 70);
- assertEquals(2, handle2.trials);
-
- handle2 = limiter.acquire(100, handle2);
- assertEquals(0, limiter.getRemainingBytes());
- assertFalse(handle2.success);
- assertEquals(handle2.acquiredPermits, 70);
- assertEquals(3, handle2.trials);
-
- handle2 = limiter.acquire(100, handle2);
- assertEquals(0, limiter.getRemainingBytes());
- assertFalse(handle2.success);
- assertEquals(handle2.acquiredPermits, 70);
- assertEquals(4, handle2.trials);
-
- // too many trials, start from scratch
- handle2 = limiter.acquire(100, handle2);
- assertEquals(70, limiter.getRemainingBytes());
- assertFalse(handle2.success);
- assertEquals(handle2.acquiredPermits, 0);
- assertEquals(1, handle2.trials);
+ public void testQueueSizeLimitReached() throws Exception {
+ @Cleanup("shutdownNow")
+ ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+
+ // Minimum queue size is 4.
+ final int queueSizeLimit = 4;
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(100, queueSizeLimit, ACQUIRE_TIMEOUT_MILLIS, executor, OpenTelemetry.noop());
+
+ assertThat(limiter.getRemainingBytes())
+ .as("Initial remaining bytes should be 100")
+ .isEqualTo(100);
+
+ // Acquire all available permits (consume 100 bytes)
+ Optional handle1 = limiter.acquire(100, null);
+ assertThat(handle1)
+ .as("The first handle should be present after acquiring all available permits")
+ .isPresent()
+ .hasValueSatisfying(handle -> assertThat(handle.success()).isTrue());
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be zero after acquiring all permits")
+ .isEqualTo(0);
+
+ // Queue up to the limit (4 requests)
+ AtomicReference handle2Reference = new AtomicReference<>();
+ assertThat(limiter.acquire(50, handle2Reference::set)).isNotPresent();
+
+ AtomicReference handle3Reference = new AtomicReference<>();
+ assertThat(limiter.acquire(50, handle3Reference::set)).isNotPresent();
+
+ AtomicReference handle4Reference = new AtomicReference<>();
+ assertThat(limiter.acquire(50, handle4Reference::set)).isNotPresent();
+
+ AtomicReference handle5Reference = new AtomicReference<>();
+ assertThat(limiter.acquire(50, handle5Reference::set)).isNotPresent();
+
+ // Attempt to add one more request, which should fail as the queue is full
+ Optional handle6 = limiter.acquire(50, null);
+ assertThat(handle6)
+ .as("The sixth handle should not be successfull since the queue is full")
+ .hasValueSatisfying(handle -> assertThat(handle.success()).isFalse());
+ }
- limiter.release(handle);
+ @Test(dataProvider = "booleanValues")
+ public void testAcquireExceedingMaxReadsInFlightSize(boolean firstInQueue) throws Exception {
+ @Cleanup("shutdownNow")
+ ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+
+ long maxReadsInFlightSize = 100;
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(maxReadsInFlightSize, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS, executor,
+ OpenTelemetry.noop());
+
+ // Initial state
+ assertThat(limiter.getRemainingBytes())
+ .as("Initial remaining bytes should match maxReadsInFlightSize")
+ .isEqualTo(maxReadsInFlightSize);
+
+ // Acquire all permits (consume 100 bytes)
+ Optional handle1 = limiter.acquire(100, null);
+ assertThat(handle1)
+ .as("The first handle should be present")
+ .isPresent();
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be zero after acquiring all permits")
+ .isEqualTo(0);
+
+
+ AtomicReference handle2Reference = new AtomicReference<>();
+
+ if (!firstInQueue) {
+ Optional handle2 = limiter.acquire(50, handle2Reference::set);
+ assertThat(handle2)
+ .as("The second handle should not be present as remaining permits are zero")
+ .isNotPresent();
+ }
- handle2 = limiter.acquire(100, handle2);
- assertEquals(0, limiter.getRemainingBytes());
- assertTrue(handle2.success);
- assertEquals(handle2.acquiredPermits, 100);
- assertEquals(2, handle2.trials);
+ // Attempt to acquire more than maxReadsInFlightSize while all permits are in use
+ AtomicReference handleExceedingMaxReference = new AtomicReference<>();
+ Optional handleExceedingMaxOptional =
+ limiter.acquire(200, handleExceedingMaxReference::set);
+ assertThat(handleExceedingMaxOptional)
+ .as("The second handle should not be present as remaining permits are zero")
+ .isNotPresent();
+
+ // Release handle1 permits
+ limiter.release(handle1.get());
+
+ if (!firstInQueue) {
+ assertThat(handle2Reference)
+ .as("Handle2 should have been set in the callback and marked successful")
+ .hasValueSatisfying(handle -> {
+ assertThat(handle.success()).isTrue();
+ assertThat(handle.permits()).isEqualTo(50);
+ });
+ limiter.release(handle2Reference.get());
+ }
+
+ assertThat(handleExceedingMaxReference)
+ .as("Handle2 should have been set in the callback and marked successful")
+ .hasValueSatisfying(handle -> {
+ assertThat(handle.success()).isTrue();
+ assertThat(handle.permits()).isEqualTo(maxReadsInFlightSize);
+ });
- limiter.release(handle2);
- assertEquals(100, limiter.getRemainingBytes());
+ limiter.release(handleExceedingMaxReference.get());
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be fully replenished after releasing all permits")
+ .isEqualTo(maxReadsInFlightSize);
+ }
+
+ @Test
+ public void testAcquireExceedingMaxReadsWhenAllPermitsAvailable() throws Exception {
+ @Cleanup("shutdownNow")
+ ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+
+ long maxReadsInFlightSize = 100;
+ InflightReadsLimiter limiter =
+ new InflightReadsLimiter(maxReadsInFlightSize, ACQUIRE_QUEUE_SIZE, ACQUIRE_TIMEOUT_MILLIS, executor,
+ OpenTelemetry.noop());
+
+ // Initial state
+ assertThat(limiter.getRemainingBytes())
+ .as("Initial remaining bytes should match maxReadsInFlightSize")
+ .isEqualTo(maxReadsInFlightSize);
+
+ // Acquire permits > maxReadsInFlightSize
+ Optional handleExceedingMaxOptional =
+ limiter.acquire(2 * maxReadsInFlightSize, null);
+ assertThat(handleExceedingMaxOptional)
+ .as("The handle for exceeding max permits should be present")
+ .hasValueSatisfying(handle -> {
+ assertThat(handle.success()).isTrue();
+ assertThat(handle.permits()).isEqualTo(maxReadsInFlightSize);
+ });
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be zero after acquiring all permits")
+ .isEqualTo(0);
+
+ // Release permits
+ limiter.release(handleExceedingMaxOptional.get());
+
+ assertThat(limiter.getRemainingBytes())
+ .as("Remaining bytes should be fully replenished after releasing all permits")
+ .isEqualTo(maxReadsInFlightSize);
}
private Pair buildOpenTelemetryAndReader() {
var metricReader = InMemoryMetricReader.create();
var openTelemetry = AutoConfiguredOpenTelemetrySdk.builder()
+ .disableShutdownHook()
+ .addPropertiesSupplier(() -> Map.of("otel.metrics.exporter", "none",
+ "otel.traces.exporter", "none",
+ "otel.logs.exporter", "none"))
.addMeterProviderCustomizer((builder, __) -> builder.registerMetricReader(metricReader))
.build()
.getOpenTelemetrySdk();
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManagerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManagerTest.java
index 01976f648aba4..55068580f62f1 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManagerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/cache/PendingReadsManagerTest.java
@@ -35,6 +35,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@@ -91,7 +92,8 @@ void setupMocks() {
config.setReadEntryTimeoutSeconds(10000);
when(rangeEntryCache.getName()).thenReturn("my-topic");
when(rangeEntryCache.getManagedLedgerConfig()).thenReturn(config);
- inflighReadsLimiter = new InflightReadsLimiter(0, OpenTelemetry.noop());
+ inflighReadsLimiter = new InflightReadsLimiter(0, 0, 0,
+ mock(ScheduledExecutorService.class), OpenTelemetry.noop());
when(rangeEntryCache.getPendingReadsLimiter()).thenReturn(inflighReadsLimiter);
pendingReadsManager = new PendingReadsManager(rangeEntryCache);
doAnswer(new Answer() {
@@ -108,7 +110,7 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
return null;
}
}).when(rangeEntryCache).asyncReadEntry0(any(), anyLong(), anyLong(),
- anyBoolean(), any(), any());
+ anyBoolean(), any(), any(), anyBoolean());
lh = mock(ReadHandle.class);
ml = mock(ManagedLedgerImpl.class);
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java
index 4bcf2cc6c4e35..b6914fd8efe49 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java
@@ -20,19 +20,23 @@
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotSame;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import com.google.common.collect.Lists;
import io.netty.util.AbstractReferenceCounted;
import io.netty.util.ReferenceCounted;
+import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import lombok.Cleanup;
import lombok.Data;
import org.apache.commons.lang3.tuple.Pair;
import org.awaitility.Awaitility;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class RangeCacheTest {
@@ -139,9 +143,14 @@ public void customWeighter() {
assertEquals(cache.getNumberOfEntries(), 2);
}
+ @DataProvider
+ public static Object[][] retainBeforeEviction() {
+ return new Object[][]{ { true }, { false } };
+ }
- @Test
- public void customTimeExtraction() {
+
+ @Test(dataProvider = "retainBeforeEviction")
+ public void customTimeExtraction(boolean retain) {
RangeCache cache = new RangeCache<>(value -> value.s.length(), x -> x.s.length());
cache.put(1, new RefString("1"));
@@ -151,13 +160,30 @@ public void customTimeExtraction() {
assertEquals(cache.getSize(), 10);
assertEquals(cache.getNumberOfEntries(), 4);
+ final var retainedEntries = cache.getRange(1, 4444);
+ for (final var entry : retainedEntries) {
+ assertEquals(entry.refCnt(), 2);
+ if (!retain) {
+ entry.release();
+ }
+ }
Pair evictedSize = cache.evictLEntriesBeforeTimestamp(3);
assertEquals(evictedSize.getRight().longValue(), 6);
assertEquals(evictedSize.getLeft().longValue(), 3);
-
assertEquals(cache.getSize(), 4);
assertEquals(cache.getNumberOfEntries(), 1);
+
+ if (retain) {
+ final var valueToRefCnt = retainedEntries.stream().collect(Collectors.toMap(RefString::getS,
+ AbstractReferenceCounted::refCnt));
+ assertEquals(valueToRefCnt, Map.of("1", 1, "22", 1, "333", 1, "4444", 2));
+ retainedEntries.forEach(AbstractReferenceCounted::release);
+ } else {
+ final var valueToRefCnt = retainedEntries.stream().filter(v -> v.refCnt() > 0).collect(Collectors.toMap(
+ RefString::getS, AbstractReferenceCounted::refCnt));
+ assertEquals(valueToRefCnt, Map.of("4444", 1));
+ }
}
@Test
@@ -338,4 +364,20 @@ public void testRemoveEntryWithInvalidMatchingKey() {
cache.clear();
assertEquals(cache.getNumberOfEntries(), 0);
}
+
+ @Test
+ public void testGetKeyWithDifferentInstance() {
+ RangeCache cache = new RangeCache<>();
+ Integer key = 129;
+ cache.put(key, new RefString("129"));
+ // create a different instance of the key
+ Integer key2 = Integer.valueOf(129);
+ // key and key2 are different instances but they are equal
+ assertNotSame(key, key2);
+ assertEquals(key, key2);
+ // get the value using key2
+ RefString s = cache.get(key2);
+ // the value should be found
+ assertEquals(s.s, "129");
+ }
}
diff --git a/microbench/README.md b/microbench/README.md
index 780e3a5a1d3e8..f50c3036ff494 100644
--- a/microbench/README.md
+++ b/microbench/README.md
@@ -41,3 +41,29 @@ For fast recompiling of the benchmarks (without compiling Pulsar modules) and cr
mvn -Pmicrobench -pl microbench clean package
```
+### Running specific benchmarks
+
+Display help:
+
+```shell
+java -jar microbench/target/microbenchmarks.jar -h
+```
+
+Listing all benchmarks:
+
+```shell
+java -jar microbench/target/microbenchmarks.jar -l
+```
+
+Running specific benchmarks:
+
+```shell
+java -jar microbench/target/microbenchmarks.jar ".*BenchmarkName.*"
+```
+
+Checking what benchmarks match the pattern:
+
+```shell
+java -jar microbench/target/microbenchmarks.jar ".*BenchmarkName.*" -lp
+```
+
diff --git a/microbench/pom.xml b/microbench/pom.xml
index bef02794adbd6..f549cf1408ec0 100644
--- a/microbench/pom.xml
+++ b/microbench/pom.xml
@@ -25,7 +25,7 @@
org.apache.pulsar
pulsar
- 4.0.0-SNAPSHOT
+ 4.1.0-SNAPSHOT
microbench
diff --git a/microbench/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBenchmark.java
index 4c069e72ea3ba..1b210258f13d2 100644
--- a/microbench/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBenchmark.java
+++ b/microbench/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBenchmark.java
@@ -33,6 +33,7 @@
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
@Fork(3)
@BenchmarkMode(Mode.Throughput)
@@ -59,23 +60,29 @@ public void teardown() {
@Benchmark
@Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
@Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
- public void consumeTokensBenchmark001Threads() {
- asyncTokenBucket.consumeTokens(1);
+ public void consumeTokensBenchmark001Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole);
}
@Threads(10)
@Benchmark
@Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
@Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
- public void consumeTokensBenchmark010Threads() {
- asyncTokenBucket.consumeTokens(1);
+ public void consumeTokensBenchmark010Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole);
}
@Threads(100)
@Benchmark
@Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
@Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
- public void consumeTokensBenchmark100Threads() {
+ public void consumeTokensBenchmark100Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole);
+ }
+
+ private void consumeTokenAndGetTokens(Blackhole blackhole) {
asyncTokenBucket.consumeTokens(1);
+ // blackhole is used to ensure that the compiler doesn't do dead code elimination
+ blackhole.consume(asyncTokenBucket.getTokens());
}
}
diff --git a/microbench/src/main/java/org/apache/pulsar/broker/qos/DefaultMonotonicSnapshotClockBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/qos/DefaultMonotonicSnapshotClockBenchmark.java
new file mode 100644
index 0000000000000..d9054b8fe4be8
--- /dev/null
+++ b/microbench/src/main/java/org/apache/pulsar/broker/qos/DefaultMonotonicSnapshotClockBenchmark.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pulsar.broker.qos;
+
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@Fork(3)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Thread)
+public class DefaultMonotonicSnapshotClockBenchmark {
+ private DefaultMonotonicSnapshotClock monotonicSnapshotClock =
+ new DefaultMonotonicSnapshotClock(TimeUnit.MILLISECONDS.toNanos(1), System::nanoTime);
+
+ @TearDown(Level.Iteration)
+ public void teardown() {
+ monotonicSnapshotClock.close();
+ }
+
+ @Threads(1)
+ @Benchmark
+ @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ public void getTickNanos001Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole, false);
+ }
+
+ @Threads(10)
+ @Benchmark
+ @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ public void getTickNanos010Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole, false);
+ }
+
+ @Threads(100)
+ @Benchmark
+ @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ public void getTickNanos100Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole, false);
+ }
+
+ @Threads(1)
+ @Benchmark
+ @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ public void getTickNanosRequestSnapshot001Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole, true);
+ }
+
+ @Threads(10)
+ @Benchmark
+ @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ public void getTickNanosRequestSnapshot010Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole, true);
+ }
+
+ @Threads(100)
+ @Benchmark
+ @Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ @Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
+ public void getTickNanosRequestSnapshot100Threads(Blackhole blackhole) {
+ consumeTokenAndGetTokens(blackhole, true);
+ }
+
+ private void consumeTokenAndGetTokens(Blackhole blackhole, boolean requestSnapshot) {
+ // blackhole is used to ensure that the compiler doesn't do dead code elimination
+ blackhole.consume(monotonicSnapshotClock.getTickNanos(requestSnapshot));
+ }
+}
diff --git a/mvnw b/mvnw
index 5643201c7d822..19529ddf8c6ea 100755
--- a/mvnw
+++ b/mvnw
@@ -19,298 +19,241 @@
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
-# Maven Start Up Batch script
-#
-# Required ENV vars:
-# ------------------
-# JAVA_HOME - location of a JDK home dir
+# Apache Maven Wrapper startup batch script, version 3.3.2
#
# Optional ENV vars
# -----------------
-# M2_HOME - location of maven2's installed home dir
-# MAVEN_OPTS - parameters passed to the Java VM when running Maven
-# e.g. to debug Maven itself, use
-# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
-# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# JAVA_HOME - location of a JDK home dir, required when download maven via java source
+# MVNW_REPOURL - repo url base for downloading maven distribution
+# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
# ----------------------------------------------------------------------------
-if [ -z "$MAVEN_SKIP_RC" ] ; then
-
- if [ -f /usr/local/etc/mavenrc ] ; then
- . /usr/local/etc/mavenrc
- fi
-
- if [ -f /etc/mavenrc ] ; then
- . /etc/mavenrc
- fi
+set -euf
+[ "${MVNW_VERBOSE-}" != debug ] || set -x
- if [ -f "$HOME/.mavenrc" ] ; then
- . "$HOME/.mavenrc"
- fi
+# OS specific support.
+native_path() { printf %s\\n "$1"; }
+case "$(uname)" in
+CYGWIN* | MINGW*)
+ [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
+ native_path() { cygpath --path --windows "$1"; }
+ ;;
+esac
-fi
+# set JAVACMD and JAVACCMD
+set_java_home() {
+ # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
+ if [ -n "${JAVA_HOME-}" ]; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ]; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ JAVACCMD="$JAVA_HOME/jre/sh/javac"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ JAVACCMD="$JAVA_HOME/bin/javac"
-# OS specific support. $var _must_ be set to either true or false.
-cygwin=false;
-darwin=false;
-mingw=false
-case "`uname`" in
- CYGWIN*) cygwin=true ;;
- MINGW*) mingw=true;;
- Darwin*) darwin=true
- # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
- # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
- if [ -z "$JAVA_HOME" ]; then
- if [ -x "/usr/libexec/java_home" ]; then
- export JAVA_HOME="`/usr/libexec/java_home`"
- else
- export JAVA_HOME="/Library/Java/Home"
+ if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
+ echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
+ echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
+ return 1
fi
fi
- ;;
-esac
-
-if [ -z "$JAVA_HOME" ] ; then
- if [ -r /etc/gentoo-release ] ; then
- JAVA_HOME=`java-config --jre-home`
+ else
+ JAVACMD="$(
+ 'set' +e
+ 'unset' -f command 2>/dev/null
+ 'command' -v java
+ )" || :
+ JAVACCMD="$(
+ 'set' +e
+ 'unset' -f command 2>/dev/null
+ 'command' -v javac
+ )" || :
+
+ if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
+ echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
+ return 1
+ fi
fi
-fi
-
-if [ -z "$M2_HOME" ] ; then
- ## resolve links - $0 may be a link to maven's home
- PRG="$0"
+}
- # need this for relative symlinks
- while [ -h "$PRG" ] ; do
- ls=`ls -ld "$PRG"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG="`dirname "$PRG"`/$link"
- fi
+# hash string like Java String::hashCode
+hash_string() {
+ str="${1:-}" h=0
+ while [ -n "$str" ]; do
+ char="${str%"${str#?}"}"
+ h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
+ str="${str#?}"
done
+ printf %x\\n $h
+}
- saveddir=`pwd`
+verbose() { :; }
+[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
- M2_HOME=`dirname "$PRG"`/..
+die() {
+ printf %s\\n "$1" >&2
+ exit 1
+}
- # make it fully qualified
- M2_HOME=`cd "$M2_HOME" && pwd`
+trim() {
+ # MWRAPPER-139:
+ # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
+ # Needed for removing poorly interpreted newline sequences when running in more
+ # exotic environments such as mingw bash on Windows.
+ printf "%s" "${1}" | tr -d '[:space:]'
+}
- cd "$saveddir"
- # echo Using m2 at $M2_HOME
-fi
+# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
+while IFS="=" read -r key value; do
+ case "${key-}" in
+ distributionUrl) distributionUrl=$(trim "${value-}") ;;
+ distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
+ esac
+done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties"
+[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties"
+
+case "${distributionUrl##*/}" in
+maven-mvnd-*bin.*)
+ MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
+ case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
+ *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
+ :Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
+ :Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
+ :Linux*x86_64*) distributionPlatform=linux-amd64 ;;
+ *)
+ echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
+ distributionPlatform=linux-amd64
+ ;;
+ esac
+ distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
+ ;;
+maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
+*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
+esac
-# For Cygwin, ensure paths are in UNIX format before anything is touched
-if $cygwin ; then
- [ -n "$M2_HOME" ] &&
- M2_HOME=`cygpath --unix "$M2_HOME"`
- [ -n "$JAVA_HOME" ] &&
- JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
- [ -n "$CLASSPATH" ] &&
- CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
-fi
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--