[improve][misc] Sync commits from apache into 4.0_ds (Oct 15th)

.github/workflows/ci-documentbot.yml

@@ -36,7 +36,7 @@ jobs:
     if: (github.repository == 'apache/pulsar') && (github.event.pull_request.state == 'open')
     permissions:
       pull-requests: write 
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Labeling
         uses: apache/pulsar-test-infra/docbot@master

.github/workflows/ci-go-functions.yaml

@@ -38,7 +38,7 @@ env:
 jobs:
   preconditions:
     name: Preconditions
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
     steps:
@@ -73,7 +73,7 @@ jobs:
     needs: preconditions
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     name: Go ${{ matrix.go-version }} Functions style check
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         go-version: ['1.23']

.github/workflows/ci-maven-cache-update.yaml

@@ -59,7 +59,7 @@ jobs:
       matrix:
         include:
           - name: all modules
-            runs-on: ubuntu-22.04
+            runs-on: ubuntu-24.04
             cache_name: 'm2-dependencies-all'
             mvn_arguments: ''
 
@@ -68,7 +68,7 @@ jobs:
             cache_name: 'm2-dependencies-all'
 
           - name: core-modules
-            runs-on: ubuntu-22.04
+            runs-on: ubuntu-24.04
             cache_name: 'm2-dependencies-core-modules'
             mvn_arguments: '-Pcore-modules,-main'
 

.github/workflows/ci-owasp-dependency-check.yaml

@@ -35,7 +35,7 @@ jobs:
     env:
       JOB_NAME: Check ${{ matrix.branch }}
       DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 75
     strategy:
       fail-fast: false

.github/workflows/ci-pulsarbot.yaml

@@ -24,7 +24,7 @@ on:
 
 jobs:
   pulsarbot:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/pulsarbot')
     steps:

.github/workflows/pulsar-ci-flaky.yaml

@@ -87,7 +87,7 @@ env:
 jobs:
   preconditions:
     name: Preconditions
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
       changed_tests: ${{ steps.changes.outputs.tests_files }}
@@ -176,7 +176,7 @@ jobs:
       THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
       NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}"
       NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 100
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     steps:

.github/workflows/pulsar-ci.yaml

@@ -87,7 +87,7 @@ env:
 jobs:
   preconditions:
     name: Preconditions
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
       changed_tests: ${{ steps.changes.outputs.tests_files }}
@@ -169,7 +169,7 @@ jobs:
       JOB_NAME: Build and License check
       DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     steps:
@@ -252,7 +252,7 @@ jobs:
       THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
       NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}"
       NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: ${{ matrix.timeout || 60 }}
     needs: ['preconditions', 'build-and-license-check']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -271,6 +271,8 @@ jobs:
             group: BROKER_GROUP_3
           - name: Brokers - Broker Group 4
             group: BROKER_GROUP_4
+          - name: Brokers - Broker Group 5
+            group: BROKER_GROUP_5
           - name: Brokers - Client Api
             group: BROKER_CLIENT_API
           - name: Brokers - Client Impl
@@ -407,7 +409,7 @@ jobs:
 
   unit-tests-upload-coverage:
     name: CI - Unit - Upload Coverage
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     needs: ['preconditions', 'unit-tests']
     env:
@@ -489,7 +491,7 @@ jobs:
 
   pulsar-java-test-image:
     name: Build Pulsar java-test-image docker image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'build-and-license-check']
     if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
@@ -571,7 +573,7 @@ jobs:
 
   integration-tests:
     name: CI - Integration - ${{ matrix.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: ${{ matrix.timeout || 60 }}
     needs: ['preconditions', 'pulsar-java-test-image']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -774,7 +776,7 @@ jobs:
 
   integration-tests-upload-coverage:
     name: CI - Integration - Upload Coverage
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     needs: ['preconditions', 'integration-tests']
     if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
@@ -861,7 +863,7 @@ jobs:
 
   delete-integration-test-docker-image-artifact:
     name: "Delete integration test docker image artifact"
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     needs: [
       'preconditions',
@@ -885,7 +887,7 @@ jobs:
 
   pulsar-test-latest-version-image:
     name: Build Pulsar docker image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'build-and-license-check']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -1009,7 +1011,7 @@ jobs:
 
   system-tests:
     name: CI - System - ${{ matrix.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'pulsar-test-latest-version-image']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -1171,7 +1173,7 @@ jobs:
 
   system-tests-upload-coverage:
     name: CI - System - Upload Coverage
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     needs: ['preconditions', 'system-tests']
     if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
@@ -1259,7 +1261,7 @@ jobs:
 
   flaky-system-tests:
     name: CI Flaky - System - ${{ matrix.name }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: [ 'preconditions', 'pulsar-test-latest-version-image' ]
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -1398,7 +1400,7 @@ jobs:
 
   delete-system-test-docker-image-artifact:
     name: "Delete system test docker image artifact"
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     needs: [
       'preconditions',
@@ -1459,7 +1461,7 @@ jobs:
 
   codeql:
     name: Run CodeQL Analysis
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'unit-tests']
     if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }}
@@ -1524,7 +1526,7 @@ jobs:
 
   owasp-dep-check:
     name: OWASP dependency check
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 120
     needs: [ 'preconditions', 'integration-tests' ]
     if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
@@ -1630,7 +1632,7 @@ jobs:
     # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled
     # jobs are not enabled for other than apache/pulsar repository.
     if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 10
     needs: [
       'preconditions',

bouncy-castle/bcfips/LICENSE

@@ -205,5 +205,6 @@
 This projects includes binary packages with the following licenses:
 Bouncy Castle License
  * Bouncy Castle -- licenses/LICENSE-bouncycastle.txt
-    - org.bouncycastle-bcpkix-fips-1.0.7.jar
-    - org.bouncycastle-bc-fips-1.0.2.6.jar
+    - org.bouncycastle-bcpkix-fips-2.0.10.jar
+    - org.bouncycastle-bc-fips-2.0.1.jar
+    - org.bouncycastle-bctutil-fips-2.0.5.jar

bouncy-castle/bcfips/pom.xml

@@ -32,6 +32,11 @@
   <name>Apache Pulsar :: Bouncy Castle :: BC-FIPS</name>
 
   <dependencies>
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcutil-fips</artifactId>
+      <version>2.0.5</version>
+    </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-common</artifactId>

build/run_unit_group.sh

@@ -82,7 +82,7 @@ function test_group_broker_group_1() {
 }
 
 function test_group_broker_group_2() {
-  mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,broker-replication,websocket,other'
+  mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other'
 }
 
 function test_group_broker_group_3() {
@@ -95,6 +95,10 @@ function test_group_broker_group_4() {
   mvn_test -pl pulsar-broker -Dgroups='cluster-migration'
 }
 
+function test_group_broker_group_5() {
+  mvn_test -pl pulsar-broker -Dgroups='broker-replication'
+}
+
 function test_group_broker_client_api() {
   mvn_test -pl pulsar-broker -Dgroups='broker-api'
 }
@@ -174,7 +178,8 @@ function test_group_other() {
            -pl '!com.datastax.oss:distribution,!com.datastax.oss:pulsar-offloader-distribution,!com.datastax.oss:pulsar-server-distribution,!com.datastax.oss:pulsar-io-distribution,!com.datastax.oss:pulsar-all-docker-image,!com.datastax.oss:pulsar-experimental-docker-image' \
            -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DskipIntegrationTests \
            -Dexclude='**/ManagedLedgerTest.java,
-                   **/OffloadersCacheTest.java
+                   **/OffloadersCacheTest.java,
+                   **/OffsetsCacheTest.java,
                   **/PrimitiveSchemaTest.java,
                   **/BlobStoreManagedLedgerOffloaderTest.java,
                   **/BlobStoreManagedLedgerOffloaderStreamingTest.java'
@@ -184,6 +189,7 @@ function test_group_other() {
 
   mvn_test -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderTest.java'
   mvn_test -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderStreamingTest.java'
+  mvn_test -pl tiered-storage/jcloud -Dinclude='**/OffsetsCacheTest.java'
 
   echo "::endgroup::"
   local modules_with_quarantined_tests=$(git grep -l '@Test.*"quarantine"' | grep '/src/test/java/' | \

buildtools/pom.xml

@@ -55,7 +55,7 @@
     <license-maven-plugin.version>4.1</license-maven-plugin.version>
     <puppycrawl.checkstyle.version>10.14.2</puppycrawl.checkstyle.version>
     <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
-    <netty.version>4.1.127.Final</netty.version>
+    <netty.version>4.1.128.Final</netty.version>
     <guice.version>4.2.3</guice.version>
     <guava.version>32.1.2-jre</guava.version>
     <ant.version>1.10.12</ant.version>