Merge branch 'master' into fix-completionObjects-leak-problem

# Conflicts:
#	bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java

.asf.yaml

@@ -21,8 +21,12 @@ github:
   description: "Apache BookKeeper - a scalable, fault tolerant and low latency storage service optimized for append-only workloads"
   homepage: https://bookkeeper.apache.org/
   labels:
+    - apache
     - bookkeeper
     - big-data
+    - distributed-log
+    - distributed-systems
+    - wal
   features:
     # Enable wiki for documentation
     wiki: false
@@ -45,20 +49,7 @@ github:
         # Contexts are the names of checks that must pass.
         # See ./github/workflows/README.md for more documentation on this list.
         contexts:
-          - PR Validation
-          - Backward compatibility tests
-          - Bookie Tests
-          - Build with macos on JDK 11
-          - Build with windows on JDK 11
-          - Client Tests
-          - Compatibility Check Java11
-          - Compatibility Check Java17
-          - Compatibility Check Java8
-          - Integration Tests
-          - Remaining Tests
-          - Replication Tests
-          - StreamStorage Tests
-          - TLS Tests
+          - BookKeeper CI checks completed
 
       required_pull_request_reviews:
         dismiss_stale_reviews: false

.github/ISSUE_TEMPLATE.md

@@ -2,7 +2,7 @@ Is this a question, feature request, bug report, or bookkeeper proposal?
 
 **QUESTION**
 
-Have you checked our documentation at http://bookkeeper.apache.org/ , If you could not find an answer there, please consider asking your question in our community forum at [email protected], as it would benefit other members of our community.
+Have you checked our documentation at https://bookkeeper.apache.org/ , If you could not find an answer there, please consider asking your question in our community forum at [email protected], as it would benefit other members of our community.
 
 
 **FEATURE REQUEST**

.github/ISSUE_TEMPLATE/question.md

@@ -8,7 +8,7 @@ assignees: ''
 
 **QUESTION**
 
-Have you checked our documentation at http://bookkeeper.apache.org/?
+Have you checked our documentation at https://bookkeeper.apache.org/?
 If you could not find an answer there, please consider asking your question
 in our community mailing list at [email protected], as it would benefit
 other members of our community.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,16 @@
 Descriptions of the changes in this PR:
 
+<!-- Either this PR fixes an issue, -->
 
+Fix #xyz
+
+<!-- or this PR is one task of an issue -->
+
+Main Issue: #xyz
+
+<!-- If the PR belongs to a BP, please add the BP link here -->
+
+BP: #xyz
 
 ### Motivation
 
@@ -10,14 +20,12 @@ Descriptions of the changes in this PR:
 
 (Describe: what changes you have made)
 
-Master Issue: #<master-issue-number>
-
 > ---
 > In order to uphold a high standard for quality for code contributions, Apache BookKeeper runs various precommit
 > checks for pull requests. A pull request can only be merged when it passes precommit checks.
 >
 > ---
-> Be sure to do all of the following to help us incorporate your contribution
+> Be sure to do all the following to help us incorporate your contribution
 > quickly and easily:
 >
 > If this PR is a BookKeeper Proposal (BP):

.github/actions/tune-runner-vm/action.yml

@@ -53,8 +53,8 @@ runs:
             # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
             # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
             # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
             # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
             # https://www.spinics.net/lists/linux-ide/msg52562.html
             for i in /sys/block/sd*/queue/discard_max_bytes; do

.github/changes-filter.yaml

@@ -9,6 +9,7 @@ docs:
   - '**/*.md'
   - '.github/changes-filter.yaml'
   - '.github/ISSUE_TEMPLATE/**'
+  - 'src/owasp-dependency-check-suppressions.xml'
 need_owasp:
   - 'pom.xml'
   - '**/pom.xml'

.github/workflows/bk-ci.yml

@@ -60,6 +60,7 @@ jobs:
         id: check_changes
         run: |
           echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
+          echo "need_owasp=${{ fromJSON(steps.changes.outputs.need_owasp) }}" >> $GITHUB_OUTPUT
 
       - name: Cache local Maven repository
         if: steps.check_changes.outputs.docs_only != 'true'
@@ -82,16 +83,19 @@ jobs:
       - name: Validate pull request
         if: steps.check_changes.outputs.docs_only != 'true'
         run: |
-          mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
-          mvn -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+          mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests
+          mvn -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests
 
       - name: Check license files
         if: steps.check_changes.outputs.docs_only != 'true'
         run: dev/check-all-licenses
 
-      - name:  Generate Javadoc
+      - name: Generate Javadoc
         if: steps.check_changes.outputs.docs_only != 'true'
         run: mvn -B -nsu -am -pl bookkeeper-common,bookkeeper-server,:bookkeeper-stats-api,:bookkeeper-stats-providers,:codahale-metrics-provider,:prometheus-metrics-provider javadoc:aggregate -DskipTests -Pdelombok -Dchesktyle.skip -Dspotbugs.skip
+    outputs:
+      docs_only: ${{ steps.check_changes.outputs.docs_only }}
+      need_owasp: ${{ steps.check_changes.outputs.need_owasp }}
 
   unit-tests:
     name: ${{ matrix.step_name }}
@@ -234,14 +238,14 @@ jobs:
           $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror
 
       - name: Build with Maven
-        run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+        run: mvn -B -nsu clean install -Pdocker -DskipTests
 
       - name: Run metadata driver tests
-        run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+        run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests
 
       - name: Run all integration tests (except backward compatibility tests)
         run: |
-          mvn -B -nsu -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO -DredirectTestOutputToFile=false -DtestRetryCount=0
+          mvn -B -nsu -f tests/pom.xml test -DintegrationTests -DredirectTestOutputToFile=false -DtestRetryCount=0
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()
@@ -319,7 +323,7 @@ jobs:
           $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror
 
       - name: Build with Maven
-        run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+        run: mvn -B -nsu clean install -Pdocker -DskipTests
 
       - name: Test current server with old clients
         run: mvn -B -nsu -DbackwardCompatTests -pl :backward-compat-current-server-old-clients test
@@ -329,7 +333,7 @@ jobs:
 
       - name: Other tests
         run: |
-          mvn -B -nsu -DbackwardCompatTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct,:yahoo-custom-version test
+          mvn -B -nsu -DbackwardCompatTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct test
 
       - name: Upload container logs on failure
         uses: actions/upload-artifact@v4
@@ -446,6 +450,8 @@ jobs:
             jdk_version: 11
           - step_name: Compatibility Check Java17
             jdk_version: 17
+          - step_name: Compatibility Check Java21
+            jdk_version: 21
 
     steps:
       - name: checkout
@@ -471,12 +477,20 @@ jobs:
           java-version: ${{ matrix.jdk_version }}
 
       - name: Build with Maven
-        run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+        run: mvn clean package -B -nsu -DskipBookKeeperServerTests
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()
         run: ./dev/ci-tool print_thread_dumps
 
+  typo-check:
+    name: Typo Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check typos
+        uses: crate-ci/typos@master
+
   owasp-dependency-check:
     name: OWASP Dependency Check
     runs-on: ubuntu-latest
@@ -503,17 +517,13 @@ jobs:
       - name: Set up JDK 11
         uses: actions/setup-java@v4
         with:
-          java-version: 11
-
-      - name: Set up Maven
-        uses: apache/pulsar-test-infra/setup-maven@master
-        with:
-          maven-version: 3.8.7
+          distribution: 'temurin'
+          java-version: 21
 
       - name: run "clean install verify" to trigger dependency check
         # excluding dlfs because it includes hadoop lib with
-        # CVEs that we cannot patch up anyways
-        run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs'
+        # CVEs that we cannot patch up anyway
+        run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs,!tests'
 
       - name: Upload report
         uses: actions/upload-artifact@v4
@@ -523,3 +533,44 @@ jobs:
           name: dependency report
           path: target/dependency-check-report.html
           retention-days: 7
+
+  bookkeeper-ci-checks-completed:
+    name: "BookKeeper CI checks completed"
+    if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/bookkeeper')) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [
+      'backward-compatibility-tests',
+      'build-and-license-check',
+      'integration-tests',
+      'jdk-compatibility-checks',
+      'macos-build',
+      'owasp-dependency-check',
+      'typo-check',
+      'unit-tests',
+      'windows-build'
+    ]
+    steps:
+      - name: Check build-and-license-check and typo-check success
+        run: |
+          if [[ ! ( \
+                   "${{ needs.build-and-license-check.result }}" == "success" \
+                && "${{ needs.typo-check.result }}" == "success" \
+               ) ]]; then
+            echo "Required jobs haven't been completed successfully."
+            exit 1
+          fi
+      - name: Check that other required jobs were completed successfully
+        if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
+        run: |
+          if [[ ! ( \
+                   "${{ needs.backward-compatibility-tests.result }}" == "success" \
+                && "${{ needs.integration-tests.result }}" == "success" \
+                && "${{ needs.jdk-compatibility-checks.result }}" == "success" \
+                && "${{ needs.macos-build.result }}" == "success" \
+                && "${{ needs.unit-tests.result }}" == "success" \
+                && "${{ needs.windows-build.result }}" == "success" \
+               ) ]]; then
+            echo "Required jobs haven't been completed successfully."
+            exit 1
+          fi

.github/workflows/codeql.yml

@@ -63,7 +63,7 @@ jobs:
     - name: Cache local Maven repository
       if: steps.check_changes.outputs.docs_only != 'true'
       id: cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: |
           ~/.m2/repository/*/*/*
@@ -79,20 +79,15 @@ jobs:
 
     - name: Set up JDK 11
       if: steps.check_changes.outputs.docs_only != 'true'
-      uses: actions/setup-java@v2
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: 11
 
-    - name: Set up Maven
-      uses: apache/pulsar-test-infra/setup-maven@master
-      with:
-        maven-version: 3.8.7
-
     - name: Validate pull request
       if: steps.check_changes.outputs.docs_only != 'true'
       run: |
-        mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+        mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests
 
     - name: Perform CodeQL Analysis
       if: steps.check_changes.outputs.docs_only != 'true'

.github/workflows/java21-daily-build.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: JDK 21 Daily Build
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
+  workflow_dispatch:
+
+jobs:
+  jdk21-daily-build:
+    name: Build on JDK 21
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+      - name: Build with Maven
+        run: mvn -B clean install
+      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure
+        if: failure()
+        continue-on-error: true
+        uses: ./.github/actions/copy-test-reports
+      - name: Upload Surefire reports
+        uses: actions/upload-artifact@v4
+        if: failure()
+        continue-on-error: true
+        with:
+          name: jdk21-tests-reports
+          path: surefire-reports

.github/workflows/owasp-daily-build.yml

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: OWASP Daily Build
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
+  workflow_dispatch:
+
+jobs:
+  owasp-daily-build:
+    name: OWASP Dependency Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+
+      - name: run "clean install verify" to trigger dependency check
+        # excluding dlfs because it includes hadoop lib with
+        # CVEs that we cannot patch up anyway
+        run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs,!tests'