Upgrade RocksDB to version 9.2.1

bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt

@@ -271,7 +271,7 @@ Apache Software License, Version 2.
 - lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar [22]
 - lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar [22]
 - lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar [22]
-- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23]
+- lib/org.rocksdb-rocksdbjni-9.2.1.jar [23]
 - lib/com.beust-jcommander-1.82.jar [24]
 - lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
 - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
@@ -375,7 +375,7 @@ Apache Software License, Version 2.
 [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
 [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622
-[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2
+[23] Source available at https://github.com/facebook/rocksdb/tree/v9.2.1
 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82
 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -635,7 +635,7 @@ This private header is also used by Apple's open source
     * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
 
 ------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-9.2.1.jar is derived from leveldb, which is under the following license.
 
 Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 

bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt

@@ -271,7 +271,7 @@ Apache Software License, Version 2.
 - lib/org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar [22]
 - lib/org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar [22]
 - lib/org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar [22]
-- lib/org.rocksdb-rocksdbjni-7.10.2.jar [23]
+- lib/org.rocksdb-rocksdbjni-9.2.1.jar [23]
 - lib/com.beust-jcommander-1.82.jar [24]
 - lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
 - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
@@ -371,7 +371,7 @@ Apache Software License, Version 2.
 [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
 [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.48.v20220622
-[23] Source available at https://github.com/facebook/rocksdb/tree/v7.10.2
+[23] Source available at https://github.com/facebook/rocksdb/tree/v9.2.1
 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82
 [25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -630,7 +630,7 @@ This private header is also used by Apple's open source
     * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h
 
 ------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-7.10.2.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-9.2.1.jar is derived from leveldb, which is under the following license.
 
 Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java

@@ -47,6 +47,7 @@
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.CompressionType;
+import org.rocksdb.ConfigOptions;
 import org.rocksdb.DBOptions;
 import org.rocksdb.Env;
 import org.rocksdb.InfoLogLevel;
@@ -154,8 +155,10 @@ private RocksDB initializeRocksDBWithConfFile(String basePath, String subPath, D
         DBOptions dbOptions = new DBOptions();
         final List<ColumnFamilyDescriptor> cfDescs = new ArrayList<>();
         final List<ColumnFamilyHandle> cfHandles = new ArrayList<>();
-        try {
-            OptionsUtil.loadOptionsFromFile(dbFilePath, Env.getDefault(), dbOptions, cfDescs, false);
+        try (final ConfigOptions cfgOpts = new ConfigOptions()
+                .setIgnoreUnknownOptions(false)
+                .setEnv(Env.getDefault())) {
+            OptionsUtil.loadOptionsFromFile(cfgOpts, dbFilePath, dbOptions, cfDescs);
             // Configure file path
             String logPath = conf.getString(ROCKSDB_LOG_PATH, "");
             if (!logPath.isEmpty()) {

pom.xml

@@ -166,7 +166,7 @@
     <protoc3.version>${protobuf.version}</protoc3.version>
     <protoc-gen-grpc-java.version>${grpc.version}</protoc-gen-grpc-java.version>
     <reflections.version>0.9.11</reflections.version>
-    <rocksdb.version>7.10.2</rocksdb.version>
+    <rocksdb.version>9.2.1</rocksdb.version>
     <shrinkwrap.version>3.3.0</shrinkwrap.version>
     <slf4j.version>2.0.12</slf4j.version>
     <snakeyaml.version>2.0</snakeyaml.version>

tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDowngrade.groovy

@@ -18,14 +18,12 @@
 package org.apache.bookkeeper.tests.backwardcompat
 
 import com.github.dockerjava.api.DockerClient
-import java.util.concurrent.TimeUnit
+import com.google.common.collect.Lists
 import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils
 import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader
 import org.jboss.arquillian.junit.Arquillian
 import org.jboss.arquillian.test.api.ArquillianResource
-import org.junit.AfterClass
 import org.junit.Assert
-import org.junit.BeforeClass
 import org.junit.FixMethodOrder
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -61,17 +59,66 @@ class TestCompatUpgradeDowngrade {
 
         LOG.info("Setting ledger storage")
 
-        for (String version: BookKeeperClusterUtils.OLD_CLIENT_VERSIONS) {
+        List<String> versions = Lists.newArrayList(BookKeeperClusterUtils.OLD_CLIENT_VERSIONS)
+        versions.add(BookKeeperClusterUtils.CURRENT_VERSION)
+
+        boolean useRocksDbVersion5 = false
+        boolean useKxxHash = false
+        for (String version: versions) {
             BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
                     "ledgerStorageClass",
                     "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage")
+
+            // versions should be in the increasing order
+            if (version.startsWith("4.17.")) {
+                // rocksDB version 5 or above should be set as default for newer versions
+                useKxxHash = true
+            }
+            if (version.startsWith("4.18.")) {
+                // rocksDB version 5 or above should be set as default for newer versions
+                useRocksDbVersion5 = true
+            }
+
+            try {
+                BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+                        "dbStorage_rocksDB_format_version",
+                        useRocksDbVersion5 ? "5" : "2" )
+                BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+                        "dbStorage_rocksDB_checksum_type",
+                        useKxxHash ? "kxxHash" : "kCRC32c")
+
+                BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+                        "conf/default_rocksdb.conf.default",
+                        "format_version",
+                        useRocksDbVersion5 ? "5" : "2")
+                BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+                        "conf/entry_location_rocksdb.conf.default",
+                        "format_version",
+                        useRocksDbVersion5 ? "5" : "2")
+                BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+                        "conf/ledger_metadata_rocksdb.conf.default",
+                        "format_version",
+                        useRocksDbVersion5 ? "5" : "2")
+
+                BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+                        "conf/default_rocksdb.conf.default",
+                        "checksum",
+                        useKxxHash ? "kxxHash" : "kCRC32c")
+                BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+                        "conf/entry_location_rocksdb.conf.default",
+                        "checksum",
+                        useKxxHash ? "kxxHash" : "kCRC32c")
+                BookKeeperClusterUtils.appendToAllBookieConf(docker, version,
+                        "conf/ledger_metadata_rocksdb.conf.default",
+                        "checksum",
+                        useKxxHash ? "kxxHash" : "kCRC32c")
+
+            } catch (Exception e) {
+                LOG.warn(version + ": Failed to set rocksdb configs, might be ok for some older version", e)
+            }
         }
-        BookKeeperClusterUtils.appendToAllBookieConf(docker, BookKeeperClusterUtils.CURRENT_VERSION,
-                "ledgerStorageClass",
-                "org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage")
     }
 
-
     // will ignore older non-supported versions
 
     @Test

tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java

@@ -143,16 +143,28 @@ public static void updateBookieConf(DockerClient docker, String containerId,
         DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e", sedProgram, confFile);
     }
 
-    public static void appendToAllBookieConf(DockerClient docker, String version, String key, String value)
+    public static void appendToAllBookieConf(DockerClient docker, String version, String confFile0,
+                                             String key, String value)
             throws Exception {
         for (String b : allBookies()) {
-            appendToBookieConf(docker, b, version, key, value);
+            appendToBookieConf(docker, b, version, confFile0, key, value);
         }
     }
 
+    public static void appendToAllBookieConf(DockerClient docker, String version,
+                                             String key, String value)
+            throws Exception {
+        appendToAllBookieConf(docker, version, "conf/bk_server.conf", key, value);
+    }
+
     public static void appendToBookieConf(DockerClient docker, String containerId,
                                         String version, String key, String value) throws Exception {
-        String confFile = "/opt/bookkeeper/" + version + "/conf/bk_server.conf";
+        appendToBookieConf(docker, containerId, version, "conf/bk_server.conf", key, value);
+    }
+
+    public static void appendToBookieConf(DockerClient docker, String containerId,
+                                          String version, String confFile0, String key, String value) throws Exception {
+        String confFile = "/opt/bookkeeper/" + version + "/" + confFile0;
         String sedProgram = String.format("$a%s=%s", key, value);
         DockerUtils.runCommand(docker, containerId, "sed", "-i", "-e", sedProgram, confFile);
     }