Merge pull request #2047 from Expensify/flo_bighash

Update SQLite so that the hash tables in the *-shm file are large enough to fit 128K entries each, instead of the default 4K
Expensify · Jan 22, 2025 · 99ff46b · 99ff46b
2 parents 2073105 + 6b71469
commit 99ff46b
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 15 deletions.
diff --git a/Makefile b/Makefile
@@ -15,7 +15,7 @@ INCLUDE = -I$(PROJECT) -I$(PROJECT)/mbedtls/include
 CXXFLAGS = -g -std=c++20 -fPIC -DSQLITE_ENABLE_NORMALIZE $(BEDROCK_OPTIM_COMPILE_FLAG) -Wall -Werror -Wformat-security -Wno-unqualified-std-cast-call -Wno-error=deprecated-declarations $(INCLUDE)
 
 # Amalgamation flags
-AMALGAMATION_FLAGS = -Wno-unused-but-set-variable -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_ENABLE_NOOP_UPDATE -DSQLITE_MUTEX_ALERT_MILLISECONDS=20 -DHAVE_USLEEP=1 -DSQLITE_MAX_MMAP_SIZE=17592186044416ull -DSQLITE_SHARED_MAPPING -DSQLITE_ENABLE_NORMALIZE -DSQLITE_MAX_PAGE_COUNT=4294967294 -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS -DSQLITE_DEFAULT_CACHE_SIZE=-51200 -DSQLITE_MAX_FUNCTION_ARG=32767
+AMALGAMATION_FLAGS = -Wno-unused-but-set-variable -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_ENABLE_NOOP_UPDATE -DSQLITE_MUTEX_ALERT_MILLISECONDS=20 -DHAVE_USLEEP=1 -DSQLITE_MAX_MMAP_SIZE=17592186044416ull -DSQLITE_SHARED_MAPPING -DSQLITE_ENABLE_NORMALIZE -DSQLITE_MAX_PAGE_COUNT=4294967294 -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS -DSQLITE_DEFAULT_CACHE_SIZE=-51200 -DSQLITE_MAX_FUNCTION_ARG=32767 -DSQLITE_WAL_BIGHASH
 
 # All our intermediate, dependency, object, etc files get hidden in here.
 INTERMEDIATEDIR = .build

diff --git a/libstuff/sqlite3.c b/libstuff/sqlite3.c
@@ -18,7 +18,7 @@
 ** separate file. This file contains only code for the core SQLite library.
 **
 ** The content in this amalgamation comes from Fossil check-in
-** b40cd7395c44b1f2d019d8e809e03de0e083.
+** dc3a24a784c95398656e0d7885f7eb0ee626.
 */
 #define SQLITE_CORE 1
 #define SQLITE_AMALGAMATION 1
@@ -465,7 +465,7 @@ extern "C" {
 */
 #define SQLITE_VERSION        "3.47.0"
 #define SQLITE_VERSION_NUMBER 3047000
-#define SQLITE_SOURCE_ID      "2024-12-20 19:37:41 b40cd7395c44b1f2d019d8e809e03de0e083c93693322a72ddb250a85640528f"
+#define SQLITE_SOURCE_ID      "2024-12-23 11:31:16 dc3a24a784c95398656e0d7885f7eb0ee626b86c896e759a6ac5c243fd6f0ab7"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -66400,8 +66400,13 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0;
 ** version-2 ("journal_mode=wal2"). Legacy clients may support version-1
 ** only.
 */
-#define WAL_VERSION1 3007000      /* For "journal_mode=wal" */
-#define WAL_VERSION2 3021000      /* For "journal_mode=wal2" */
+#ifdef SQLITE_WAL_BIGHASH
+# define WAL_VERSION1 3007001      /* For "journal_mode=wal" */
+# define WAL_VERSION2 3021001      /* For "journal_mode=wal2" */
+#else
+# define WAL_VERSION1 3007000      /* For "journal_mode=wal" */
+# define WAL_VERSION2 3021000      /* For "journal_mode=wal2" */
+#endif
 
 #define SQLITE_ENABLE_WAL2NOCKSUM 1
 
@@ -66786,7 +66791,11 @@ struct Wal {
 ** Each page of the wal-index mapping contains a hash-table made up of
 ** an array of HASHTABLE_NSLOT elements of the following type.
 */
-typedef u16 ht_slot;
+#ifdef SQLITE_WAL_BIGHASH
+ typedef u32 ht_slot;
+#else
+ typedef u16 ht_slot;
+#endif
 
 /*
 ** This structure is used to implement an iterator that loops through
@@ -66823,9 +66832,14 @@ struct WalIterator {
 ** Changing any of these constants will alter the wal-index format and
 ** create incompatibilities.
 */
-#define HASHTABLE_NPAGE      4096                 /* Must be power of 2 */
-#define HASHTABLE_HASH_1     383                  /* Should be prime */
-#define HASHTABLE_NSLOT      (HASHTABLE_NPAGE*2)  /* Must be a power of 2 */
+#ifdef SQLITE_WAL_BIGHASH
+# define HASHTABLE_BITS       17                   /* 128K frames per hash */
+#else
+# define HASHTABLE_BITS       12                   /* 4K frames per hash */
+#endif
+# define HASHTABLE_NPAGE      (1<<HASHTABLE_BITS)  /* Must be power of 2 */
+# define HASHTABLE_HASH_1     383                  /* Should be prime */
+# define HASHTABLE_NSLOT      (HASHTABLE_NPAGE*2)  /* Must be a power of 2 */
 
 /*
 ** The block of page numbers associated with the first hash-table in a
@@ -68166,11 +68180,13 @@ SQLITE_PRIVATE int sqlite3WalOpen(
   assert(    40 ==  sizeof(WalCkptInfo)  );
   assert(   120 ==  WALINDEX_LOCK_OFFSET );
   assert(   136 ==  WALINDEX_HDR_SIZE    );
+#ifndef SQLITE_WAL_BIGHASH
   assert(  4096 ==  HASHTABLE_NPAGE      );
   assert(  4062 ==  HASHTABLE_NPAGE_ONE  );
   assert(  8192 ==  HASHTABLE_NSLOT      );
   assert(   383 ==  HASHTABLE_HASH_1     );
   assert( 32768 ==  WALINDEX_PGSZ        );
+#endif
   assert(     8 ==  SQLITE_SHM_NLOCK     );
   assert(     5 ==  WAL_NREADER          );
   assert(    24 ==  WAL_FRAME_HDRSIZE    );
@@ -68385,7 +68401,7 @@ static void walMergesort(
   ht_slot *aMerge = 0;            /* List to be merged */
   int iList;                      /* Index into input list */
   u32 iSub = 0;                   /* Index into aSub array */
-  struct Sublist aSub[13];        /* Array of sub-lists */
+  struct Sublist aSub[HASHTABLE_BITS+1];    /* Array of sub-lists */
 
   memset(aSub, 0, sizeof(aSub));
   assert( nList<=HASHTABLE_NPAGE && nList>0 );
@@ -93497,7 +93513,7 @@ SQLITE_PRIVATE void sqlite3CommitTimeLog(u64 *aCommit){
       }
       zStr = sqlite3_mprintf("%z%s%s%d%s", zStr, (zStr?", ":""),zHash,iVal,zU);
     }
-    sqlite3_log(SQLITE_WARNING, "slow commit (v=20): (%s)", zStr);
+    sqlite3_log(SQLITE_WARNING, "slow commit (v=21): (%s)", zStr);
     sqlite3_free(zStr);
   }
 }
@@ -93525,7 +93541,7 @@ SQLITE_PRIVATE void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrep
     }
     if( nByte<0 ){ nByte = sqlite3Strlen30(zSql); }
     sqlite3_log(SQLITE_WARNING,
-        "slow prepare (v=20): (%s) [%.*s]", zStr, nByte, zSql
+        "slow prepare (v=21): (%s) [%.*s]", zStr, nByte, zSql
     );
     sqlite3_free(zStr);
   }
@@ -93546,7 +93562,7 @@ SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchema, const char *zFile){
       }
       zStr = sqlite3_mprintf("%z%s%d", zStr, (zStr?", ":""), val);
     }
-    sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=20): (%s)", zFile, zStr);
+    sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=21): (%s)", zFile, zStr);
     sqlite3_free(zStr);
   }
 }
@@ -258026,7 +258042,7 @@ static void fts5SourceIdFunc(
 ){
   assert( nArg==0 );
   UNUSED_PARAM2(nArg, apUnused);
-  sqlite3_result_text(pCtx, "fts5: 2024-12-20 19:37:41 b40cd7395c44b1f2d019d8e809e03de0e083c93693322a72ddb250a85640528f", -1, SQLITE_TRANSIENT);
+  sqlite3_result_text(pCtx, "fts5: 2024-12-23 11:31:16 dc3a24a784c95398656e0d7885f7eb0ee626b86c896e759a6ac5c243fd6f0ab7", -1, SQLITE_TRANSIENT);
 }
 
 /*

diff --git a/libstuff/sqlite3.h b/libstuff/sqlite3.h
@@ -148,7 +148,7 @@ extern "C" {
 */
 #define SQLITE_VERSION        "3.47.0"
 #define SQLITE_VERSION_NUMBER 3047000
-#define SQLITE_SOURCE_ID      "2024-12-20 19:37:41 b40cd7395c44b1f2d019d8e809e03de0e083c93693322a72ddb250a85640528f"
+#define SQLITE_SOURCE_ID      "2024-12-23 11:31:16 dc3a24a784c95398656e0d7885f7eb0ee626b86c896e759a6ac5c243fd6f0ab7"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers