chore: patch changes from enterprise (#25776)

- reduce parquet row group size to 100k
- add cli option to disable cached parquet loader

Author: hiltontj

influxdb3_clap_blocks/src/datafusion.rs

@@ -31,6 +31,20 @@ pub struct IoxQueryDatafusionConfig {
     )]
     pub max_parquet_fanout: usize,
 
+    /// Use a cached parquet loader when reading parquet files from object store
+    ///
+    /// This reduces IO operations to a remote object store as parquet is typically read via
+    /// multiple read_range requests which would each require a IO operation. This will cache the
+    /// entire parquet file in memory and serve the read_range requests from the cached data, thus
+    /// requiring a single IO operation.
+    #[clap(
+        long = "datafusion-use-cached-parquet-loader",
+        env = "INFLUXDB3_DATAFUSION_USE_CACHED_PARQUET_LOADER",
+        default_value = "true",
+        action
+    )]
+    pub use_cached_parquet_loader: bool,
+
     /// Provide custom configuration to DataFusion as a comma-separated list of key:value pairs.
     ///
     /// # Example
@@ -64,6 +78,13 @@ impl IoxQueryDatafusionConfig {
             format!("{prefix}.max_parquet_fanout", prefix = IoxConfigExt::PREFIX),
             self.max_parquet_fanout.to_string(),
         );
+        self.datafusion_config.insert(
+            format!(
+                "{prefix}.use_cached_parquet_loader",
+                prefix = IoxConfigExt::PREFIX
+            ),
+            self.use_cached_parquet_loader.to_string(),
+        );
         self.datafusion_config
     }
 }

influxdb3_write/src/persister.rs

@@ -392,7 +392,7 @@ pub struct TrackedMemoryArrowWriter<W: Write + Send> {
 }
 
 /// Parquet row group write size
-pub const ROW_GROUP_WRITE_SIZE: usize = 1024 * 1024;
+pub const ROW_GROUP_WRITE_SIZE: usize = 100_000;
 
 impl<W: Write + Send> TrackedMemoryArrowWriter<W> {
     /// create a new `TrackedMemoryArrowWriter<`