refactor: Use the configuration in HoodieStorageConfig to calculate the estimated proportions

1. Use the configuration in HoodieStorageConfig to calculate the estimated proportions

Signed-off-by: TheR1sing3un <[email protected]>

Author: TheR1sing3un

hudi-common/src/main/java/org/apache/hudi/common/config/HoodieReaderConfig.java

@@ -21,8 +21,6 @@
 
 import javax.annotation.concurrent.Immutable;
 
-import java.util.Map;
-
 /**
  * Configurations for reading a file group
  */
@@ -91,15 +89,4 @@ public class HoodieReaderConfig extends HoodieConfig {
       "hoodie.write.record.merge.custom.implementation.classes";
   public static final String RECORD_MERGE_IMPL_CLASSES_DEPRECATED_WRITE_CONFIG_KEY =
       "hoodie.datasource.write.record.merger.impls";
-
-  public static final ConfigProperty<Double> LOG_FILE_TO_PARQUET_FORMAT_SIZE_ESTIMATION_FRACTION = ConfigProperty
-      .key("hoodie.logfile.to.parquet.format.size.estimation.fraction")
-      .defaultValue(0.80)
-      .markAdvanced()
-      .withDocumentation("Estimate the size of the log file in the parquet file format."
-          + "For AVRO-encoded log blocks, lower this value to get a more accurate estimate");
-
-  public static Double getLogFileToParquetFormatSizeEstimationFraction(Map<String, String> options) {
-    return Double.parseDouble(options.getOrDefault(LOG_FILE_TO_PARQUET_FORMAT_SIZE_ESTIMATION_FRACTION.key(), LOG_FILE_TO_PARQUET_FORMAT_SIZE_ESTIMATION_FRACTION.defaultValue().toString()));
-  }
 }

hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java

@@ -186,7 +186,8 @@ public class HoodieStorageConfig extends HoodieConfig {
       .defaultValue(String.valueOf(0.35))
       .markAdvanced()
       .withDocumentation("Expected additional compression as records move from log files to parquet. Used for merge_on_read "
-          + "table to send inserts into log files & control the size of compacted parquet file.");
+          + "table to send inserts into log files & control the size of compacted parquet file."
+          + "When encoding log blocks in parquet format, increase this value for a more accurate estimation");
 
   // Configs that control the bloom filter that is written to the file footer
   public static final ConfigProperty<String> BLOOM_FILTER_TYPE = ConfigProperty

hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala

@@ -21,7 +21,7 @@ import org.apache.hudi.BaseHoodieTableFileIndex.PartitionPath
 import org.apache.hudi.DataSourceWriteOptions.{PARTITIONPATH_FIELD, PRECOMBINE_FIELD, RECORDKEY_FIELD}
 import org.apache.hudi.HoodieFileIndex.{collectReferencedColumns, convertFilterForTimestampKeyGenerator, getConfigProperties, DataSkippingFailureMode}
 import org.apache.hudi.HoodieSparkConfUtils.getConfigValue
-import org.apache.hudi.common.config.{HoodieMetadataConfig, HoodieReaderConfig, TypedProperties}
+import org.apache.hudi.common.config.{HoodieMetadataConfig, HoodieStorageConfig, TypedProperties}
 import org.apache.hudi.common.config.TimestampKeyGeneratorConfig.{TIMESTAMP_INPUT_DATE_FORMAT, TIMESTAMP_OUTPUT_DATE_FORMAT}
 import org.apache.hudi.common.model.{FileSlice, HoodieBaseFile, HoodieLogFile}
 import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
@@ -169,7 +169,8 @@ case class HoodieFileIndex(spark: SparkSession,
     val prunedPartitionsAndFilteredFileSlices = filterFileSlices(dataFilters, partitionFilters).map {
       case (partitionOpt, fileSlices) =>
         if (shouldEmbedFileSlices) {
-          val logFileEstimationFraction = HoodieReaderConfig.getLogFileToParquetFormatSizeEstimationFraction(options.asJava)
+          val logFileEstimationFraction = options.getOrElse(HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.key(),
+            HoodieStorageConfig.LOGFILE_TO_PARQUET_COMPRESSION_RATIO_FRACTION.defaultValue()).toDouble
           // 1. Generate a disguised representative file for each file slice, which spark uses to optimize rdd partition parallelism based on data such as file size
           // For file slice only has base file, we directly use the base file size as representative file size
           // For file slice has log file, we estimate the representative file size based on the log file size and option(base file) size