Add EnableStoreDecimalAsInteger to WriterProperties (#572)

Author: haianhng31

cpp/WriterProperties.cpp

@@ -162,4 +162,9 @@ extern "C"
 	{
 		TRYCATCH(*memory_pool = (*writer_properties)->memory_pool();)
 	}
+
+	PARQUETSHARP_EXPORT ExceptionInfo* WriterProperties_Store_Decimal_As_Integer(const std::shared_ptr<WriterProperties>* writer_properties, bool* store_decimal_as_integer)
+	{
+		TRYCATCH(*store_decimal_as_integer = (*writer_properties)->store_decimal_as_integer();)
+	}
 }

cpp/WriterPropertiesBuilder.cpp

@@ -231,4 +231,14 @@ extern "C"
 	{
 		TRYCATCH(builder->memory_pool(pool);)
 	}
+
+	PARQUETSHARP_EXPORT ExceptionInfo* WriterPropertiesBuilder_Enable_Store_Decimal_As_Integer(WriterProperties::Builder* builder)
+	{
+		TRYCATCH(builder->enable_store_decimal_as_integer();)
+	}
+
+	PARQUETSHARP_EXPORT ExceptionInfo* WriterPropertiesBuilder_Disable_Store_Decimal_As_Integer(WriterProperties::Builder* builder)
+	{
+		TRYCATCH(builder->disable_store_decimal_as_integer();)
+	}
 }

csharp.test/TestWriterProperties.cs

@@ -27,6 +27,7 @@ public static void TestDefaultProperties()
             Assert.True(p.WritePageIndex);
             Assert.False(p.PageChecksumEnabled);
             Assert.That(p.MemoryPool.BackendName, Is.Not.Empty);
+            Assert.False(p.StoreDecimalAsInteger);
         }
 
         [Test]
@@ -46,6 +47,7 @@ public static void TestPropertiesBuilder()
                 .DisableWritePageIndex()
                 .EnablePageChecksum()
                 .MemoryPool(MemoryPool.SystemMemoryPool())
+                .EnableStoreDecimalAsInteger()
                 .Build();
 
             Assert.AreEqual("Meeeee!!!", p.CreatedBy);
@@ -61,6 +63,7 @@ public static void TestPropertiesBuilder()
             Assert.False(p.WritePageIndex);
             Assert.True(p.PageChecksumEnabled);
             Assert.AreEqual("system", p.MemoryPool.BackendName);
+            Assert.True(p.StoreDecimalAsInteger);
         }
 
         [Test]

csharp/PublicAPI.Unshipped.txt

@@ -7,3 +7,6 @@ ParquetSharp.ReaderProperties.ThriftContainerSizeLimit.get -> int
 ParquetSharp.ReaderProperties.SetThriftContainerSizeLimit(int size) -> void
 ParquetSharp.ReaderProperties.FooterReadSize.get -> long
 ParquetSharp.ReaderProperties.SetFooterReadSize(long size) -> void
+ParquetSharp.WriterProperties.StoreDecimalAsInteger.get -> bool
+ParquetSharp.WriterPropertiesBuilder.EnableStoreDecimalAsInteger() -> ParquetSharp.WriterPropertiesBuilder!
+ParquetSharp.WriterPropertiesBuilder.DisableStoreDecimalAsInteger() -> ParquetSharp.WriterPropertiesBuilder!

csharp/WriterProperties.cs

@@ -264,6 +264,11 @@ public SortingColumn[] SortingColumns()
         /// </summary>
         public MemoryPool MemoryPool => new MemoryPool(ExceptionInfo.Return<IntPtr>(Handle, WriterProperties_Memory_Pool));
 
+        /// <summary>
+        /// Whether decimals with precision between 1 and 18 (inclusive) are stored as integers.
+        /// </summary>
+        public bool StoreDecimalAsInteger => ExceptionInfo.Return<bool>(Handle, WriterProperties_Store_Decimal_As_Integer);
+
         internal readonly ParquetHandle Handle;
 
         [DllImport(ParquetDll.Name)]
@@ -340,5 +345,8 @@ public SortingColumn[] SortingColumns()
 
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr WriterProperties_Memory_Pool(IntPtr writerProperties, out IntPtr memoryPool);
+
+        [DllImport(ParquetDll.Name)]
+        private static extern IntPtr WriterProperties_Store_Decimal_As_Integer(IntPtr writerProperties, out bool storeDecimalAsInteger);
     }
 }

csharp/WriterPropertiesBuilder.cs

@@ -618,6 +618,37 @@ private void ApplyDefaults()
             });
         }
 
+        /// <summary>
+        /// Allow decimals with precision between 1 and 18 (inclusive) to be stored as integers.
+        /// By default, this is DISABLED and all decimal types annotate fixed_len_byte_array.
+        ///
+        /// When enabled, the Parquet writer will use following physical types to store decimals:
+        /// - int32: for precision between 1 and 9.
+        /// - int64: for precision between 10 and 18.
+        /// - fixed_len_byte_array: for precision > 18.
+        ///
+        /// As a consequence, decimal columns stored in integer types are more compact.
+        /// </summary>
+        public WriterPropertiesBuilder EnableStoreDecimalAsInteger()
+        {
+            ExceptionInfo.Check(WriterPropertiesBuilder_Enable_Store_Decimal_As_Integer(_handle.IntPtr));
+            GC.KeepAlive(_handle);
+            return this;
+        }
+
+        /// <summary>
+        /// Disable decimal logical type with precision between 1 and 18 (inclusive) to be stored
+        /// as integer physical type.
+        /// 
+        /// This is the default.
+        /// </summary>
+        public WriterPropertiesBuilder DisableStoreDecimalAsInteger()
+        {
+            ExceptionInfo.Check(WriterPropertiesBuilder_Disable_Store_Decimal_As_Integer(_handle.IntPtr));
+            GC.KeepAlive(_handle);
+            return this;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static void OnDefaultProperty<T>(T? defaultPropertyValue, Action<T> setProperty)
             where T : struct
@@ -766,6 +797,12 @@ private static void OnDefaultRefProperty<T>(T? defaultPropertyValue, Action<T> s
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr WriterPropertiesBuilder_Memory_Pool(IntPtr builder, IntPtr memoryPool);
 
+        [DllImport(ParquetDll.Name)]
+        private static extern IntPtr WriterPropertiesBuilder_Enable_Store_Decimal_As_Integer(IntPtr builder);
+
+        [DllImport(ParquetDll.Name)]
+        private static extern IntPtr WriterPropertiesBuilder_Disable_Store_Decimal_As_Integer(IntPtr builder);
+
         private readonly ParquetHandle _handle;
     }
 }