Add support for setting field_id on nodes (#509)

CurtHagenlocher · web-flow · commit 95c83793e5c8 · 2025-03-14T14:55:01.000+13:00
diff --git a/cpp/GroupNode.cpp b/cpp/GroupNode.cpp
@@ -14,6 +14,7 @@ extern "C"
 		const std::shared_ptr<schema::Node>** fields, 
 		const int numFields, 
 		const std::shared_ptr<const LogicalType>* logical_type, 
+		const int field_id, 
 		std::shared_ptr<schema::Node>** groupNode)
 	{
 		TRYCATCH
@@ -25,7 +26,7 @@ extern "C"
 				nodes.push_back(*fields[i]);
 			}
 
-			*groupNode = new std::shared_ptr(schema::GroupNode::Make(name, repetition, nodes, logical_type == nullptr ? nullptr : *logical_type));
+			*groupNode = new std::shared_ptr(schema::GroupNode::Make(name, repetition, nodes, logical_type == nullptr ? nullptr : *logical_type, field_id));
 		)
 	}
 
diff --git a/cpp/PrimitiveNode.cpp b/cpp/PrimitiveNode.cpp
@@ -14,9 +14,10 @@ extern "C"
 		const std::shared_ptr<const LogicalType>* logical_type, 
 		const Type::type primitive_type, 
 		const int primitive_length, 
+		const int field_id,
 		std::shared_ptr<schema::Node>** primitive_node)
 	{
-		TRYCATCH(*primitive_node = new std::shared_ptr<schema::Node>(schema::PrimitiveNode::Make(name, repetition, logical_type == nullptr ? nullptr : *logical_type, primitive_type, primitive_length));)
+		TRYCATCH(*primitive_node = new std::shared_ptr<schema::Node>(schema::PrimitiveNode::Make(name, repetition, logical_type == nullptr ? nullptr : *logical_type, primitive_type, primitive_length, field_id));)
 	}
 
 	PARQUETSHARP_EXPORT ExceptionInfo* PrimitiveNode_Column_Order(const std::shared_ptr<schema::PrimitiveNode>* node, ColumnOrder::type* column_order)
diff --git a/csharp.test/TestNode.cs b/csharp.test/TestNode.cs
@@ -83,6 +83,22 @@ public static void TestNodeUtf8Name()
             Assert.AreEqual(name, primitiveNode.Name);
         }
 
+        [Test]
+        public static void TestNodeFieldId()
+        {
+            using var groupNode = new GroupNode("group", Repetition.Required, new Node[0], fieldId: 42);
+            using var logicalType = LogicalType.Int(32, true);
+            using var primitiveNode = new PrimitiveNode("primitive", Repetition.Required, logicalType, PhysicalType.Int32, fieldId: 64);
+
+            using var clonedGroupNode = groupNode.DeepClone();
+            using var clonedPrimitiveNode = primitiveNode.DeepClone();
+
+            Assert.AreEqual(42, groupNode.FieldId);
+            Assert.AreEqual(42, clonedGroupNode.FieldId);
+            Assert.AreEqual(64, primitiveNode.FieldId);
+            Assert.AreEqual(64, clonedPrimitiveNode.FieldId);
+        }
+
         /// <summary>
         /// Verify that two nodes are not references to the same instance,
         /// and that none of their fields are.
diff --git a/csharp/PublicAPI/net471/PublicAPI.Shipped.txt b/csharp/PublicAPI/net471/PublicAPI.Shipped.txt
@@ -678,7 +678,8 @@ ParquetSharp.Schema.GroupNode.FieldCount.get -> int
 ParquetSharp.Schema.GroupNode.FieldIndex(ParquetSharp.Schema.Node! node) -> int
 ParquetSharp.Schema.GroupNode.FieldIndex(string! name) -> int
 ParquetSharp.Schema.GroupNode.Fields.get -> ParquetSharp.Schema.Node![]!
-ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType = null) -> void
+ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType) -> void
+ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType = null, int fieldId = -1) -> void
 ParquetSharp.Schema.Node
 ParquetSharp.Schema.Node.ConvertedType.get -> ParquetSharp.Schema.ConvertedType
 ParquetSharp.Schema.Node.Dispose() -> void
@@ -697,7 +698,8 @@ ParquetSharp.Schema.NodeType.Primitive = 0 -> ParquetSharp.Schema.NodeType
 ParquetSharp.Schema.PrimitiveNode
 ParquetSharp.Schema.PrimitiveNode.ColumnOrder.get -> ParquetSharp.ColumnOrder
 ParquetSharp.Schema.PrimitiveNode.PhysicalType.get -> ParquetSharp.PhysicalType
-ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength = -1) -> void
+ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength) -> void
+ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength = -1, int fieldId = -1) -> void
 ParquetSharp.Schema.PrimitiveNode.TypeLength.get -> int
 ParquetSharp.SchemaDescriptor
 ParquetSharp.SchemaDescriptor.Column(int i) -> ParquetSharp.ColumnDescriptor!
diff --git a/csharp/PublicAPI/net6/PublicAPI.Shipped.txt b/csharp/PublicAPI/net6/PublicAPI.Shipped.txt
@@ -682,7 +682,8 @@ ParquetSharp.Schema.GroupNode.FieldCount.get -> int
 ParquetSharp.Schema.GroupNode.FieldIndex(ParquetSharp.Schema.Node! node) -> int
 ParquetSharp.Schema.GroupNode.FieldIndex(string! name) -> int
 ParquetSharp.Schema.GroupNode.Fields.get -> ParquetSharp.Schema.Node![]!
-ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType = null) -> void
+ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType) -> void
+ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType = null, int fieldId = -1) -> void
 ParquetSharp.Schema.Node
 ParquetSharp.Schema.Node.ConvertedType.get -> ParquetSharp.Schema.ConvertedType
 ParquetSharp.Schema.Node.Dispose() -> void
@@ -701,7 +702,8 @@ ParquetSharp.Schema.NodeType.Primitive = 0 -> ParquetSharp.Schema.NodeType
 ParquetSharp.Schema.PrimitiveNode
 ParquetSharp.Schema.PrimitiveNode.ColumnOrder.get -> ParquetSharp.ColumnOrder
 ParquetSharp.Schema.PrimitiveNode.PhysicalType.get -> ParquetSharp.PhysicalType
-ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength = -1) -> void
+ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength) -> void
+ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength = -1, int fieldId = -1) -> void
 ParquetSharp.Schema.PrimitiveNode.TypeLength.get -> int
 ParquetSharp.SchemaDescriptor
 ParquetSharp.SchemaDescriptor.Column(int i) -> ParquetSharp.ColumnDescriptor!
diff --git a/csharp/PublicAPI/netstandard2.1/PublicAPI.Shipped.txt b/csharp/PublicAPI/netstandard2.1/PublicAPI.Shipped.txt
@@ -678,7 +678,8 @@ ParquetSharp.Schema.GroupNode.FieldCount.get -> int
 ParquetSharp.Schema.GroupNode.FieldIndex(ParquetSharp.Schema.Node! node) -> int
 ParquetSharp.Schema.GroupNode.FieldIndex(string! name) -> int
 ParquetSharp.Schema.GroupNode.Fields.get -> ParquetSharp.Schema.Node![]!
-ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType = null) -> void
+ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType) -> void
+ParquetSharp.Schema.GroupNode.GroupNode(string! name, ParquetSharp.Repetition repetition, System.Collections.Generic.IReadOnlyList<ParquetSharp.Schema.Node!>! fields, ParquetSharp.LogicalType? logicalType = null, int fieldId = -1) -> void
 ParquetSharp.Schema.Node
 ParquetSharp.Schema.Node.ConvertedType.get -> ParquetSharp.Schema.ConvertedType
 ParquetSharp.Schema.Node.Dispose() -> void
@@ -697,7 +698,8 @@ ParquetSharp.Schema.NodeType.Primitive = 0 -> ParquetSharp.Schema.NodeType
 ParquetSharp.Schema.PrimitiveNode
 ParquetSharp.Schema.PrimitiveNode.ColumnOrder.get -> ParquetSharp.ColumnOrder
 ParquetSharp.Schema.PrimitiveNode.PhysicalType.get -> ParquetSharp.PhysicalType
-ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength = -1) -> void
+ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength) -> void
+ParquetSharp.Schema.PrimitiveNode.PrimitiveNode(string! name, ParquetSharp.Repetition repetition, ParquetSharp.LogicalType! logicalType, ParquetSharp.PhysicalType physicalType, int primitiveLength = -1, int fieldId = -1) -> void
 ParquetSharp.Schema.PrimitiveNode.TypeLength.get -> int
 ParquetSharp.SchemaDescriptor
 ParquetSharp.SchemaDescriptor.Column(int i) -> ParquetSharp.ColumnDescriptor!
diff --git a/csharp/Schema/GroupNode.cs b/csharp/Schema/GroupNode.cs
@@ -7,8 +7,13 @@ namespace ParquetSharp.Schema
 {
     public sealed class GroupNode : Node
     {
-        public GroupNode(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType = null)
-            : this(Make(name, repetition, fields, logicalType))
+        public GroupNode(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType)
+            : this(Make(name, repetition, fields, logicalType, -1))
+        {
+        }
+
+        public GroupNode(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType = null, int fieldId = -1)
+            : this(Make(name, repetition, fields, logicalType, fieldId))
         {
         }
 
@@ -46,7 +51,8 @@ public override Node DeepClone()
                     Name,
                     Repetition,
                     clonedFields,
-                    logicalType is NoneLogicalType ? null : logicalType);
+                    logicalType is NoneLogicalType ? null : logicalType,
+                    FieldId);
             }
             finally
             {
@@ -57,13 +63,13 @@ public override Node DeepClone()
             }
         }
 
-        private static unsafe IntPtr Make(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType)
+        private static unsafe IntPtr Make(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType, int fieldId)
         {
             var handles = fields.Select(f => f.Handle.IntPtr).ToArray();
 
             fixed (IntPtr* pHandles = handles)
             {
-                ExceptionInfo.Check(GroupNode_Make(name, repetition, (IntPtr) pHandles, handles.Length, logicalType?.Handle.IntPtr ?? IntPtr.Zero, out var groupNode));
+                ExceptionInfo.Check(GroupNode_Make(name, repetition, (IntPtr) pHandles, handles.Length, logicalType?.Handle.IntPtr ?? IntPtr.Zero, fieldId, out var groupNode));
                 GC.KeepAlive(fields);
                 GC.KeepAlive(logicalType);
                 return groupNode;
@@ -72,7 +78,7 @@ private static unsafe IntPtr Make(string name, Repetition repetition, IReadOnlyL
 
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr GroupNode_Make(
-            [MarshalAs(UnmanagedType.LPUTF8Str)] string name, Repetition repetition, IntPtr fields, int numFields, IntPtr logicalType, out IntPtr groupNode);
+            [MarshalAs(UnmanagedType.LPUTF8Str)] string name, Repetition repetition, IntPtr fields, int numFields, IntPtr logicalType, int fieldId, out IntPtr groupNode);
 
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr GroupNode_Field(IntPtr groupNode, int i, out IntPtr field);
diff --git a/csharp/Schema/PrimitiveNode.cs b/csharp/Schema/PrimitiveNode.cs
@@ -15,8 +15,19 @@ public PrimitiveNode(
             Repetition repetition,
             LogicalType logicalType,
             PhysicalType physicalType,
-            int primitiveLength = -1)
-            : this(Make(name, repetition, logicalType, physicalType, primitiveLength))
+            int primitiveLength)
+            : this(name, repetition, logicalType, physicalType, primitiveLength, -1)
+        {
+        }
+
+        public PrimitiveNode(
+            string name,
+            Repetition repetition,
+            LogicalType logicalType,
+            PhysicalType physicalType,
+            int primitiveLength = -1,
+            int fieldId = -1)
+            : this(Make(name, repetition, logicalType, physicalType, primitiveLength, fieldId))
         {
         }
 
@@ -37,27 +48,29 @@ public override Node DeepClone()
                 Repetition,
                 logicalType,
                 PhysicalType,
-                TypeLength);
+                TypeLength,
+                FieldId);
         }
 
         private static IntPtr Make(
             string name,
             Repetition repetition,
             LogicalType logicalType,
             PhysicalType physicalType,
-            int primitiveLength)
+            int primitiveLength,
+            int fieldId)
         {
             if (name == null) throw new ArgumentNullException(nameof(name));
             if (logicalType == null) throw new ArgumentNullException(nameof(logicalType));
 
-            ExceptionInfo.Check(PrimitiveNode_Make(name, repetition, logicalType.Handle.IntPtr, physicalType, primitiveLength, out var primitiveNode));
+            ExceptionInfo.Check(PrimitiveNode_Make(name, repetition, logicalType.Handle.IntPtr, physicalType, primitiveLength, fieldId, out var primitiveNode));
             GC.KeepAlive(logicalType.Handle);
             return primitiveNode;
         }
 
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr PrimitiveNode_Make(
-            [MarshalAs(UnmanagedType.LPUTF8Str)] string name, Repetition repetition, IntPtr logicalType, PhysicalType type, int primitiveLength, out IntPtr primitiveNode);
+            [MarshalAs(UnmanagedType.LPUTF8Str)] string name, Repetition repetition, IntPtr logicalType, PhysicalType type, int primitiveLength, int fieldId, out IntPtr primitiveNode);
 
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr PrimitiveNode_Column_Order(IntPtr node, out ColumnOrder columnOrder);

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,7 @@ extern "C"`
`14`	`14`	`const std::shared_ptr<schema::Node>** fields,`
`15`	`15`	`const int numFields,`
`16`	`16`	`const std::shared_ptr<const LogicalType>* logical_type,`
	`17`	`+ const int field_id,`
`17`	`18`	`std::shared_ptr<schema::Node>** groupNode)`
`18`	`19`	`{`
`19`	`20`	`TRYCATCH`
`@@ -25,7 +26,7 @@ extern "C"`
`25`	`26`	`nodes.push_back(*fields[i]);`
`26`	`27`	`}`
`27`	`28`
`28`		`- groupNode = new std::shared_ptr(schema::GroupNode::Make(name, repetition, nodes, logical_type == nullptr ? nullptr : logical_type));`
	`29`	`+ groupNode = new std::shared_ptr(schema::GroupNode::Make(name, repetition, nodes, logical_type == nullptr ? nullptr : logical_type, field_id));`
`29`	`30`	`)`
`30`	`31`	`}`
`31`	`32`
Original file line number	Diff line number	Diff line change
`@@ -14,9 +14,10 @@ extern "C"`
`14`	`14`	`const std::shared_ptr<const LogicalType>* logical_type,`
`15`	`15`	`const Type::type primitive_type,`
`16`	`16`	`const int primitive_length,`
	`17`	`+ const int field_id,`
`17`	`18`	`std::shared_ptr<schema::Node>** primitive_node)`
`18`	`19`	`{`
`19`		`- TRYCATCH(primitive_node = new std::shared_ptr<schema::Node>(schema::PrimitiveNode::Make(name, repetition, logical_type == nullptr ? nullptr : logical_type, primitive_type, primitive_length));)`
	`20`	`+ TRYCATCH(primitive_node = new std::shared_ptr<schema::Node>(schema::PrimitiveNode::Make(name, repetition, logical_type == nullptr ? nullptr : logical_type, primitive_type, primitive_length, field_id));)`
`20`	`21`	`}`
`21`	`22`
`22`	`23`	`PARQUETSHARP_EXPORT ExceptionInfo* PrimitiveNode_Column_Order(const std::shared_ptr<schema::PrimitiveNode>* node, ColumnOrder::type* column_order)`
Original file line number	Diff line number	Diff line change
`@@ -7,8 +7,13 @@ namespace ParquetSharp.Schema`
`7`	`7`	`{`
`8`	`8`	`public sealed class GroupNode : Node`
`9`	`9`	`{`
`10`		`- public GroupNode(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType = null)`
`11`		`- : this(Make(name, repetition, fields, logicalType))`
	`10`	`+ public GroupNode(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType)`
	`11`	`+ : this(Make(name, repetition, fields, logicalType, -1))`
	`12`	`+ {`
	`13`	`+ }`
	`14`	`+`
	`15`	`+ public GroupNode(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType = null, int fieldId = -1)`
	`16`	`+ : this(Make(name, repetition, fields, logicalType, fieldId))`
`12`	`17`	`{`
`13`	`18`	`}`
`14`	`19`
`@@ -46,7 +51,8 @@ public override Node DeepClone()`
`46`	`51`	`Name,`
`47`	`52`	`Repetition,`
`48`	`53`	`clonedFields,`
`49`		`- logicalType is NoneLogicalType ? null : logicalType);`
	`54`	`+ logicalType is NoneLogicalType ? null : logicalType,`
	`55`	`+ FieldId);`
`50`	`56`	`}`
`51`	`57`	`finally`
`52`	`58`	`{`
`@@ -57,13 +63,13 @@ public override Node DeepClone()`
`57`	`63`	`}`
`58`	`64`	`}`
`59`	`65`
`60`		`- private static unsafe IntPtr Make(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType)`
	`66`	`+ private static unsafe IntPtr Make(string name, Repetition repetition, IReadOnlyList<Node> fields, LogicalType? logicalType, int fieldId)`
`61`	`67`	`{`
`62`	`68`	`var handles = fields.Select(f => f.Handle.IntPtr).ToArray();`
`63`	`69`
`64`	`70`	`fixed (IntPtr* pHandles = handles)`
`65`	`71`	`{`
`66`		`- ExceptionInfo.Check(GroupNode_Make(name, repetition, (IntPtr) pHandles, handles.Length, logicalType?.Handle.IntPtr ?? IntPtr.Zero, out var groupNode));`
	`72`	`+ ExceptionInfo.Check(GroupNode_Make(name, repetition, (IntPtr) pHandles, handles.Length, logicalType?.Handle.IntPtr ?? IntPtr.Zero, fieldId, out var groupNode));`
`67`	`73`	`GC.KeepAlive(fields);`
`68`	`74`	`GC.KeepAlive(logicalType);`
`69`	`75`	`return groupNode;`
`@@ -72,7 +78,7 @@ private static unsafe IntPtr Make(string name, Repetition repetition, IReadOnlyL`
`72`	`78`
`73`	`79`	`[DllImport(ParquetDll.Name)]`
`74`	`80`	`private static extern IntPtr GroupNode_Make(`
`75`		`- [MarshalAs(UnmanagedType.LPUTF8Str)] string name, Repetition repetition, IntPtr fields, int numFields, IntPtr logicalType, out IntPtr groupNode);`
	`81`	`+ [MarshalAs(UnmanagedType.LPUTF8Str)] string name, Repetition repetition, IntPtr fields, int numFields, IntPtr logicalType, int fieldId, out IntPtr groupNode);`
`76`	`82`
`77`	`83`	`[DllImport(ParquetDll.Name)]`
`78`	`84`	`private static extern IntPtr GroupNode_Field(IntPtr groupNode, int i, out IntPtr field);`