fix: required fields in nested pydantic models need to follow parent nullability (#2199)

* fix: required fields in nested pydantic models need to follow parent nullability

* test: add inverse test

Author: joscha

dlt/common/libs/pydantic.py

@@ -164,6 +164,10 @@ def pydantic_to_table_schema_columns(
                 result[schema_key] = {
                     **hints,
                     "name": snake_case_naming_convention.make_path(name, hints["name"]),
+                    # if the outer field containing the nested mode is optional,
+                    # then each field in the model itself has to be nullable as well,
+                    # as otherwise we end up with flattened non-nullable optional nested fields
+                    "nullable": hints["nullable"] or nullable,
                 }
         elif data_type == "json" and skip_nested_types:
             continue

tests/libs/test_pydantic.py

@@ -743,3 +743,15 @@ class MyModel(BaseModel):
 
     with pytest.raises(ValidationError):
         m = MyModel(column_type={"data_type": "invalid_type"})  # type: ignore[typeddict-item]
+
+
+def test_parent_nullable_means_children_nullable():
+    class MyParent(BaseModel):
+        optional_child: Optional[ChildModel]
+        non_optional_child: ChildModel
+        dlt_config: ClassVar[DltConfig] = {"skip_nested_types": True}
+
+    schema = pydantic_to_table_schema_columns(MyParent)
+
+    assert schema["optional_child__child_attribute"]["nullable"]
+    assert schema["non_optional_child__child_attribute"]["nullable"] is False