Skip to content

Commit

Permalink
[components] Resolver -> Renderer
Browse files Browse the repository at this point in the history
  • Loading branch information
OwenKephart committed Dec 31, 2024
1 parent 174254c commit d46d16c
Show file tree
Hide file tree
Showing 10 changed files with 44 additions and 44 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
from pydantic import TypeAdapter
from typing_extensions import Self

from dagster_components.core.component_rendering import TemplatedValueResolver
from dagster_components.core.component_rendering import TemplatedValueRenderer


class ComponentDeclNode: ...
Expand Down Expand Up @@ -212,7 +212,7 @@ class ComponentLoadContext:
resources: Mapping[str, object]
registry: ComponentTypeRegistry
decl_node: Optional[ComponentDeclNode]
templated_value_resolver: TemplatedValueResolver
templated_value_renderer: TemplatedValueRenderer

@staticmethod
def for_test(
Expand All @@ -225,7 +225,7 @@ def for_test(
resources=resources or {},
registry=registry or ComponentTypeRegistry.empty(),
decl_node=decl_node,
templated_value_resolver=TemplatedValueResolver.default(),
templated_value_renderer=TemplatedValueRenderer.default(),
)

@property
Expand All @@ -240,7 +240,7 @@ def path(self) -> Path:
def with_rendering_scope(self, rendering_scope: Mapping[str, Any]) -> "ComponentLoadContext":
return dataclasses.replace(
self,
templated_value_resolver=self.templated_value_resolver.with_context(**rendering_scope),
templated_value_renderer=self.templated_value_renderer.with_context(**rendering_scope),
)

def for_decl_node(self, decl_node: ComponentDeclNode) -> "ComponentLoadContext":
Expand All @@ -255,7 +255,7 @@ def _raw_params(self) -> Optional[Mapping[str, Any]]:

def load_params(self, params_schema: Type[T]) -> T:
with pushd(str(self.path)):
preprocessed_params = self.templated_value_resolver.render_params(
preprocessed_params = self.templated_value_renderer.render_params(
self._raw_params(), params_schema
)
return TypeAdapter(params_schema).validate_python(preprocessed_params)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
Component,
ComponentLoadContext,
ComponentTypeRegistry,
TemplatedValueResolver,
TemplatedValueRenderer,
get_component_type_name,
is_registered_component_type,
)
Expand Down Expand Up @@ -103,7 +103,7 @@ def build_defs_from_component_path(
resources=resources,
registry=registry,
decl_node=decl_node,
templated_value_resolver=TemplatedValueResolver.default(),
templated_value_renderer=TemplatedValueRenderer.default(),
)
components = load_components_from_context(context)
return defs_from_components(resources=resources, context=context, components=components)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,18 +81,18 @@ class RenderedModel(BaseModel):
model_config = ConfigDict(json_schema_extra={JSON_SCHEMA_EXTRA_KEY: True})

def _render_property(
self, key: str, raw_value: Any, value_resolver: "TemplatedValueResolver"
self, key: str, raw_value: Any, value_renderer: "TemplatedValueRenderer"
) -> Any:
return value_resolver.render_obj(raw_value)
return value_renderer.render_obj(raw_value)

def render_properties(self, value_resolver: "TemplatedValueResolver") -> Mapping[str, Any]:
def render_properties(self, value_renderer: "TemplatedValueRenderer") -> Mapping[str, Any]:
"""Returns a dictionary of rendered properties for this class."""
raw_properties = self.model_dump(exclude_unset=True)

# validate that the rendered properties match the output type
rendered_properties = {}
for k, v in raw_properties.items():
rendered = self._render_property(k, v, value_resolver)
rendered = self._render_property(k, v, value_renderer)
annotation = self.__annotations__[k]
expected_type = _get_expected_type(annotation)
if expected_type is not None:
Expand All @@ -106,17 +106,17 @@ def render_properties(self, value_resolver: "TemplatedValueResolver") -> Mapping


@record
class TemplatedValueResolver:
class TemplatedValueRenderer:
context: Mapping[str, Any]

@staticmethod
def default() -> "TemplatedValueResolver":
return TemplatedValueResolver(
def default() -> "TemplatedValueRenderer":
return TemplatedValueRenderer(
context={"env": _env, "automation_condition": automation_condition_scope()}
)

def with_context(self, **additional_context) -> "TemplatedValueResolver":
return TemplatedValueResolver(context={**self.context, **additional_context})
def with_context(self, **additional_context) -> "TemplatedValueRenderer":
return TemplatedValueRenderer(context={**self.context, **additional_context})

def _render_value(self, val: Any) -> Any:
"""Renders a single value, if it is a templated string."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from dagster_components.core.component_rendering import (
RenderedModel,
RenderingMetadata,
TemplatedValueResolver,
TemplatedValueRenderer,
)


Expand All @@ -42,8 +42,8 @@ class AssetAttributesModel(RenderedModel):
Optional[str], RenderingMetadata(output_type=Optional[AutomationCondition])
] = None

def _render_property(self, key, raw_value, value_resolver):
rendered = super()._render_property(key, raw_value, value_resolver)
def _render_property(self, key, raw_value, value_renderer):
rendered = super()._render_property(key, raw_value, value_renderer)
if key == "key":
# coerce the string asset key into an AssetKey object
return AssetKey.from_user_string(rendered) if rendered else None
Expand All @@ -62,24 +62,24 @@ def _apply_to_spec(self, spec: AssetSpec, attributes: Mapping[str, Any]) -> Asse
def apply_to_spec(
self,
spec: AssetSpec,
value_resolver: TemplatedValueResolver,
value_renderer: TemplatedValueRenderer,
target_keys: AbstractSet[AssetKey],
) -> AssetSpec:
if spec.key not in target_keys:
return spec

# add the original spec to the context and resolve values
return self._apply_to_spec(
spec, self.attributes.render_properties(value_resolver.with_context(asset=spec))
spec, self.attributes.render_properties(value_renderer.with_context(asset=spec))
)

def apply(self, defs: Definitions, value_resolver: TemplatedValueResolver) -> Definitions:
def apply(self, defs: Definitions, value_renderer: TemplatedValueRenderer) -> Definitions:
target_selection = AssetSelection.from_string(self.target, include_sources=True)
target_keys = target_selection.resolve(defs.get_asset_graph())

mappable = [d for d in defs.assets or [] if isinstance(d, (AssetsDefinition, AssetSpec))]
mapped_assets = map_asset_specs(
lambda spec: self.apply_to_spec(spec, value_resolver, target_keys),
lambda spec: self.apply_to_spec(spec, value_renderer, target_keys),
mappable,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from dagster_components import Component, ComponentLoadContext
from dagster_components.core.component import (
ComponentGenerateRequest,
TemplatedValueResolver,
TemplatedValueRenderer,
component_type,
)
from dagster_components.core.dsl_schema import (
Expand Down Expand Up @@ -43,16 +43,16 @@ def __init__(
self,
*,
params: Optional[AssetAttributesModel],
value_resolver: TemplatedValueResolver,
value_renderer: TemplatedValueRenderer,
):
self.params = params or AssetAttributesModel()
self.value_resolver = value_resolver
self.value_renderer = value_renderer

def _get_rendered_attribute(
self, attribute: str, dbt_resource_props: Mapping[str, Any], default_method
) -> Any:
resolver = self.value_resolver.with_context(node=dbt_resource_props)
rendered_attribute = self.params.render_properties(resolver).get(attribute)
renderer = self.value_renderer.with_context(node=dbt_resource_props)
rendered_attribute = self.params.render_properties(renderer).get(attribute)
return (
rendered_attribute
if rendered_attribute is not None
Expand Down Expand Up @@ -102,7 +102,7 @@ def load(cls, context: ComponentLoadContext) -> Self:
op_spec=loaded_params.op,
dbt_translator=DbtProjectComponentTranslator(
params=loaded_params.translator,
value_resolver=context.templated_value_resolver,
value_renderer=context.templated_value_renderer,
),
asset_processors=loaded_params.asset_attributes or [],
)
Expand All @@ -123,7 +123,7 @@ def _fn(context: AssetExecutionContext):

defs = Definitions(assets=[_fn])
for transform in self.asset_processors:
defs = transform.apply(defs, context.templated_value_resolver)
defs = transform.apply(defs, context.templated_value_renderer)
return defs

@classmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def load(cls, context: ComponentLoadContext) -> "PipesSubprocessScriptCollection
if not script_path.exists():
raise FileNotFoundError(f"Script {script_path} does not exist")
path_specs[script_path] = [
AssetSpec(**asset.render_properties(context.templated_value_resolver))
AssetSpec(**asset.render_properties(context.templated_value_renderer))
for asset in script.assets
]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def _fn(context: AssetExecutionContext, sling: SlingResource):

defs = Definitions(assets=[_fn], resources={"sling": self.resource})
for transform in self.asset_processors:
defs = transform.apply(defs, context.templated_value_resolver)
defs = transform.apply(defs, context.templated_value_renderer)
return defs

@classmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from dagster_components.core.component_rendering import (
RenderedModel,
RenderingMetadata,
TemplatedValueResolver,
TemplatedValueRenderer,
can_render_with_default_scope,
)
from pydantic import BaseModel, TypeAdapter, ValidationError
Expand Down Expand Up @@ -71,7 +71,7 @@ def test_render() -> None:
},
}

renderer = TemplatedValueResolver(context={"foo_val": "foo", "bar_val": "bar"})
renderer = TemplatedValueRenderer(context={"foo_val": "foo", "bar_val": "bar"})
rendered_data = renderer.render_params(data, Outer)

assert rendered_data == {
Expand Down Expand Up @@ -104,8 +104,8 @@ def test_valid_rendering() -> None:
the_str="{{ some_str }}",
the_opt_int="{{ some_int }}",
)
resolver = TemplatedValueResolver(context={"some_int": 1, "some_str": "aaa"})
resolved_properties = rm.render_properties(resolver)
renderer = TemplatedValueRenderer(context={"some_int": 1, "some_str": "aaa"})
resolved_properties = rm.render_properties(renderer)

assert resolved_properties == {
"the_renderable_int": 1,
Expand All @@ -123,8 +123,8 @@ def test_invalid_rendering() -> None:
the_opt_int="{{ some_str }}",
)

resolver = TemplatedValueResolver(context={"some_int": 1, "some_str": "aaa"})
renderer = TemplatedValueRenderer(context={"some_int": 1, "some_str": "aaa"})

with pytest.raises(ValidationError):
# string is not a valid output type for the_opt_int
rm.render_properties(resolver)
rm.render_properties(renderer)
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
AssetAttributesModel,
MergeAttributes,
ReplaceAttributes,
TemplatedValueResolver,
TemplatedValueRenderer,
)
from pydantic import BaseModel, TypeAdapter

Expand All @@ -30,7 +30,7 @@ def test_replace_attributes() -> None:
attributes=AssetAttributesModel(tags={"newtag": "newval"}),
)

newdefs = op.apply(defs, TemplatedValueResolver.default())
newdefs = op.apply(defs, TemplatedValueRenderer.default())
asset_graph = newdefs.get_asset_graph()
assert asset_graph.get(AssetKey("a")).tags == {}
assert asset_graph.get(AssetKey("b")).tags == {"newtag": "newval"}
Expand All @@ -44,7 +44,7 @@ def test_merge_attributes() -> None:
attributes=AssetAttributesModel(tags={"newtag": "newval"}),
)

newdefs = op.apply(defs, TemplatedValueResolver.default())
newdefs = op.apply(defs, TemplatedValueRenderer.default())
asset_graph = newdefs.get_asset_graph()
assert asset_graph.get(AssetKey("a")).tags == {}
assert asset_graph.get(AssetKey("b")).tags == {"newtag": "newval"}
Expand All @@ -56,7 +56,7 @@ def test_render_attributes_asset_context() -> None:
attributes=AssetAttributesModel(tags={"group_name_tag": "group__{{ asset.group_name }}"})
)

newdefs = op.apply(defs, TemplatedValueResolver.default().with_context(foo="theval"))
newdefs = op.apply(defs, TemplatedValueRenderer.default().with_context(foo="theval"))
asset_graph = newdefs.get_asset_graph()
assert asset_graph.get(AssetKey("a")).tags == {"group_name_tag": "group__g1"}
assert asset_graph.get(AssetKey("b")).tags == {"group_name_tag": "group__g2"}
Expand All @@ -80,7 +80,7 @@ def _custom_cron(s):
metadata = {"a": 1, "b": "str", "d": 1.23}
newdefs = op.apply(
defs,
TemplatedValueResolver.default().with_context(
TemplatedValueRenderer.default().with_context(
foo="theval", metadata=metadata, custom_cron=_custom_cron
),
)
Expand Down
2 changes: 1 addition & 1 deletion python_modules/libraries/dagster-components/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def get_version() -> str:


ver = get_version()
# dont pin dev installs to avoid pip dep resolver issues
# dont pin dev installs to avoid pip dep renderer issues
pin = "" if ver == "1!0+dev" else f"=={ver}"
setup(
name="dagster-components",
Expand Down

0 comments on commit d46d16c

Please sign in to comment.