diff --git a/python/examples/evaluate.py b/python/examples/evaluate.py index f9050f39..87041d6b 100644 --- a/python/examples/evaluate.py +++ b/python/examples/evaluate.py @@ -138,7 +138,8 @@ def prepare_criteria_data(run: Run, example: Example): " and/or asks a useful followup question." }, }, - ).as_run_evaluator(prepare_data=prepare_criteria_data), + prepare_data=prepare_criteria_data, + ), ], summary_evaluators=[precision], ) diff --git a/python/langsmith/evaluation/integrations/_langchain.py b/python/langsmith/evaluation/integrations/_langchain.py index e291a9a4..39da628f 100644 --- a/python/langsmith/evaluation/integrations/_langchain.py +++ b/python/langsmith/evaluation/integrations/_langchain.py @@ -123,7 +123,13 @@ class LangChainStringEvaluator: """ # noqa: E501 def __init__( - self, evaluator: Union[StringEvaluator, str], *, config: Optional[dict] = None + self, + evaluator: Union[StringEvaluator, str], + *, + config: Optional[dict] = None, + prepare_data: Optional[ + Callable[[Run, Optional[Example]], SingleEvaluatorInput] + ] = None, ): """Initialize a LangChainStringEvaluator. @@ -143,11 +149,10 @@ def __init__( else: raise NotImplementedError(f"Unsupported evaluator type: {type(evaluator)}") + self._prepare_data = prepare_data + def as_run_evaluator( self, - prepare_data: Optional[ - Callable[[Run, Optional[Example]], SingleEvaluatorInput] - ] = None, ) -> RunEvaluator: """Convert the LangChainStringEvaluator to a RunEvaluator. @@ -234,8 +239,8 @@ def prepare_evaluator_inputs( def evaluate(run: Run, example: Optional[Example] = None) -> dict: eval_inputs = ( prepare_evaluator_inputs(run, example) - if prepare_data is None - else prepare_data(run, example) + if self._prepare_data is None + else self._prepare_data(run, example) ) results = self.evaluator.evaluate_strings(**eval_inputs) return {"key": self.evaluator.evaluation_name, **results}