dep

.github/actions/python-integration-tests/action.yml

@@ -10,6 +10,9 @@ inputs:
  openai-api-key:
  description: "OpenAI API key"
  required: false
+ anthropic-api-key:
+ description: "Anthropic API key"
+ required: false
 runs:
  using: "composite"
  steps:
@@ -31,6 +34,7 @@ runs:
  run: |
  poetry install --with dev
  poetry run pip install -U langchain
+ poetry run pip install -U langchain_anthropic
  shell: bash
  working-directory: python
 
@@ -48,6 +52,7 @@ runs:
  LANGCHAIN_TRACING_V2: "true"
  LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }}
  OPENAI_API_KEY: ${{ inputs.openai-api-key }}
+ ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
  run: make doctest
  shell: bash
  working-directory: python

.github/workflows/integration_tests.yml

@@ -49,6 +49,7 @@ jobs:
  python-version: 3.11
  langchain-api-key: ${{ secrets.LANGCHAIN_API_KEY }}
  openai-api-key: ${{ secrets.OPENAI_API_KEY }}
+ anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
 
  js_integration_test:
  name: JS Integration Test

python/langsmith/evaluation/evaluator.py

@@ -196,7 +196,7 @@ def __call__(
 
  def __repr__(self) -> str:
  """String representation of the DynamicRunEvaluator object."""
- return f"<DynamicRunEvaluator {self.func.__name__}>"
+ return f"<DynamicRunEvaluator {getattr(self.func, '__name__')}>"
 
 
 def run_evaluator(

python/langsmith/evaluation/integrations/_langchain.py

@@ -169,13 +169,11 @@ def as_run_evaluator(
  else ""
  )
  customization_error_str = f"""
-
-lc_evaluator = load_evaluator('<evaluator_type>', ...)
-def compute_score(run, example):
- evaluation_inputs = {{
+def prepare_data(run, example):
+ return {{
  "prediction": run.outputs['my_output'],{reference_str}{input_str}
  }}
- return lc_evaluator.evaluate_strings(run.outputs['my_output'])
+evaluator = LangChainStringEvaluator(..., prepare_data=prepare_data)
 """
 
  @traceable
@@ -184,9 +182,9 @@ def prepare_evaluator_inputs(
  ) -> SingleEvaluatorInput:
  if run.outputs and len(run.outputs) > 1:
  raise ValueError(
- f"Evaluator {self.evaluator} only supports a single output. "
- "Please ensure that the run has a single output."
- " Or create a custom evaluator yourself:\n\n"
+ f"Evaluator {self.evaluator} only supports a single prediction "
+ "key. Please ensure that the run has a single output."
+ " Or initialize with a prepare_data:\n"
  f"{customization_error_str}"
  )
  if (
@@ -196,9 +194,9 @@ def prepare_evaluator_inputs(
  and len(example.outputs) > 1
  ):
  raise ValueError(
- f"Evaluator {self.evaluator} nly supports a single output. "
+ f"Evaluator {self.evaluator} nly supports a single reference key. "
  "Please ensure that the example has a single output."
- " Or create a custom evaluator yourself:\n\n"
+ " Or create a custom evaluator yourself:\n"
  f"{customization_error_str}"
  )
  if (
@@ -208,9 +206,9 @@ def prepare_evaluator_inputs(
  and len(example.inputs) > 1
  ):
  raise ValueError(
- f"Evaluator {self.evaluator} only supports a single input. "
+ f"Evaluator {self.evaluator} only supports a single input key. "
  "Please ensure that the example has a single input."
- " Or create a custom evaluator yourself:\n\n"
+ " Or initialize with a prepare_data:\n"
  f"{customization_error_str}"
  )