[Python] Add beta evaluate() (#542)

- [X] base  `evaluate` api
- [X] support wrapper to support OTS evaluators from langchain
- [X] add examples

.github/actions/python-integration-tests/action.yml

@@ -10,6 +10,9 @@ inputs:
  openai-api-key:
  description: "OpenAI API key"
  required: false
+ anthropic-api-key:
+ description: "Anthropic API key"
+ required: false
 runs:
  using: "composite"
  steps:
@@ -30,7 +33,7 @@ runs:
  - name: Install dependencies
  run: |
  poetry install --with dev
- poetry run pip install -U langchain
+ poetry run pip install -U langchain langchain_anthropic tiktoken rapidfuzz
  shell: bash
  working-directory: python
 
@@ -42,3 +45,13 @@ runs:
  run: make integration_tests_fast
  shell: bash
  working-directory: python
+
+ - name: Run doctest
+ env:
+ LANGCHAIN_TRACING_V2: "true"
+ LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }}
+ OPENAI_API_KEY: ${{ inputs.openai-api-key }}
+ ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
+ run: make doctest
+ shell: bash
+ working-directory: python

.github/workflows/integration_tests.yml

@@ -49,6 +49,7 @@ jobs:
  python-version: 3.11
  langchain-api-key: ${{ secrets.LANGCHAIN_API_KEY }}
  openai-api-key: ${{ secrets.OPENAI_API_KEY }}
+ anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
 
  js_integration_test:
  name: JS Integration Test

python/Makefile

@@ -12,6 +12,9 @@ integration_tests:
 integration_tests_fast:
  poetry run pytest -n auto --durations=10 -v --cov=langsmith --cov-report=term-missing --cov-report=html --cov-config=.coveragerc tests/integration_tests
 
+doctest:
+ poetry run pytest -n auto --durations=10 --doctest-modules langsmith
+
 lint:
  poetry run ruff .
  poetry run mypy .

python/langsmith/beta/__init__.py

@@ -1,5 +1,6 @@
 """Beta functionality prone to change."""
 
 from langsmith.beta._evals import compute_test_metrics, convert_runs_to_test
+from langsmith.beta._utils import warn_beta
 
-__all__ = ["convert_runs_to_test", "compute_test_metrics"]
+__all__ = ["convert_runs_to_test", "compute_test_metrics", "warn_beta"]

python/langsmith/client.py

@@ -49,11 +49,12 @@
 from langsmith import env as ls_env
 from langsmith import schemas as ls_schemas
 from langsmith import utils as ls_utils
-from langsmith.evaluation import evaluator as ls_evaluator
 
 if TYPE_CHECKING:
  import pandas as pd # type: ignore
 
+ from langsmith.evaluation import evaluator as ls_evaluator
+
 logger = logging.getLogger(__name__)
 _urllib3_logger = logging.getLogger("urllib3.connectionpool")
 
@@ -728,7 +729,9 @@ def request_with_retries(
  args = list(e.args)
  msg = args[1] if len(args) > 1 else ""
  msg = msg.replace("session", "session (project)")
- emsg = "\n".join([args[0]] + [msg] + args[2:])
+ emsg = "\n".join(
+ [str(args[0])] + [msg] + [str(arg) for arg in args[2:]]
+ )
  raise ls_utils.LangSmithError(
  f"Failed to {request_method} {url} in LangSmith API. {emsg}"
  ) from e
@@ -3144,11 +3147,20 @@ def _resolve_example_id(
  def _select_eval_results(
  self,
  results: Union[ls_evaluator.EvaluationResult, ls_evaluator.EvaluationResults],
+ *,
+ fn_name: Optional[str] = None,
  ) -> List[ls_evaluator.EvaluationResult]:
+ from langsmith.evaluation import evaluator as ls_evaluator # noqa: F811
+
  if isinstance(results, ls_evaluator.EvaluationResult):
  results_ = [results]
- elif isinstance(results, dict) and "results" in results:
- results_ = cast(List[ls_evaluator.EvaluationResult], results["results"])
+ elif isinstance(results, dict):
+ if "results" in results:
+ results_ = cast(List[ls_evaluator.EvaluationResult], results["results"])
+ else:
+ results_ = [
+ ls_evaluator.EvaluationResult(**{"key": fn_name, **results})
+ ]
  else:
  raise TypeError(
  f"Invalid evaluation result type {type(results)}."
@@ -3208,15 +3220,20 @@ def _log_evaluation_feedback(
  evaluator_response: Union[
  ls_evaluator.EvaluationResult, ls_evaluator.EvaluationResults
  ],
- run: ls_schemas.Run,
+ run: Optional[ls_schemas.Run] = None,
  source_info: Optional[Dict[str, Any]] = None,
+ project_id: Optional[ID_TYPE] = None,
  ) -> List[ls_evaluator.EvaluationResult]:
  results = self._select_eval_results(evaluator_response)
  for res in results:
  source_info_ = source_info or {}
  if res.evaluator_info:
  source_info_ = {**res.evaluator_info, **source_info_}
- run_id_ = res.target_run_id if res.target_run_id else run.id
+ run_id_ = None
+ if res.target_run_id:
+ run_id_ = res.target_run_id
+ elif run is not None:
+ run_id_ = run.id
  self.create_feedback(
  run_id_,
  res.key,
@@ -3227,6 +3244,7 @@ def _log_evaluation_feedback(
  source_info=source_info_,
  source_run_id=res.source_run_id,
  feedback_source_type=ls_schemas.FeedbackSourceType.MODEL,
+ project_id=project_id,
  )
  return results
 

python/langsmith/evaluation/__init__.py

@@ -1,11 +1,13 @@
 """Evaluation Helpers."""
 
+from langsmith.evaluation._runner import evaluate, evaluate_existing
 from langsmith.evaluation.evaluator import (
  EvaluationResult,
  EvaluationResults,
  RunEvaluator,
  run_evaluator,
 )
+from langsmith.evaluation.integrations._langchain import LangChainStringEvaluator
 from langsmith.evaluation.string_evaluator import StringEvaluator
 
 __all__ = [
@@ -14,4 +16,7 @@
  "EvaluationResults",
  "RunEvaluator",
  "StringEvaluator",
+ "evaluate",
+ "evaluate_existing",
+ "LangChainStringEvaluator",
 ]