diff --git a/.agents/skills/sdk-integrations/SKILL.md b/.agents/skills/sdk-integrations/SKILL.md index 54f1d406..89c16c2e 100644 --- a/.agents/skills/sdk-integrations/SKILL.md +++ b/.agents/skills/sdk-integrations/SKILL.md @@ -156,6 +156,8 @@ Canonical shape is OpenAI Chat Completions format. Providers with dedicated UI n The integration that directly owns the model/provider API response owns token accounting. Orchestration/framework integrations should not log token metrics when underlying provider integrations create leaf spans with usage. Agentic parent `task` spans MAY aggregate across children when the framework doesn't delegate to a separately instrumented provider client. Do not add "if OpenAI is patched, skip metrics" checks — define clear ownership instead. +**Framework `llm`-like spans that delegate to a separately instrumented client:** if the framework's per-model-call span sits above a call the framework itself dispatches through another patched client (e.g. DSPy's LM callback → LiteLLM → OpenAI), do NOT type the framework's span `llm`. Two nested `llm` spans for one API call is confusing in the UI and — because the framework typically cannot supply tokens from its own callback contract — would produce a token-less `llm` span that violates the "tokens required for LLM spans" rule. Leave the framework span untyped (or `task`), keep `metadata.model` / `metadata.provider` on it for attribution, and let the underlying provider integration own the `llm` leaf with usage. The same logic applies to any framework that always delegates the transport layer to another instrumentable client. + ### Shaping guidance - Flatten positional args into named fields; normalize SDK objects into dicts/lists/scalars; drop noisy transport fields. diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index 9fa1b52c..f76a246d 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -42,23 +42,30 @@ def test_dspy_callback(memory_logger): spans_by_name = {span["span_attributes"]["name"]: span for span in spans} + module_span = spans_by_name["dspy.module.ChainOfThought"] + assert module_span["span_attributes"]["type"] == "task" + assert module_span["metadata"]["module_class"].endswith("ChainOfThought") + lm_span = spans_by_name["dspy.lm"] assert lm_span["context"]["span_origin"]["instrumentation"]["name"] == "dspy-auto" + assert lm_span["span_attributes"].get("type") != "llm" assert "metadata" in lm_span assert "model" in lm_span["metadata"] assert MODEL in lm_span["metadata"]["model"] + assert lm_span["metadata"]["provider"] == "openai" assert "input" in lm_span assert "output" in lm_span format_span = spans_by_name["dspy.adapter.format"] - parse_span = spans_by_name["dspy.adapter.parse"] - + assert format_span["span_attributes"]["type"] == "task" assert format_span["metadata"]["adapter_class"].endswith("ChatAdapter") assert "signature" in format_span["input"] assert "demos" in format_span["input"] assert "inputs" in format_span["input"] assert isinstance(format_span["output"], list) + parse_span = spans_by_name["dspy.adapter.parse"] + assert parse_span["span_attributes"]["type"] == "task" assert parse_span["metadata"]["adapter_class"].endswith("ChatAdapter") assert "signature" in parse_span["input"] assert "completion" in parse_span["input"] diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index e44b70f1..2348493d 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -2,12 +2,15 @@ from typing import Any -from braintrust.logger import current_span from braintrust.logger import start_span as _bt_start_span +from braintrust.span_types import SpanTypeAttribute _INSTRUMENTATION = "dspy-auto" +_LM_METADATA_PARAM_ALLOWLIST = ("temperature", "max_tokens", "top_p", "top_k", "stop") +_EVALUATE_METADATA_ALLOWLIST = ("accuracy", "score", "total", "correct") + def start_span(*args, **kwargs): internal = dict(kwargs.get("internal") or {}) @@ -16,7 +19,13 @@ def start_span(*args, **kwargs): return _bt_start_span(*args, **kwargs) -from braintrust.span_types import SpanTypeAttribute +def _extract_provider(instance: Any, model: Any) -> str | None: + provider = getattr(instance, "provider", None) + if isinstance(provider, str) and provider: + return provider + if isinstance(model, str) and "/" in model: + return model.split("/", 1)[0] + return None try: @@ -113,24 +122,22 @@ def on_lm_start( instance: The LM instance being called inputs: Input parameters to the LM """ - metadata = {} - if hasattr(instance, "model"): - metadata["model"] = instance.model - if hasattr(instance, "provider"): - metadata["provider"] = str(instance.provider) - - for key in ["temperature", "max_tokens", "top_p", "top_k", "stop"]: + metadata: dict[str, Any] = {} + model = getattr(instance, "model", None) + if model is not None: + metadata["model"] = model + provider = _extract_provider(instance, model) + if provider is not None: + metadata["provider"] = provider + + for key in _LM_METADATA_PARAM_ALLOWLIST: if key in inputs: metadata[key] = inputs[key] - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name="dspy.lm", input=inputs, metadata=metadata, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -191,14 +198,11 @@ def on_module_start( cls_name = cls.__name__ module_name = f"{cls.__module__}.{cls_name}" - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name=f"dspy.module.{cls_name}", + type=SpanTypeAttribute.TASK, input=inputs, metadata={"module_class": module_name}, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -216,11 +220,6 @@ def on_module_end( outputs: Output from the module, or None if there was an exception exception: Exception raised during execution, if any """ - if outputs is not None: - if hasattr(outputs, "toDict"): - outputs = outputs.toDict() - elif hasattr(outputs, "__dict__"): - outputs = outputs.__dict__ self._end_span(call_id, outputs, exception) def _start_adapter_span( @@ -234,14 +233,11 @@ def _start_adapter_span( cls = instance.__class__ metadata = {"adapter_class": f"{cls.__module__}.{cls.__name__}"} - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name=span_name, + type=SpanTypeAttribute.TASK, input=inputs, metadata=metadata, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -327,14 +323,10 @@ def on_tool_start( elif hasattr(instance, "func") and hasattr(instance.func, "__name__"): tool_name = instance.func.__name__ - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name=tool_name, - span_attributes={"type": SpanTypeAttribute.TOOL}, + type=SpanTypeAttribute.TOOL, input=inputs, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -374,14 +366,11 @@ def on_evaluate_start( if hasattr(instance, "num_threads"): metadata["num_threads"] = instance.num_threads - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name="dspy.evaluate", + type=SpanTypeAttribute.TASK, input=inputs, metadata=metadata, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -404,21 +393,18 @@ def on_evaluate_end( return try: - log_data = {} + log_data: dict[str, Any] = {} if exception: log_data["error"] = exception if outputs is not None: log_data["output"] = outputs if isinstance(outputs, dict): - metrics = {} - for key in ["accuracy", "score", "total", "correct"]: + metadata: dict[str, Any] = {} + for key in _EVALUATE_METADATA_ALLOWLIST: if key in outputs: - try: - metrics[key] = float(outputs[key]) - except (ValueError, TypeError): - pass - if metrics: - log_data["metrics"] = metrics + metadata[key] = outputs[key] + if metadata: + log_data["metadata"] = metadata if log_data: span.log(**log_data)