From db87b066672051dd19b46da528655b0bc4fc9573 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Fri, 17 Jul 2026 16:34:20 +0000 Subject: [PATCH 1/2] fix(dspy): align span shape with spec, drop over-serialization, avoid nested llm spans - Type module/adapter/evaluate spans as `task` per spec. - Add `metadata.provider` (derived from litellm-style "openai/..." model prefix) and keep `metadata.model` on the LM span. - Drop `outputs.toDict()` / `outputs.__dict__` conversion on module end; braintrust serializes at send time. - Fix `on_evaluate_end` to place aggregate eval stats (accuracy/score/total/correct) in `metadata` via an allowlist instead of inventing new `metrics` keys. - Remove redundant `current_span().export()` parent-passing; start_span picks up the parent from contextvars. - Do NOT type `dspy.lm` as `llm`. DSPy delegates transport to LiteLLM/patched provider clients, which own the `llm` leaf span and its token accounting; typing dspy.lm as `llm` would double-emit and produce a token-less `llm` span. Document the pattern under Token Metrics in the sdk-integrations SKILL. Co-Authored-By: Claude Opus 4.7 --- .agents/skills/sdk-integrations/SKILL.md | 2 + .../braintrust/integrations/dspy/test_dspy.py | 14 +++ .../braintrust/integrations/dspy/tracing.py | 86 +++++++++---------- 3 files changed, 58 insertions(+), 44 deletions(-) diff --git a/.agents/skills/sdk-integrations/SKILL.md b/.agents/skills/sdk-integrations/SKILL.md index 54f1d406..89c16c2e 100644 --- a/.agents/skills/sdk-integrations/SKILL.md +++ b/.agents/skills/sdk-integrations/SKILL.md @@ -156,6 +156,8 @@ Canonical shape is OpenAI Chat Completions format. Providers with dedicated UI n The integration that directly owns the model/provider API response owns token accounting. Orchestration/framework integrations should not log token metrics when underlying provider integrations create leaf spans with usage. Agentic parent `task` spans MAY aggregate across children when the framework doesn't delegate to a separately instrumented provider client. Do not add "if OpenAI is patched, skip metrics" checks — define clear ownership instead. +**Framework `llm`-like spans that delegate to a separately instrumented client:** if the framework's per-model-call span sits above a call the framework itself dispatches through another patched client (e.g. DSPy's LM callback → LiteLLM → OpenAI), do NOT type the framework's span `llm`. Two nested `llm` spans for one API call is confusing in the UI and — because the framework typically cannot supply tokens from its own callback contract — would produce a token-less `llm` span that violates the "tokens required for LLM spans" rule. Leave the framework span untyped (or `task`), keep `metadata.model` / `metadata.provider` on it for attribution, and let the underlying provider integration own the `llm` leaf with usage. The same logic applies to any framework that always delegates the transport layer to another instrumentable client. + ### Shaping guidance - Flatten positional args into named fields; normalize SDK objects into dicts/lists/scalars; drop noisy transport fields. diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index 9fa1b52c..283016f9 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -42,23 +42,37 @@ def test_dspy_callback(memory_logger): spans_by_name = {span["span_attributes"]["name"]: span for span in spans} + # DSPy modules are pipeline steps → task per spec. + module_span = spans_by_name.get("dspy.module.ChainOfThought") + assert module_span is not None + assert module_span["span_attributes"]["type"] == "task" + assert module_span["metadata"]["module_class"].endswith("ChainOfThought") + lm_span = spans_by_name["dspy.lm"] assert lm_span["context"]["span_origin"]["instrumentation"]["name"] == "dspy-auto" + # dspy.lm intentionally has no `llm` type — the underlying provider integration + # (litellm/openai/anthropic) owns the `llm` leaf and its token accounting. + assert lm_span["span_attributes"].get("type") != "llm" assert "metadata" in lm_span assert "model" in lm_span["metadata"] assert MODEL in lm_span["metadata"]["model"] + # Still record provider so downstream tools can attribute the call correctly; + # derived from the litellm-style "openai/..." prefix. + assert lm_span["metadata"]["provider"] == "openai" assert "input" in lm_span assert "output" in lm_span format_span = spans_by_name["dspy.adapter.format"] parse_span = spans_by_name["dspy.adapter.parse"] + assert format_span["span_attributes"]["type"] == "task" assert format_span["metadata"]["adapter_class"].endswith("ChatAdapter") assert "signature" in format_span["input"] assert "demos" in format_span["input"] assert "inputs" in format_span["input"] assert isinstance(format_span["output"], list) + assert parse_span["span_attributes"]["type"] == "task" assert parse_span["metadata"]["adapter_class"].endswith("ChatAdapter") assert "signature" in parse_span["input"] assert "completion" in parse_span["input"] diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index e44b70f1..f6587255 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -2,12 +2,20 @@ from typing import Any -from braintrust.logger import current_span from braintrust.logger import start_span as _bt_start_span +from braintrust.span_types import SpanTypeAttribute _INSTRUMENTATION = "dspy-auto" +# LiteLLM-style model strings are "/" (e.g. "openai/gpt-4o-mini"). +# The prefix identifies the provider whose pricing applies. +_LM_METADATA_PARAM_ALLOWLIST = ("temperature", "max_tokens", "top_p", "top_k", "stop") + +# Aggregate eval stats that DSPy Evaluate surfaces on its return dict. These are +# domain-level scores, not spec-listed span `metrics` keys, so they go in metadata. +_EVALUATE_METADATA_ALLOWLIST = ("accuracy", "score", "total", "correct") + def start_span(*args, **kwargs): internal = dict(kwargs.get("internal") or {}) @@ -16,7 +24,13 @@ def start_span(*args, **kwargs): return _bt_start_span(*args, **kwargs) -from braintrust.span_types import SpanTypeAttribute +def _extract_provider(instance: Any, model: Any) -> str | None: + provider = getattr(instance, "provider", None) + if isinstance(provider, str) and provider: + return provider + if isinstance(model, str) and "/" in model: + return model.split("/", 1)[0] + return None try: @@ -113,24 +127,29 @@ def on_lm_start( instance: The LM instance being called inputs: Input parameters to the LM """ - metadata = {} - if hasattr(instance, "model"): - metadata["model"] = instance.model - if hasattr(instance, "provider"): - metadata["provider"] = str(instance.provider) - - for key in ["temperature", "max_tokens", "top_p", "top_k", "stop"]: + metadata: dict[str, Any] = {} + model = getattr(instance, "model", None) + if model is not None: + metadata["model"] = model + provider = _extract_provider(instance, model) + if provider is not None: + metadata["provider"] = provider + + for key in _LM_METADATA_PARAM_ALLOWLIST: if key in inputs: metadata[key] = inputs[key] - parent = current_span() - parent_export = parent.export() if parent else None - + # dspy.lm is intentionally NOT typed as `llm`. DSPy delegates the actual + # provider call to LiteLLM (or a patched provider client), which owns the + # `llm` leaf span and its token accounting. Typing this parent as `llm` + # would produce two `llm` spans per model call when the underlying + # provider is also instrumented, and this span cannot supply the tokens + # required for `llm` because DSPy's callback contract does not expose + # usage in `outputs`. span = start_span( name="dspy.lm", input=inputs, metadata=metadata, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -191,14 +210,11 @@ def on_module_start( cls_name = cls.__name__ module_name = f"{cls.__module__}.{cls_name}" - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name=f"dspy.module.{cls_name}", + type=SpanTypeAttribute.TASK, input=inputs, metadata={"module_class": module_name}, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -216,11 +232,6 @@ def on_module_end( outputs: Output from the module, or None if there was an exception exception: Exception raised during execution, if any """ - if outputs is not None: - if hasattr(outputs, "toDict"): - outputs = outputs.toDict() - elif hasattr(outputs, "__dict__"): - outputs = outputs.__dict__ self._end_span(call_id, outputs, exception) def _start_adapter_span( @@ -234,14 +245,11 @@ def _start_adapter_span( cls = instance.__class__ metadata = {"adapter_class": f"{cls.__module__}.{cls.__name__}"} - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name=span_name, + type=SpanTypeAttribute.TASK, input=inputs, metadata=metadata, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -327,14 +335,10 @@ def on_tool_start( elif hasattr(instance, "func") and hasattr(instance.func, "__name__"): tool_name = instance.func.__name__ - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name=tool_name, - span_attributes={"type": SpanTypeAttribute.TOOL}, + type=SpanTypeAttribute.TOOL, input=inputs, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -374,14 +378,11 @@ def on_evaluate_start( if hasattr(instance, "num_threads"): metadata["num_threads"] = instance.num_threads - parent = current_span() - parent_export = parent.export() if parent else None - span = start_span( name="dspy.evaluate", + type=SpanTypeAttribute.TASK, input=inputs, metadata=metadata, - parent=parent_export, ) span.set_current() self._spans[call_id] = span @@ -404,21 +405,18 @@ def on_evaluate_end( return try: - log_data = {} + log_data: dict[str, Any] = {} if exception: log_data["error"] = exception if outputs is not None: log_data["output"] = outputs if isinstance(outputs, dict): - metrics = {} - for key in ["accuracy", "score", "total", "correct"]: + metadata: dict[str, Any] = {} + for key in _EVALUATE_METADATA_ALLOWLIST: if key in outputs: - try: - metrics[key] = float(outputs[key]) - except (ValueError, TypeError): - pass - if metrics: - log_data["metrics"] = metrics + metadata[key] = outputs[key] + if metadata: + log_data["metadata"] = metadata if log_data: span.log(**log_data) From e74bb9fa9de32c1a157c1bb14762e2e096a21afe Mon Sep 17 00:00:00 2001 From: Starfolk Date: Fri, 17 Jul 2026 16:37:09 +0000 Subject: [PATCH 2/2] clean up: drop redundant comments The SKILL.md paragraph carries the "why" for dspy.lm; variable names carry the rest. Co-Authored-By: Claude Opus 4.7 --- py/src/braintrust/integrations/dspy/test_dspy.py | 11 ++--------- py/src/braintrust/integrations/dspy/tracing.py | 12 ------------ 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index 283016f9..f76a246d 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -42,29 +42,21 @@ def test_dspy_callback(memory_logger): spans_by_name = {span["span_attributes"]["name"]: span for span in spans} - # DSPy modules are pipeline steps → task per spec. - module_span = spans_by_name.get("dspy.module.ChainOfThought") - assert module_span is not None + module_span = spans_by_name["dspy.module.ChainOfThought"] assert module_span["span_attributes"]["type"] == "task" assert module_span["metadata"]["module_class"].endswith("ChainOfThought") lm_span = spans_by_name["dspy.lm"] assert lm_span["context"]["span_origin"]["instrumentation"]["name"] == "dspy-auto" - # dspy.lm intentionally has no `llm` type — the underlying provider integration - # (litellm/openai/anthropic) owns the `llm` leaf and its token accounting. assert lm_span["span_attributes"].get("type") != "llm" assert "metadata" in lm_span assert "model" in lm_span["metadata"] assert MODEL in lm_span["metadata"]["model"] - # Still record provider so downstream tools can attribute the call correctly; - # derived from the litellm-style "openai/..." prefix. assert lm_span["metadata"]["provider"] == "openai" assert "input" in lm_span assert "output" in lm_span format_span = spans_by_name["dspy.adapter.format"] - parse_span = spans_by_name["dspy.adapter.parse"] - assert format_span["span_attributes"]["type"] == "task" assert format_span["metadata"]["adapter_class"].endswith("ChatAdapter") assert "signature" in format_span["input"] @@ -72,6 +64,7 @@ def test_dspy_callback(memory_logger): assert "inputs" in format_span["input"] assert isinstance(format_span["output"], list) + parse_span = spans_by_name["dspy.adapter.parse"] assert parse_span["span_attributes"]["type"] == "task" assert parse_span["metadata"]["adapter_class"].endswith("ChatAdapter") assert "signature" in parse_span["input"] diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index f6587255..2348493d 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -8,12 +8,7 @@ _INSTRUMENTATION = "dspy-auto" -# LiteLLM-style model strings are "/" (e.g. "openai/gpt-4o-mini"). -# The prefix identifies the provider whose pricing applies. _LM_METADATA_PARAM_ALLOWLIST = ("temperature", "max_tokens", "top_p", "top_k", "stop") - -# Aggregate eval stats that DSPy Evaluate surfaces on its return dict. These are -# domain-level scores, not spec-listed span `metrics` keys, so they go in metadata. _EVALUATE_METADATA_ALLOWLIST = ("accuracy", "score", "total", "correct") @@ -139,13 +134,6 @@ def on_lm_start( if key in inputs: metadata[key] = inputs[key] - # dspy.lm is intentionally NOT typed as `llm`. DSPy delegates the actual - # provider call to LiteLLM (or a patched provider client), which owns the - # `llm` leaf span and its token accounting. Typing this parent as `llm` - # would produce two `llm` spans per model call when the underlying - # provider is also instrumented, and this span cannot supply the tokens - # required for `llm` because DSPy's callback contract does not expose - # usage in `outputs`. span = start_span( name="dspy.lm", input=inputs,