From c6292e9857befa792f69c4c35e641bf325434288 Mon Sep 17 00:00:00 2001 From: evrtt Date: Fri, 17 Jul 2026 13:19:32 -0700 Subject: [PATCH 1/3] fix(framework): normalize score dicts in lists --- py/src/braintrust/framework.py | 11 +++++++- py/src/braintrust/test_framework.py | 40 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py index a1fe7742..14e8429c 100644 --- a/py/src/braintrust/framework.py +++ b/py/src/braintrust/framework.py @@ -1461,12 +1461,21 @@ async def await_or_run_scorer(root_span, scorer, name, **kwargs): raise ValueError(f"When returning a dict, it must be a valid Score object. Got: {result}") from e if isinstance(result, Iterable) and not isinstance(result, (str, bytes, Mapping)): + normalized_scores = [] for s in result: + if isinstance(s, dict): + try: + s = Score.from_dict(s) + except Exception as e: + raise ValueError( + f"When returning an array of scores, each score must be a valid Score object. Got: {s}" + ) from e if not is_score(s): raise ValueError( f"When returning an array of scores, each score must be a valid Score object. Got: {s}" ) - result = list(result) + normalized_scores.append(s) + result = normalized_scores elif is_score(result): result = [result] else: diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py index e204c4c5..5c000bdf 100644 --- a/py/src/braintrust/test_framework.py +++ b/py/src/braintrust/test_framework.py @@ -296,6 +296,46 @@ def _run_eval_sync(self, *args, **kwargs): assert result.summary.scores[scorer_name].score == 1.0 +@pytest.mark.asyncio +async def test_run_evaluator_normalizes_list_of_dict_scores(): + data = [ + EvalCase(input=1, expected=1), + EvalCase(input=2, expected=2), + ] + + def simple_task(input_value): + return input_value + + def list_dict_scorer(input_value, output, expected): + return [ + {"name": "per_result", "score": 1.0}, + {"name": "summary_only", "score": 1.0}, + ] + + evaluator = Evaluator( + project_name="test-project", + eval_name="test-list-dict-scorer", + data=data, + task=simple_task, + scores=[list_dict_scorer], + experiment_name=None, + metadata=None, + ) + + result = await run_evaluator(None, evaluator, None, []) + + assert isinstance(result, EvalResultWithSummary) + assert len(result.results) == 2 + + for eval_result in result.results: + assert eval_result.scores["per_result"] == 1.0 + assert eval_result.scores["summary_only"] == 1.0 + + assert result.summary.project_name == "test-project" + assert result.summary.scores["per_result"].score == 1.0 + assert result.summary.scores["summary_only"].score == 1.0 + + @pytest.mark.asyncio @pytest.mark.skipif( sys.version_info < (3, 14), From 62ead8dd91ac64cbf634605c8fe4ae800fc77f01 Mon Sep 17 00:00:00 2001 From: evrtt Date: Fri, 17 Jul 2026 14:07:30 -0700 Subject: [PATCH 2/3] clean up duplicated error --- py/src/braintrust/framework.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py index 14e8429c..5863290a 100644 --- a/py/src/braintrust/framework.py +++ b/py/src/braintrust/framework.py @@ -1463,17 +1463,17 @@ async def await_or_run_scorer(root_span, scorer, name, **kwargs): if isinstance(result, Iterable) and not isinstance(result, (str, bytes, Mapping)): normalized_scores = [] for s in result: + original_score = s + score_error = None if isinstance(s, dict): try: s = Score.from_dict(s) except Exception as e: - raise ValueError( - f"When returning an array of scores, each score must be a valid Score object. Got: {s}" - ) from e - if not is_score(s): + score_error = e + if score_error is not None or not is_score(s): raise ValueError( - f"When returning an array of scores, each score must be a valid Score object. Got: {s}" - ) + f"When returning an array of scores, each score must be a valid Score object. Got: {original_score}" + ) from score_error normalized_scores.append(s) result = normalized_scores elif is_score(result): From e84eadb79696047182d66d41eb179c80dff699d1 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 20 Jul 2026 17:23:53 -0700 Subject: [PATCH 3/3] ref: Cleanup with helper --- py/src/braintrust/framework.py | 40 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py index 5863290a..091f65e5 100644 --- a/py/src/braintrust/framework.py +++ b/py/src/braintrust/framework.py @@ -1324,6 +1324,19 @@ def _build_classification_span_output( return grouped_output +def _normalize_score(value: Any, error_message: str) -> ScoreLike: + original_value = value + if isinstance(value, dict): + try: + value = Score.from_dict(value) + except Exception as e: + raise ValueError(f"{error_message} Got: {original_value}") from e + + if not is_score(value): + raise ValueError(f"{error_message} Got: {original_value}") + return value + + def _validate_classification_result(value: Any, classifier_name: str) -> Classification: if isinstance(value, Mapping): if not value: @@ -1455,27 +1468,16 @@ async def await_or_run_scorer(root_span, scorer, name, **kwargs): result = await call_user_fn(event_loop, score, **kwargs) if isinstance(result, dict): - try: - result = Score.from_dict(result) - except Exception as e: - raise ValueError(f"When returning a dict, it must be a valid Score object. Got: {result}") from e + result = _normalize_score(result, "When returning a dict, it must be a valid Score object.") if isinstance(result, Iterable) and not isinstance(result, (str, bytes, Mapping)): - normalized_scores = [] - for s in result: - original_score = s - score_error = None - if isinstance(s, dict): - try: - s = Score.from_dict(s) - except Exception as e: - score_error = e - if score_error is not None or not is_score(s): - raise ValueError( - f"When returning an array of scores, each score must be a valid Score object. Got: {original_score}" - ) from score_error - normalized_scores.append(s) - result = normalized_scores + result = [ + _normalize_score( + score, + "When returning an array of scores, each score must be a valid Score object.", + ) + for score in result + ] elif is_score(result): result = [result] else: