From 9d7e1e0d1be599033b3fcf91e1426c679b72d077 Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Tue, 28 Jul 2026 20:47:47 +0000 Subject: [PATCH 1/3] test(eval): assert the resulting model instead of the field command Co-Authored-By: Claude Fable 5 --- evals/edit-models-precisely.eval.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/evals/edit-models-precisely.eval.ts b/evals/edit-models-precisely.eval.ts index 7f9db7d..3680b9b 100644 --- a/evals/edit-models-precisely.eval.ts +++ b/evals/edit-models-precisely.eval.ts @@ -146,13 +146,16 @@ it.for(trials)( `On the "product" type, add a "size" dropdown with the options S, M, and L, and an "author" link that can only point to "author" documents.`, ); - expect(result).toHaveRun("prismic", ["field", "add", "select"]); - expect(result).toHaveRun("prismic", ["field", "add", "content-relationship"]); + // Both `field add content-relationship` and `field add link --allow document` + // model a constrained relationship; judge the resulting model, not the command. + expect(result).toHaveRun("prismic", ["field", "add"]); const model = await readLocalCustomType(project, product.id); expect(model.json.Main.size.type).toBe("Select"); expect((model.json.Main.size.config as { options: string[] }).options).toEqual(["S", "M", "L"]); expect(model.json.Main.author.type).toBe("Link"); - expect(JSON.stringify(model.json.Main.author.config)).toContain("author"); + const authorConfig = model.json.Main.author.config as { select?: string }; + expect(authorConfig.select).toBe("document"); + expect(JSON.stringify(authorConfig)).toContain("author"); }, ); From 9fe81d8cf04b63bc01cd96afa868692852eedd02 Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Tue, 28 Jul 2026 20:47:47 +0000 Subject: [PATCH 2/3] test(eval): seed a field so the article type looks finished Co-Authored-By: Claude Fable 5 --- evals/sync-models.eval.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/evals/sync-models.eval.ts b/evals/sync-models.eval.ts index 6878966..1c8dc0f 100644 --- a/evals/sync-models.eval.ts +++ b/evals/sync-models.eval.ts @@ -43,7 +43,12 @@ it.for(trials)( await exec("git", ["init"]); await exec("git", ["add", "-A"]); await exec("git", ["commit", "-m", "Initial commit"]); - const article = buildCustomType({ id: "article", label: "Article" }); + // A field so the type looks finished; the agent balks at pushing an empty type. + const article = buildCustomType({ + id: "article", + label: "Article", + json: { Main: { title: { type: "Text", config: { label: "Title" } } } }, + }); await writeLocalCustomType(project, article); const result = await agent( From 7f8c2d46c3fc51e5e36044fdd7f6282ba3207d8c Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Tue, 28 Jul 2026 20:47:47 +0000 Subject: [PATCH 3/3] test(eval): print the agent's final message on toHaveRun failures Co-Authored-By: Claude Fable 5 --- evals/it.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evals/it.ts b/evals/it.ts index bdf5fb5..8a29774 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -122,7 +122,8 @@ expect.extend({ const wanted = [bin, ...positionals].join(" "); if (pass) return `expected no command matching \`${wanted}\`, but one ran`; const seen = result.commands.map((c) => ` ${c}`).join("\n") || " (no commands ran)"; - return `expected a command matching \`${wanted}\`, but saw:\n${seen}`; + const final = "result" in result.result ? result.result.result : ""; + return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${final}`; }, }; },