Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions evals/edit-models-precisely.eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,13 +146,16 @@ it.for(trials)(
`On the "product" type, add a "size" dropdown with the options S, M, and L, and an "author" link that can only point to "author" documents.`,
);

expect(result).toHaveRun("prismic", ["field", "add", "select"]);
expect(result).toHaveRun("prismic", ["field", "add", "content-relationship"]);
// Both `field add content-relationship` and `field add link --allow document`
// model a constrained relationship; judge the resulting model, not the command.
expect(result).toHaveRun("prismic", ["field", "add"]);
const model = await readLocalCustomType(project, product.id);
expect(model.json.Main.size.type).toBe("Select");
expect((model.json.Main.size.config as { options: string[] }).options).toEqual(["S", "M", "L"]);
expect(model.json.Main.author.type).toBe("Link");
expect(JSON.stringify(model.json.Main.author.config)).toContain("author");
const authorConfig = model.json.Main.author.config as { select?: string };
expect(authorConfig.select).toBe("document");
expect(JSON.stringify(authorConfig)).toContain("author");
},
);

Expand Down
3 changes: 2 additions & 1 deletion evals/it.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ expect.extend({
const wanted = [bin, ...positionals].join(" ");
if (pass) return `expected no command matching \`${wanted}\`, but one ran`;
const seen = result.commands.map((c) => ` ${c}`).join("\n") || " (no commands ran)";
return `expected a command matching \`${wanted}\`, but saw:\n${seen}`;
const final = "result" in result.result ? result.result.result : "";
return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${final}`;
},
};
},
Expand Down
7 changes: 6 additions & 1 deletion evals/sync-models.eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,12 @@ it.for(trials)(
await exec("git", ["init"]);
await exec("git", ["add", "-A"]);
await exec("git", ["commit", "-m", "Initial commit"]);
const article = buildCustomType({ id: "article", label: "Article" });
// A field so the type looks finished; the agent balks at pushing an empty type.
const article = buildCustomType({
id: "article",
label: "Article",
json: { Main: { title: { type: "Text", config: { label: "Title" } } } },
});
await writeLocalCustomType(project, article);

const result = await agent(
Expand Down
Loading