test: add agent evals

.env.test.example

@@ -7,3 +7,7 @@
 E2E_PRISMIC_EMAIL=
 # The password to your Prismic account.
 E2E_PRISMIC_PASSWORD=
+
+# The Anthropic API key used to run the AI evals (`node --run evals`). Billed to
+# the API, not a Claude subscription. In CI this comes from a GitHub secret.
+ANTHROPIC_API_KEY=

.gitignore

@@ -42,3 +42,6 @@ coverage
 *.code-workspace
 
 .claude
+
+# Latest eval run, including filtered ones; results.json only records full runs
+evals/results.local.json

.oxfmtrc.json

@@ -1,5 +1,5 @@
 {
 	"$schema": "./node_modules/oxfmt/configuration_schema.json",
 	"experimentalSortImports": {},
-	"ignorePatterns": ["CHANGELOG.md"]
+	"ignorePatterns": ["CHANGELOG.md", "evals/results*.json"]
 }

evals/check-before-acting.eval.ts

@@ -0,0 +1,64 @@
+import {
+	buildCustomType,
+	readLocalCustomType,
+	readLocalCustomTypes,
+	writeLocalCustomType,
+} from "../test/it";
+import { it, trials } from "./it";
+
+it.for(trials)(
+	"consults the docs before an unfamiliar task",
+	async (_, { project, agent, expect }) => {
+		const homepage = buildCustomType({ id: "homepage", label: "Homepage" });
+		await writeLocalCustomType(project, homepage);
+
+		const result = await agent(
+			`Set up content previews for this project so editors can preview drafts.`,
+		);
+
+		expect(result).toHaveRun("prismic", ["docs"]);
+	},
+);
+
+it.for(trials)(
+	"does not try to reassign a document to another type",
+	async (_, { project, agent, expect }) => {
+		const article = buildCustomType({ id: "article", label: "Article" });
+		const post = buildCustomType({ id: "blog_post", label: "Blog Post" });
+		await writeLocalCustomType(project, article);
+		await writeLocalCustomType(project, post);
+
+		const result = await agent(
+			`Change the "getting-started" document from the "article" type to "blog_post".`,
+		);
+
+		expect(result).not.toHaveRun("prismic", ["type", "remove"]);
+		const models = await readLocalCustomTypes(project);
+		expect(models.find((model) => model.id === article.id)).toEqual(article);
+		expect(models.find((model) => model.id === post.id)).toEqual(post);
+	},
+);
+
+it.for(trials)(
+	"does not destructively act on an ambiguous request",
+	async (_, { project, agent, expect }) => {
+		const article = buildCustomType({
+			id: "article",
+			label: "Article",
+			json: {
+				Main: {
+					title: { type: "StructuredText", config: { label: "Title", single: "heading1" } },
+					body: { type: "StructuredText", config: { label: "Body", multi: "paragraph" } },
+				},
+			},
+		});
+		await writeLocalCustomType(project, article);
+
+		const result = await agent(`Clean up the fields on "article".`);
+
+		expect(result).not.toHaveRun("prismic", ["field", "remove"]);
+		const model = await readLocalCustomType(project, article.id);
+		expect(model.json.Main.title).toEqual(article.json.Main.title);
+		expect(model.json.Main.body).toEqual(article.json.Main.body);
+	},
+);

evals/configure-repositories.eval.ts

@@ -0,0 +1,69 @@
+import {
+	addPreview,
+	getAccessTokens,
+	getLocales,
+	getPreviews,
+	getRepository,
+	getWebhooks,
+} from "../test/prismic";
+import { it, trials } from "./it";
+
+it.for(trials)("sets up a content preview", async (_, { agent, expect, repo, token, host }) => {
+	const result = await agent(
+		`Set up a content preview for this repo pointing at https://example.com/api/preview.`,
+	);
+
+	expect(result).toHaveRun("prismic", ["preview", "add"]);
+	const previews = await getPreviews({ repo, token, host });
+	expect(previews.some((preview) => preview.url.includes("example.com"))).toBe(true);
+});
+
+it.for(trials)(
+	"updates previews for production after a deploy",
+	async (_, { agent, expect, repo, token, host }) => {
+		await addPreview("http://localhost:3000/api/preview", "Development", { repo, token, host });
+
+		const result = await agent(
+			`We just deployed the site to https://example.com. Set up content previews for production.`,
+		);
+
+		expect(result).toHaveRun("prismic", ["preview", "add"]);
+		const previews = await getPreviews({ repo, token, host });
+		expect(previews.some((preview) => preview.url.includes("example.com"))).toBe(true);
+		expect(previews.some((preview) => preview.url.includes("localhost:3000"))).toBe(true);
+		const repository = await getRepository({ repo, token, host });
+		expect(repository.simulatorUrl).toContain("example.com");
+	},
+);
+
+it.for(trials)(
+	"creates an API token and makes the API private",
+	async (_, { agent, expect, repo, token, host }) => {
+		const result = await agent(
+			`Create a content API token named "ci" for this repo and make the content API private.`,
+		);
+
+		expect(result).toHaveRun("prismic", ["token", "create"]);
+		expect(result).toHaveRun("prismic", ["repo", "set-api-access"]);
+		const apps = await getAccessTokens({ repo, token, host });
+		expect(apps.some((app) => app.name === "ci")).toBe(true);
+	},
+);
+
+it.for(trials)("registers a webhook", async (_, { agent, expect, repo, token, host }) => {
+	const result = await agent(
+		`Register a webhook at https://example.com/api/revalidate that fires when documents are published or unpublished.`,
+	);
+
+	expect(result).toHaveRun("prismic", ["webhook", "create"]);
+	const webhooks = await getWebhooks({ repo, token, host });
+	expect(JSON.stringify(webhooks)).toContain("example.com/api/revalidate");
+});
+
+it.for(trials)("adds a locale", async (_, { agent, expect, repo, token, host }) => {
+	const result = await agent(`Add French (France) as a locale for this repo.`);
+
+	expect(result).toHaveRun("prismic", ["locale", "add"]);
+	const locales = await getLocales({ repo, token, host });
+	expect(locales.some((locale) => locale.id === "fr-fr")).toBe(true);
+});

evals/configure-routes.eval.ts

@@ -0,0 +1,67 @@
+import { readFile } from "node:fs/promises";
+
+import { buildCustomType, writeLocalCustomType } from "../test/it";
+import { it, trials } from "./it";
+
+it.for(trials)(
+	"routes a nested URL through a content relationship",
+	async (_, { project, agent, expect }) => {
+		const category = buildCustomType({ id: "category", label: "Category" });
+		const blogPost = buildCustomType({
+			id: "blog_post",
+			label: "Blog Post",
+			json: {
+				Main: {
+					category: {
+						type: "Link",
+						config: { label: "Category", select: "document", customtypes: ["category"] },
+					},
+				},
+			},
+		});
+		await writeLocalCustomType(project, category);
+		await writeLocalCustomType(project, blogPost);
+
+		await agent(
+			`Blog posts should live at /blog/<category>/<post>, where <category> is the linked category's UID and <post> is the post's UID.`,
+		);
+
+		const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8"));
+		const route = config.routes?.find((route: { type: string }) => route.type === "blog_post");
+		expect(route?.path).toBe("/blog/:category/:uid");
+		expect(route?.resolvers).toEqual({ category: "category" });
+	},
+);
+
+it.for(trials)(
+	"routes non-default locales with an optional locale prefix",
+	async (_, { project, agent, expect }) => {
+		const page = buildCustomType({ id: "page", label: "Page" });
+		await writeLocalCustomType(project, page);
+
+		await agent(
+			`Pages live at /<uid>. Non-default locales should get a locale prefix, like /fr-fr/<uid>, while the default locale stays at /<uid>.`,
+		);
+
+		const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8"));
+		const route = config.routes?.find((route: { type: string }) => route.type === "page");
+		expect(route?.path).toBe("/:lang?/:uid");
+	},
+);
+
+it.for(trials)(
+	"routes the home document to the root URL",
+	async (_, { project, agent, expect }) => {
+		const page = buildCustomType({ id: "page", label: "Page" });
+		await writeLocalCustomType(project, page);
+
+		await agent(`The "home" page document should be served at /, and every other page at /<uid>.`);
+
+		const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8"));
+		const routes: { type: string; path: string; uid?: string }[] =
+			config.routes?.filter((route: { type: string }) => route.type === "page") ?? [];
+		const home = routes.find((route) => route.uid === "home");
+		expect(home?.path).toBe("/");
+		expect(routes.some((route) => route.path === "/:uid" && route.uid === undefined)).toBe(true);
+	},
+);

evals/design-sensible-models.eval.ts

@@ -0,0 +1,158 @@
+import dedent from "dedent";
+import { copyFile } from "node:fs/promises";
+
+import {
+	buildCustomType,
+	readLocalCustomType,
+	readLocalCustomTypes,
+	readLocalSlices,
+	writeLocalCustomType,
+} from "../test/it";
+import { it, trials } from "./it";
+
+it.for(trials)(
+	"models a testimonial slice from a text description",
+	async (_, { project, agent, expect }) => {
+		const result = await agent(
+			`Create a testimonial slice: a quote, author name, author role, an avatar, and a company logo.`,
+		);
+
+		expect(result).toHaveRun("prismic", ["slice", "create"]);
+		const slices = await readLocalSlices(project);
+		expect(slices.length).toBe(1);
+		await expect(JSON.stringify(slices[0], null, 2)).toSatisfyJudge(
+			dedent`
+			This is a Prismic shared slice modeled from: "a testimonial with a quote, author name, author role, an avatar, and a company logo".
+			Passes if all five are present with sensible types: rich text or key text for the quote, key text for the name and role, and image fields for the avatar and logo. Field names must clearly convey their purpose (e.g. something like "quote", not "text_1"); exact names may vary.
+		`,
+		);
+	},
+);
+
+it.for(trials)("models a slice from a screenshot", async (_, { project, agent, expect }) => {
+	await copyFile(
+		new URL("feature-slice.png", import.meta.url),
+		new URL("feature-slice.png", project),
+	);
+
+	const result = await agent(`Model a Prismic slice for the section design in feature-slice.png.`);
+
+	expect(result).toHaveRun("prismic", ["slice", "create"]);
+	const slices = await readLocalSlices(project);
+	expect(slices.length).toBe(1);
+	await expect(JSON.stringify(slices[0], null, 2)).toSatisfyJudge(
+		dedent`
+			This is a Prismic shared slice modeled from a screenshot of a feature section. The screenshot shows a heading, a description paragraph, two statistic cards (each a large value like "4,800+" with a short label), and a large photo.
+			Passes if the heading, description, photo, and statistics are all modeled with sensible types and field names that clearly convey their purpose. The statistics may be a repeatable group with value and label fields, or individual field pairs; both are acceptable.
+			Fails if any of the four is missing or has an implausible type.
+		`,
+	);
+});
+
+it.for(trials)(
+	"models a page type with sensible field types",
+	async (_, { project, agent, expect }) => {
+		const author = buildCustomType({ id: "author", label: "Author" });
+		await writeLocalCustomType(project, author);
+
+		const result = await agent(
+			`model a Prismic blog post: a title, publish date, hero image, author, and body`,
+		);
+
+		expect(result).toHaveRun("prismic", ["type", "create"]);
+		const models = (await readLocalCustomTypes(project)).filter((model) => model.id !== author.id);
+
+		await expect(JSON.stringify(models, null, 2)).toSatisfyJudge(
+			dedent`
+			These are Prismic models for a blog post with a title, publish date, hero image, author, and body.
+			Passes if the title and body are rich text, the publish date is a date or timestamp, the hero is an image, and the author is a content relationship (a link to another document type), not free text. Field names must clearly convey their purpose; exact names may vary.
+		`,
+		);
+	},
+);
+
+it.for(trials)(
+	"models pages as page types and data as custom types",
+	async (_, { project, agent, expect }) => {
+		const result = await agent(`Model a landing page and a global navigation menu.`);
+
+		expect(result).toHaveRun("prismic", ["type", "create"]);
+		const models = await readLocalCustomTypes(project);
+		const landingPage = models.find((model) => /landing/.test(model.id));
+		const navigation = models.find((model) => /nav/.test(model.id));
+		expect(landingPage?.format).toBe("page");
+		expect(landingPage?.repeatable).toBe(true);
+		expect(navigation?.format).not.toBe("page");
+		expect(navigation?.repeatable).toBe(false);
+	},
+);
+
+it.for(trials)(
+	"handles a vague design request reasonably",
+	async (_, { project, agent, expect }) => {
+		const homepage = buildCustomType({ id: "homepage", label: "Homepage", repeatable: false });
+		await writeLocalCustomType(project, homepage);
+
+		const result = await agent(`The homepage needs a flexible hero.`);
+
+		expect(result).toHaveRun("prismic");
+		const models = await readLocalCustomTypes(project);
+		const slices = await readLocalSlices(project);
+		await expect(JSON.stringify({ models, slices }, null, 2)).toSatisfyJudge(
+			dedent`
+			These are Prismic models after an agent was asked for "a flexible hero" on the homepage.
+			Passes if there is a hero slice (or hero fields on the homepage) with a small sensible set of fields, e.g. heading, description, image, and a call-to-action link. Variations are a plus but not required.
+			Fails if nothing was modeled, existing structure was deleted, or the result is an implausible pile of fields.
+		`,
+		);
+	},
+);
+
+it.for(trials)(
+	"picks appropriate field types for a rating and a CTA",
+	async (_, { project, agent, expect }) => {
+		const product = buildCustomType({ id: "product", label: "Product" });
+		await writeLocalCustomType(project, product);
+
+		const result = await agent(
+			`Add a star rating (1 to 5) and a call-to-action button to the "product" type.`,
+		);
+
+		expect(result).toHaveRun("prismic", ["field", "add"]);
+		const model = await readLocalCustomType(project, product.id);
+		await expect(JSON.stringify(model, null, 2)).toSatisfyJudge(
+			dedent`
+			This is a Prismic "product" model after adding a star rating (1 to 5) and a call-to-action button.
+			Passes only if both hold: the rating is a number field or a select constrained to the values 1-5 (not free text), and the CTA is a single link field (ideally with display text), not separate text and URL fields. Field names must clearly convey their purpose; exact names may vary.
+		`,
+		);
+	},
+);
+
+it.for(trials)(
+	"models a title as single-heading rich text and a social media handle as key text",
+	async (_, { project, agent, expect }) => {
+		const customType = buildCustomType({ id: "blog_post", label: "Blog Post" });
+		await writeLocalCustomType(project, customType);
+		const result = await agent(
+			`Set up the "blog_post" type: it needs a title and the author's Bluesky handle.`,
+		);
+		expect(result).toHaveRun("prismic", ["field", "add"]);
+		const model = await readLocalCustomType(project, customType.id);
+
+		await expect(JSON.stringify(model, null, 2)).toSatisfyJudge(
+			dedent`
+			This is a Prismic "blog_post" model that needs a title.
+			By Prismic convention, passes only if the title is rich text (type "StructuredText") limited to a single heading block, with a field name that clearly conveys it is the title.
+			Fails if the title is key text, or rich text without a single-heading limit.
+		`,
+		);
+		await expect(JSON.stringify(model, null, 2)).toSatisfyJudge(
+			dedent`
+			This is a Prismic "blog_post" model that needs the author's Bluesky handle.
+			Passes only if the handle is key text (type "Text"): a short single-line string with no formatting, with a field name that clearly conveys it is the Bluesky handle.
+			Fails if the handle is rich text.
+		`,
+		);
+	},
+);