Jev 3: add decision records and shared draft primitives

apps/server/src/questions.service.test.ts

@@ -119,12 +119,20 @@ test("a batched ask creates independently addressed decision records and nodes",
 		{
 			status: "answered",
 			resolver: "Storage",
-			answers: [{ question: "Where should room state live?", choices: ["MDX on disk"] }],
+			answers: [{
+				question: "Where should room state live?",
+				choices: ["MDX on disk"],
+				optionIds: [records[0]!.definition.questions[0]!.options[0]!.id],
+			}],
 		},
 		{
 			status: "answered",
 			resolver: "Scope",
-			answers: [{ question: "What belongs in the first cut?", choices: ["Anchors"] }],
+			answers: [{
+				question: "What belongs in the first cut?",
+				choices: ["Anchors"],
+				optionIds: [records[1]!.definition.questions[0]!.options[0]!.id],
+			}],
 		},
 	]);
 });
@@ -356,7 +364,11 @@ test("an appended option is durable in the record, draft store and plan before a
 	if (!edited.open || !edited.accepted) throw new Error("could not choose the option");
 	let claimed = Store.claimSubmit(plan.questions, id, edited.revision, "ana");
 	if (!claimed.ok) throw new Error("could not claim");
-	expect(claimed.answers).toEqual([{ question: item.question, choices: ["A third way"] }]);
+	expect(claimed.answers).toEqual([{
+		question: item.question,
+		choices: ["A third way"],
+		optionIds: [reply.option.id],
+	}]);
 	Store.commit(plan.questions, claimed.claim);
 	await asked.waiting;
 });

apps/server/src/questions/record-fields.ts

@@ -0,0 +1,31 @@
+export const STATUSES = new Set(["open", "answered", "reopened", "discarded", "cancelled"]);
+export const ORIGINS = new Set(["chat", "planner", "human"]);
+export const MAX_UNIX_SECONDS = 253_402_300_799;
+
+export type ObjectValue = { [key: string]: unknown };
+
+export function invalid(): never {
+	throw new Error("hosted channel has an invalid question record");
+}
+
+export function object(value: unknown): ObjectValue {
+	if (!value || typeof value !== "object" || Array.isArray(value)) invalid();
+	return value as ObjectValue;
+}
+
+export function text(value: unknown, max?: number): string {
+	if (typeof value !== "string" || !value.trim() || max !== undefined && value.length > max) {
+		invalid();
+	}
+	return value as string;
+}
+
+export function unix(value: unknown): number {
+	if (
+		typeof value !== "number" || !Number.isSafeInteger(value)
+		|| value < 0 || value > MAX_UNIX_SECONDS
+	) invalid();
+	return value as number;
+}
+
+export type Known = { questions: Map<string, { multiple: boolean }>; options: Map<string, string> };

apps/server/src/questions/record-provenance.ts

@@ -0,0 +1,45 @@
+import type { ConversationPlan } from "@chopin/protocol";
+
+/** A question mention is provenance only when it is the card thread's exact saved ref. */
+export function matchesQuestionSource(
+	source: ConversationPlan.SourceRef,
+	thread: ConversationPlan.Thread | undefined,
+): boolean {
+	return source.role === "question"
+		&& !!thread?.questionSources.some(ref =>
+			ref.role === "question" && ref.messageId === source.messageId
+			&& ref.quote === source.quote && ref.start === source.start && ref.end === source.end
+			&& ref.author.kind === source.author.kind
+			&& (ref.author.kind !== "member" || source.author.kind === "member"
+					&& ref.author.handle === source.author.handle)
+		);
+}
+
+function words(value: string): string[] {
+	return value.normalize("NFKC").toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
+}
+
+function containsWords(haystack: string[], part: string[]): boolean {
+	return part.length > 0
+		&& haystack.some((_, index) => part.every((word, offset) => haystack[index + offset] === word));
+}
+
+/** Match a named alternative as whole, adjacent words in the saved question quote. */
+export function questionMentionsOption(quote: string, label: string): boolean {
+	let mention = words(quote);
+	let option = words(label);
+	if (["use", "using", "choose"].includes(option[0] ?? "")) option.shift();
+	if (option[0] === "host") {
+		if (option[1] === "on") option.splice(0, 2);
+		else {
+			let on = option.indexOf("on", 2);
+			let topicEnd = quote.indexOf(":");
+			if (
+				on > 1 && on <= 6 && topicEnd >= 0
+				&& containsWords(words(quote.slice(0, topicEnd)), option.slice(1, on))
+			) option.splice(0, on + 1);
+		}
+	}
+	if (["a", "an", "the"].includes(option[0] ?? "")) option.shift();
+	return containsWords(mention, option);
+}

apps/server/src/questions/record-types.ts

@@ -0,0 +1,39 @@
+import type { ConversationPlan, Plan as Wired } from "@chopin/protocol";
+import type { Definition } from "@chopin/question";
+
+export type OptionOrigin = {
+	origin: "chat" | "planner" | "human";
+	rationale?: string;
+	by?: string;
+	source?: ConversationPlan.SourceRef;
+};
+
+export type DecisionEntry = {
+	choices: string[];
+	/** Text for older answers without durable option IDs, keyed by question. */
+	answers?: { [questionId: string]: string };
+	owner: string;
+	at: number;
+};
+
+export type Record = {
+	id: string;
+	definition: Definition;
+	/** "answered" is the stored status of a decided card. */
+	status: "open" | "answered" | "reopened" | "discarded" | "cancelled";
+	answers?: { [question: string]: string };
+	resolver?: string;
+	at?: number;
+	anchors?: Wired.WidgetAnchors;
+	origin: "planner" | "conversation";
+	threadId?: string;
+	owner?: string;
+	decidedAt?: number;
+	choices?: string[];
+	history: DecisionEntry[];
+	prose?: Wired.Anchor[];
+	optionOrigins: { [optionId: string]: OptionOrigin };
+	editors: string[];
+	/** Durable request keys for shared option appends, bounded by the option limit. */
+	appended?: { [key: string]: string };
+};

apps/server/src/questions/record-validation.ts

@@ -0,0 +1,162 @@
+import type { ConversationPlan, Plan as Wired } from "@chopin/protocol";
+import { limits } from "@chopin/question";
+import { assertSourceShape } from "../conversation-plan/sources";
+import type { DecisionEntry, Record } from "./record-types";
+import { invalid, object, ORIGINS, text, unix } from "./record-fields";
+import type { Known } from "./record-fields";
+
+export function options(definition: unknown, pending: boolean): Known {
+	let source = object(definition);
+	if (
+		!Array.isArray(source.questions) || source.questions.length === 0
+		|| source.questions.length > limits.MAX_QUESTIONS
+	) invalid();
+	let ids = new Map<string, string>();
+	let questions = new Map<string, { multiple: boolean }>();
+	for (let candidate of source.questions) {
+		let question = object(candidate);
+		let id = text(question.id);
+		if (questions.has(id)) invalid();
+		if (
+			typeof question.multiple !== "boolean"
+			|| !Array.isArray(question.options)
+			|| question.options.length === 0
+				&& !(pending && source.questions.length === 1 && question.multiple === false)
+			|| question.options.length > limits.MAX_OPTIONS
+		) invalid();
+		questions.set(id, { multiple: question.multiple as boolean });
+		for (let candidate of question.options) {
+			let option = object(candidate);
+			let optionId = text(option.id);
+			if (ids.has(optionId)) invalid();
+			ids.set(optionId, id);
+		}
+	}
+	return { questions, options: ids };
+}
+
+export function choices(value: unknown, known: Known): string[] {
+	if (!Array.isArray(value)) invalid();
+	let found = new Set<string>();
+	let counts = new Map<string, number>();
+	for (let id of value) {
+		if (typeof id !== "string" || !known.options.has(id) || found.has(id)) invalid();
+		found.add(id);
+		let question = known.options.get(id)!;
+		let count = (counts.get(question) ?? 0) + 1;
+		if (count > (known.questions.get(question)!.multiple ? limits.MAX_OPTIONS : 1)) invalid();
+		counts.set(question, count);
+	}
+	return value as string[];
+}
+
+export function answers(value: unknown, known: Known): { [question: string]: string } {
+	let entries = object(value);
+	for (let [id, answer] of Object.entries(entries)) {
+		if (!known.questions.has(id)) invalid();
+		text(answer, limits.MAX_CUSTOM);
+	}
+	return value as { [question: string]: string };
+}
+
+export function editors(value: unknown): string[] {
+	if (!Array.isArray(value)) invalid();
+	let found = new Set<string>();
+	for (let item of value) {
+		let handle = text(item);
+		if (found.has(handle)) invalid();
+		found.add(handle);
+	}
+	return value as string[];
+}
+
+export function history(value: unknown, known: Known): DecisionEntry[] {
+	if (!Array.isArray(value)) invalid();
+	for (let candidate of value) {
+		let entry = object(candidate);
+		if (
+			Object.keys(entry).some(key => !["choices", "answers", "owner", "at"].includes(key))
+			|| !Object.hasOwn(entry, "choices")
+			|| !Object.hasOwn(entry, "owner") || !Object.hasOwn(entry, "at")
+		) invalid();
+		let selected = choices(entry.choices, known);
+		let legacy = Object.hasOwn(entry, "answers") ? answers(entry.answers, known) : {};
+		let chosenQuestions = new Set(selected.map(id => known.options.get(id)!));
+		for (let id of Object.keys(legacy)) {
+			if (chosenQuestions.has(id)) invalid();
+			chosenQuestions.add(id);
+		}
+		if (chosenQuestions.size !== known.questions.size) invalid();
+		text(entry.owner);
+		unix(entry.at);
+	}
+	return value as DecisionEntry[];
+}
+
+export function optionOrigins(
+	value: unknown,
+	known: Set<string>,
+	conversationThread: boolean,
+): Record["optionOrigins"] {
+	let origins = object(value);
+	for (let [id, candidate] of Object.entries(origins)) {
+		if (!known.has(id)) invalid();
+		let entry = object(candidate);
+		if (
+			!ORIGINS.has(entry.origin as string)
+			|| Object.keys(entry).some(key =>
+				key !== "origin" && key !== "rationale" && key !== "by" && key !== "source"
+			)
+		) {
+			invalid();
+		}
+		if (Object.hasOwn(entry, "rationale")) text(entry.rationale);
+		if (Object.hasOwn(entry, "by")) text(entry.by);
+		if (Object.hasOwn(entry, "source")) {
+			if (entry.origin !== "planner" && entry.origin !== "chat") invalid();
+			try {
+				assertSourceShape(entry.source);
+				let role = (entry.source as ConversationPlan.SourceRef).role;
+				if (
+					role !== "option" && !(role === "question" && conversationThread
+						&& entry.origin === "planner")
+				) {
+					invalid();
+				}
+			} catch {
+				invalid();
+			}
+		}
+	}
+	return value as Record["optionOrigins"];
+}
+
+export function prose(value: unknown): Wired.Anchor[] {
+	if (!Array.isArray(value)) invalid();
+	for (let candidate of value) {
+		let anchor = object(candidate);
+		if (
+			Object.keys(anchor).some(key =>
+				key !== "epoch" && key !== "position" && key !== "digest" && key !== "orphaned"
+				&& key !== "recoverOnNextEdit"
+			)
+		) invalid();
+		text(anchor.epoch);
+		if (
+			typeof anchor.position !== "string" || !anchor.position
+			|| !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(
+				anchor.position,
+			)
+		) {
+			invalid();
+		}
+		if (typeof anchor.digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(anchor.digest)) {
+			invalid();
+		}
+		if (Object.hasOwn(anchor, "orphaned") && anchor.orphaned !== true) invalid();
+		if (Object.hasOwn(anchor, "recoverOnNextEdit")) {
+			if (anchor.recoverOnNextEdit !== true || anchor.orphaned !== true) invalid();
+		}
+	}
+	return value as Wired.Anchor[];
+}

apps/server/src/questions/records.bounds.test.ts

@@ -0,0 +1,48 @@
+import { expect, test } from "bun:test";
+import { normalizeRecord } from "./records";
+
+test("conversation thread IDs accept 200 UTF-16 units and reject 201", () => {
+	let record = {
+		id: "card-1",
+		status: "open",
+		origin: "conversation",
+		threadId: "🧪".repeat(100),
+		definition: {
+			questions: [{
+				id: "question-1",
+				header: "Auth",
+				question: "Which system?",
+				multiple: false,
+				options: [],
+			}],
+		},
+	};
+	expect(record.threadId).toHaveLength(200);
+	expect(normalizeRecord(record).threadId).toBe(record.threadId);
+	expect(() => normalizeRecord({ ...record, threadId: record.threadId + "x" }))
+		.toThrow("hosted channel has an invalid question record");
+});
+
+test("durable option append keys survive record normalization and reject foreign option IDs", () => {
+	let record = {
+		id: "card-1",
+		status: "open",
+		definition: {
+			questions: [{ id: "question-1", multiple: false, options: [{ id: "option-1" }] }],
+		},
+		appended: { "key-once-0001": "option-1" },
+	};
+	expect(normalizeRecord(record).appended).toEqual(record.appended);
+	for (
+		let appended of [
+			{ bad: "option-1" },
+			{ "key-once-0001": "foreign-option" },
+			{ "key-once-0001": "option-1", "key-once-0002": "option-1" },
+			[],
+			null,
+		]
+	) {
+		expect(() => normalizeRecord({ ...record, appended }))
+			.toThrow("hosted channel has an invalid question record");
+	}
+});