From 3baf8b1038f588df1fd3022fa241cc7c548452e3 Mon Sep 17 00:00:00 2001 From: DisturbedCrow Date: Thu, 23 Jul 2026 13:20:45 +0530 Subject: [PATCH 1/3] fix(mcp): require confirmation for similar forgets --- apps/mcp/e2e/helpers.ts | 21 +++- apps/mcp/e2e/memory.test.ts | 123 +++++++++++++++++++++++- apps/mcp/src/client.ts | 186 +++++++++++++++++++++++++++++++----- apps/mcp/src/server.ts | 42 ++++---- 4 files changed, 325 insertions(+), 47 deletions(-) diff --git a/apps/mcp/e2e/helpers.ts b/apps/mcp/e2e/helpers.ts index 1c9c2a52f..a9e07df48 100644 --- a/apps/mcp/e2e/helpers.ts +++ b/apps/mcp/e2e/helpers.ts @@ -161,7 +161,26 @@ export async function forgetUntilForgotten( action: "forget", ...(containerTag ? { containerTag } : {}), }) - if (!res.isError && /forgot/i.test(textOf(res))) return textOf(res) + const previewText = textOf(res) + if (!res.isError && /Successfully forgot/i.test(previewText)) { + return previewText + } + + const confirmationToken = previewText.match(/confirmationToken: (\S+)/)?.[1] + if (confirmationToken) { + const confirmed = await callTool(client, "memory", { + content, + action: "forget", + confirmationToken, + ...(containerTag ? { containerTag } : {}), + }) + if ( + !confirmed.isError && + /Successfully forgot/i.test(textOf(confirmed)) + ) { + return textOf(confirmed) + } + } await sleep(delayMs) } return null diff --git a/apps/mcp/e2e/memory.test.ts b/apps/mcp/e2e/memory.test.ts index 65f936128..cb5dbceeb 100644 --- a/apps/mcp/e2e/memory.test.ts +++ b/apps/mcp/e2e/memory.test.ts @@ -1,12 +1,14 @@ import { randomUUID } from "node:crypto" import { afterAll, beforeAll, describe, expect, it } from "vitest" import { + API_URL, API_KEY, callTool, connect, forgetUntilForgotten, recallUntil, recallUntilAbsent, + sleep, type Session, textOf, } from "./helpers" @@ -20,10 +22,10 @@ describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { }) afterAll(async () => { for (const { content, containerTag } of created) { - await callTool(s.client, "memory", { - content, - action: "forget", - ...(containerTag ? { containerTag } : {}), + await forgetUntilForgotten(s.client, content, { + tries: 3, + delayMs: 1000, + containerTag, }).catch(() => {}) } await s?.close() @@ -91,6 +93,119 @@ describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { } }, 240_000) + it("forget previews similar memories and requires signed confirmation", async () => { + const containerTag = "sm_e2e_safe_forget" + const marker = `safe-fg-${randomUUID()}` + const content = `e2e safe forget target. token=${marker}. Vault color is marigold.` + const scoped = await connect({ containerTag }) + let documentId: string | undefined + + try { + const saved = await callTool(scoped.client, "memory", { + content, + action: "save", + }) + documentId = textOf(saved).match(/id: ([^)]+)/)?.[1] + expect(documentId, "save should return a source document ID").toBeTruthy() + + let extractedMemory = "" + for (let i = 0; i < 18; i++) { + const listed = await callTool(scoped.client, "listMemories") + extractedMemory = + textOf(listed) + .split("\n") + .find((line) => line.startsWith("- ") && line.includes(marker)) + ?.slice(2) ?? "" + if (extractedMemory) break + await sleep(5000) + } + expect( + extractedMemory, + "saved content should produce an extracted memory", + ).toBeTruthy() + const nearMatch = `${extractedMemory} please` + + let previewText = "" + for (let i = 0; i < 18; i++) { + const preview = await callTool(scoped.client, "memory", { + content: nearMatch, + action: "forget", + }) + expect(preview.isError).toBeFalsy() + previewText = textOf(preview) + if (previewText.includes("confirmationToken:")) break + await sleep(5000) + } + expect(previewText).toContain( + "No exact memory matched. No changes were made.", + ) + const confirmationToken = previewText.match( + /confirmationToken: (\S+)/, + )?.[1] + expect( + confirmationToken, + "preview should return a signed confirmation token", + ).toBeTruthy() + expect( + previewText, + "preview should identify the intended candidate", + ).toContain(marker) + + const rejected = await callTool(scoped.client, "memory", { + content: `${nearMatch} altered`, + action: "forget", + confirmationToken, + }) + expect(textOf(rejected)).toContain( + "Invalid or expired forget confirmation. No changes were made.", + ) + + const wrongContainer = await callTool(s.client, "memory", { + content: nearMatch, + action: "forget", + confirmationToken, + containerTag: `${containerTag}_other`, + }) + expect(textOf(wrongContainer)).toContain( + "Invalid or expired forget confirmation. No changes were made.", + ) + + const afterPreview = await callTool(scoped.client, "listMemories") + expect( + textOf(afterPreview), + "preview must not delete the candidate", + ).toContain(marker) + + const confirmed = await callTool(scoped.client, "memory", { + content: nearMatch, + action: "forget", + confirmationToken, + }) + expect(confirmed.isError).toBeFalsy() + expect(textOf(confirmed)).toMatch(/Successfully forgot memory with ID/i) + + let removed = false + for (let i = 0; i < 12; i++) { + const listed = await callTool(scoped.client, "listMemories") + if (!textOf(listed).includes(marker)) { + removed = true + break + } + await sleep(3000) + } + expect(removed, "confirmed candidate should be removed").toBe(true) + } finally { + if (documentId) { + const cleanup = await fetch(`${API_URL}/v3/documents/${documentId}`, { + method: "DELETE", + headers: { Authorization: `Bearer ${API_KEY}` }, + }) + expect(cleanup.ok || cleanup.status === 404).toBe(true) + } + await scoped.close() + } + }, 300_000) + it("containerTag scopes memories (isolation)", async () => { // Fixed tags (not per-run UUIDs) so the test doesn't mint a new project each run. const tagA = "sm_e2e_scope_a" diff --git a/apps/mcp/src/client.ts b/apps/mcp/src/client.ts index 2ce8d41a8..aaaa1a104 100644 --- a/apps/mcp/src/client.ts +++ b/apps/mcp/src/client.ts @@ -3,6 +3,9 @@ import Supermemory from "supermemory" const MAX_CHARS = 200000 // ~50k tokens (character-based limit) const DEFAULT_PROJECT_ID = "sm_project_default" const FETCH_TIMEOUT_MS = 30_000 +const FORGET_CONFIRMATION_TTL_MS = 5 * 60 * 1000 +const textEncoder = new TextEncoder() +const textDecoder = new TextDecoder() interface MemoryRichFields { metadata?: Record | null @@ -170,11 +173,146 @@ export class SupermemoryClient { } } - // Delete/forget memory - try exact match first, then semantic search + private toBase64Url(bytes: Uint8Array): string { + let binary = "" + for (const byte of bytes) binary += String.fromCharCode(byte) + return btoa(binary) + .replaceAll("+", "-") + .replaceAll("/", "_") + .replace(/=+$/, "") + } + + private fromBase64Url(value: string): Uint8Array { + const base64 = value.replaceAll("-", "+").replaceAll("_", "/") + const padding = (4 - (base64.length % 4)) % 4 + const binary = atob(base64.padEnd(base64.length + padding, "=")) + return Uint8Array.from(binary, (character) => character.charCodeAt(0)) + } + + private async contentHash(content: string): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + textEncoder.encode(content), + ) + return this.toBase64Url(new Uint8Array(digest)) + } + + private async confirmationKey(): Promise { + return crypto.subtle.importKey( + "raw", + textEncoder.encode(this.bearerToken), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign", "verify"], + ) + } + + private async createForgetConfirmation( + content: string, + memoryId: string, + ): Promise { + const payload = this.toBase64Url( + textEncoder.encode( + JSON.stringify({ + version: 1, + memoryId, + containerTag: this.containerTag, + contentHash: await this.contentHash(content), + expiresAt: Date.now() + FORGET_CONFIRMATION_TTL_MS, + }), + ), + ) + const signature = await crypto.subtle.sign( + "HMAC", + await this.confirmationKey(), + textEncoder.encode(`mcp-forget-v1.${payload}`), + ) + return `${payload}.${this.toBase64Url(new Uint8Array(signature))}` + } + + private async verifyForgetConfirmation( + content: string, + confirmationToken: string, + ): Promise { + try { + const parts = confirmationToken.split(".") + if (parts.length !== 2) return null + const [payload, signature] = parts + if (!(payload && signature)) return null + + const valid = await crypto.subtle.verify( + "HMAC", + await this.confirmationKey(), + this.fromBase64Url(signature), + textEncoder.encode(`mcp-forget-v1.${payload}`), + ) + if (!valid) return null + + const confirmation = JSON.parse( + textDecoder.decode(this.fromBase64Url(payload)), + ) as Record + if ( + confirmation.version !== 1 || + typeof confirmation.memoryId !== "string" || + confirmation.containerTag !== this.containerTag || + typeof confirmation.contentHash !== "string" || + typeof confirmation.expiresAt !== "number" || + confirmation.expiresAt <= Date.now() || + confirmation.contentHash !== (await this.contentHash(content)) + ) { + return null + } + + return confirmation.memoryId + } catch { + return null + } + } + + // Delete/forget memory by exact content or a signed preview confirmation async forgetMemory( content: string, + confirmationToken?: string, ): Promise<{ success: boolean; message: string; containerTag: string }> { try { + if (confirmationToken) { + const memoryId = await this.verifyForgetConfirmation( + content, + confirmationToken, + ) + if (!memoryId) { + return { + success: false, + message: + "Invalid or expired forget confirmation. No changes were made. Run the forget preview again.", + containerTag: this.containerTag, + } + } + + try { + const result = await this.client.memories.forget({ + id: memoryId, + containerTag: this.containerTag, + }) + return { + success: true, + message: `Successfully forgot memory with ID: ${result.id}`, + containerTag: this.containerTag, + } + } catch (error: unknown) { + const status = + error && typeof error === "object" && "status" in error + ? (error as Record).status + : undefined + if (status !== 404) throw error + return { + success: false, + message: `No memory exists with ID ${memoryId}. No changes were made.`, + containerTag: this.containerTag, + } + } + } + // Try exact content matching first try { const result = await this.client.memories.forget({ @@ -196,43 +334,39 @@ export class SupermemoryClient { if (status !== 404) { throw error } - // Otherwise continue to semantic search fallback + // Otherwise preview semantic candidates without deleting any of them } - // Fallback to semantic search if exact match fails - const SIMILARITY_THRESHOLD = 0.85 // High threshold - only very similar memories - const searchResult = await this.search(content, 5, SIMILARITY_THRESHOLD) - - if (searchResult.results.length === 0) { - return { - success: false, - message: `No matching memory found to forget. Tried exact match and semantic search with similarity threshold ${SIMILARITY_THRESHOLD}.`, - containerTag: this.containerTag, - } - } + const SIMILARITY_THRESHOLD = 0.85 + const searchResult = await this.search(content, 5, SIMILARITY_THRESHOLD, { + searchMode: "memories", + }) + const candidates = searchResult.results.filter( + (result) => "memory" in result, + ) - // Only actual memories (not chunks) can be forgotten - const memoryToDelete = searchResult.results.find((r) => "memory" in r) - if (!memoryToDelete) { + if (candidates.length === 0) { return { success: false, message: - "No matching memory found to forget (only document chunks matched in semantic search).", + "No exact memory matched. No changes were made, and no similar memory candidates were found.", containerTag: this.containerTag, } } - // Delete using the ID from semantic search - await this.client.memories.forget({ - id: memoryToDelete.id, - containerTag: this.containerTag, - }) + const candidateLines = await Promise.all( + candidates.map(async (candidate) => { + const confirmation = await this.createForgetConfirmation( + content, + candidate.id, + ) + return `- confirmationToken: ${confirmation}\n similarity: ${candidate.similarity.toFixed(2)}\n content: ${JSON.stringify(limitByChars(getMemoryText(candidate) || candidate.content || "", 200))}` + }), + ) - const memoryText = - getMemoryText(memoryToDelete) || memoryToDelete.content || "" return { - success: true, - message: `Forgot similar memory (semantic match, similarity: ${memoryToDelete.similarity.toFixed(2)}): "${limitByChars(memoryText, 100)}"`, + success: false, + message: `No exact memory matched. No changes were made.\n\nSimilar memory candidates (preview only):\n${candidateLines.join("\n")}\n\nAsk the user to confirm one candidate, then call memory again with action "forget", the same content, and that confirmationToken. Confirmations expire after 5 minutes.`, containerTag: this.containerTag, } } catch (error) { diff --git a/apps/mcp/src/server.ts b/apps/mcp/src/server.ts index 86012e687..1c67c9f5b 100644 --- a/apps/mcp/src/server.ts +++ b/apps/mcp/src/server.ts @@ -94,6 +94,14 @@ export class SupermemoryMCP extends McpAgent { .max(200000, "Content exceeds maximum length of 200,000 characters") .describe("The memory content to save or forget"), action: z.enum(["save", "forget"]).optional().default("save"), + confirmationToken: z + .string() + .min(1) + .max(2048) + .describe( + "For action 'forget' only: a signed token returned by a previous forget preview. After the user confirms that candidate, resend the same content with this token within 5 minutes.", + ) + .optional(), ...(hasRootContainerTag ? {} : containerTagField), }) @@ -146,7 +154,7 @@ export class SupermemoryMCP extends McpAgent { "memory", { description: - "DO NOT USE ANY OTHER MEMORY TOOL ONLY USE THIS ONE. Save or forget information about the user. Use 'save' when user shares preferences, facts, or asks to remember something. Use 'forget' when information is outdated or user requests removal.", + "DO NOT USE ANY OTHER MEMORY TOOL ONLY USE THIS ONE. Save or forget information about the user. Use 'save' when user shares preferences, facts, or asks to remember something. Forget removes only exact content. If no exact match exists, it returns similar candidates without changing them; ask the user to confirm one, then retry with the same content and its confirmationToken within 5 minutes.", inputSchema: memorySchema, annotations: MEMORY_TOOL_ANNOTATIONS, }, @@ -594,9 +602,10 @@ export class SupermemoryMCP extends McpAgent { private async handleMemory(args: { content: string action?: "save" | "forget" + confirmationToken?: string containerTag?: string }) { - const { content, action = "save", containerTag } = args + const { content, action = "save", confirmationToken, containerTag } = args const effectiveContainerTag = containerTag || this.props?.containerTag try { @@ -604,20 +613,21 @@ export class SupermemoryMCP extends McpAgent { const clientInfo = await this.getClientInfo() if (action === "forget") { - const result = await client.forgetMemory(content) - - // Track forget event - posthog - .memoryForgot({ - userId: this.props?.userId || "unknown", - content_length: content.length, - source: "mcp", - mcp_client_name: clientInfo?.name, - mcp_client_version: clientInfo?.version, - sessionId: this.getMcpSessionId(), - containerTag: result.containerTag, - }) - .catch((error) => console.error("PostHog tracking error:", error)) + const result = await client.forgetMemory(content, confirmationToken) + + if (result.success) { + posthog + .memoryForgot({ + userId: this.props?.userId || "unknown", + ...(confirmationToken ? {} : { content_length: content.length }), + source: "mcp", + mcp_client_name: clientInfo?.name, + mcp_client_version: clientInfo?.version, + sessionId: this.getMcpSessionId(), + containerTag: result.containerTag, + }) + .catch((error) => console.error("PostHog tracking error:", error)) + } return { content: [ From 4c1f796c6b771c20e6f18b698c563f71cd29e71c Mon Sep 17 00:00:00 2001 From: DisturbedCrow Date: Thu, 23 Jul 2026 15:17:45 +0530 Subject: [PATCH 2/3] fix(mcp): harden similar forget confirmation --- apps/mcp/e2e/helpers.ts | 70 +++++--- apps/mcp/e2e/list-memories.test.ts | 46 ++++-- apps/mcp/e2e/memory.test.ts | 171 ++++++++++++-------- apps/mcp/e2e/root-scope.test.ts | 27 +++- apps/mcp/package.json | 1 + apps/mcp/src/client.ts | 6 +- apps/mcp/src/forget.test.ts | 252 +++++++++++++++++++++++++++++ apps/mcp/src/server.ts | 62 ++++++- bun.lock | 1 + 9 files changed, 520 insertions(+), 116 deletions(-) create mode 100644 apps/mcp/src/forget.test.ts diff --git a/apps/mcp/e2e/helpers.ts b/apps/mcp/e2e/helpers.ts index a9e07df48..795ee740c 100644 --- a/apps/mcp/e2e/helpers.ts +++ b/apps/mcp/e2e/helpers.ts @@ -1,5 +1,9 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js" import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js" +import { + ElicitRequestSchema, + type ElicitResult, +} from "@modelcontextprotocol/sdk/types.js" export const MCP_URL = process.env.SUPERMEMORY_MCP_URL ?? "https://mcp.supermemory.ai/mcp" @@ -89,12 +93,45 @@ export function textOf(res: CallResult): string { .join("\n") } +export function documentIdOf(res: CallResult): string | undefined { + return textOf(res).match(/id: ([^)]+)/)?.[1] +} + export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +export async function deleteDocument( + documentId: string, + { tries = 60, delayMs = 5000 } = {}, +): Promise { + if (!API_KEY) throw new Error("SUPERMEMORY_API_KEY is required") + for (let i = 0; i < tries; i++) { + const response = await fetch( + `${API_URL}/v3/documents/${encodeURIComponent(documentId)}`, + { + method: "DELETE", + headers: { Authorization: `Bearer ${API_KEY}` }, + }, + ) + if (response.ok || response.status === 404) return + if (response.status !== 409) { + throw new Error( + `Document cleanup failed for ${documentId}: ${response.status}`, + ) + } + if (i < tries - 1) await sleep(delayMs) + } + throw new Error(`Document cleanup remained busy for ${documentId}`) +} + export type Session = { client: Client; close: () => Promise } export async function connect( - opts: { apiKey?: string; token?: string; containerTag?: string } = {}, + opts: { + apiKey?: string + token?: string + containerTag?: string + onElicitation?: () => ElicitResult | Promise + } = {}, ): Promise { const headers: Record = { Authorization: `Bearer ${opts.token ?? opts.apiKey ?? API_KEY}`, @@ -104,7 +141,15 @@ export async function connect( const transport = new StreamableHTTPClientTransport(new URL(MCP_URL), { requestInit: { headers }, }) - const client = new Client({ name: "sm-mcp-e2e", version: "0.0.1" }) + const client = new Client( + { name: "sm-mcp-e2e", version: "0.0.1" }, + opts.onElicitation + ? { capabilities: { elicitation: { form: {} } } } + : undefined, + ) + if (opts.onElicitation) { + client.setRequestHandler(ElicitRequestSchema, opts.onElicitation) + } await client.connect(transport) return { client, @@ -161,26 +206,7 @@ export async function forgetUntilForgotten( action: "forget", ...(containerTag ? { containerTag } : {}), }) - const previewText = textOf(res) - if (!res.isError && /Successfully forgot/i.test(previewText)) { - return previewText - } - - const confirmationToken = previewText.match(/confirmationToken: (\S+)/)?.[1] - if (confirmationToken) { - const confirmed = await callTool(client, "memory", { - content, - action: "forget", - confirmationToken, - ...(containerTag ? { containerTag } : {}), - }) - if ( - !confirmed.isError && - /Successfully forgot/i.test(textOf(confirmed)) - ) { - return textOf(confirmed) - } - } + if (!res.isError && /forgot/i.test(textOf(res))) return textOf(res) await sleep(delayMs) } return null diff --git a/apps/mcp/e2e/list-memories.test.ts b/apps/mcp/e2e/list-memories.test.ts index 4bec0751b..704fb947c 100644 --- a/apps/mcp/e2e/list-memories.test.ts +++ b/apps/mcp/e2e/list-memories.test.ts @@ -1,16 +1,17 @@ import { randomUUID } from "node:crypto" import { afterAll, beforeAll, describe, expect, it } from "vitest" import { + API_URL, API_KEY, callTool, connect, + deleteDocument, type Session, sleep, textOf, } from "./helpers" -// listMemories reads extracted memory entries, which appear only after the -// async ingestion pipeline finishes — poll like recallUntil does. +// listMemories can be briefly eventually-consistent after fixture creation. async function listUntil( s: Session, needle: string, @@ -28,30 +29,45 @@ async function listUntil( describe.skipIf(!API_KEY)("MCP — listMemories", () => { let s: Session - const created: string[] = [] + const createdDocumentIds: string[] = [] beforeAll(async () => { s = await connect() }) afterAll(async () => { - for (const content of created) { - await callTool(s.client, "memory", { - content, - action: "forget", - }).catch(() => {}) + try { + await Promise.all( + createdDocumentIds.map((documentId) => deleteDocument(documentId)), + ) + } finally { + await s?.close() } - await s?.close() - }) + }, 330_000) it("lists a saved memory without dumping document content", async () => { const marker = `lm-${randomUUID()}` const content = `e2e listMemories. token=${marker}. The list test fruit is rambutan.` - created.push(content) - - const save = await callTool(s.client, "memory", { content, action: "save" }) - expect(save.isError).toBeFalsy() + const created = await fetch(`${API_URL}/v4/memories`, { + method: "POST", + headers: { + Authorization: `Bearer ${API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + containerTag: "sm_project_default", + memories: [{ content }], + }), + }) + expect( + created.ok, + `memory fixture creation failed: ${created.status}`, + ).toBe(true) + const documentId = ((await created.json()) as { documentId?: string }) + .documentId + expect(documentId).toBeTruthy() + if (documentId) createdDocumentIds.push(documentId) - const listing = await listUntil(s, marker) + const listing = await listUntil(s, marker, { tries: 6, delayMs: 1000 }) expect( listing, `listMemories never returned marker ${marker}`, diff --git a/apps/mcp/e2e/memory.test.ts b/apps/mcp/e2e/memory.test.ts index cb5dbceeb..1b413e996 100644 --- a/apps/mcp/e2e/memory.test.ts +++ b/apps/mcp/e2e/memory.test.ts @@ -5,9 +5,9 @@ import { API_KEY, callTool, connect, - forgetUntilForgotten, + deleteDocument, + documentIdOf, recallUntil, - recallUntilAbsent, sleep, type Session, textOf, @@ -15,30 +15,30 @@ import { describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { let s: Session - const created: Array<{ content: string; containerTag?: string }> = [] + const createdDocumentIds: string[] = [] beforeAll(async () => { s = await connect() }) afterAll(async () => { - for (const { content, containerTag } of created) { - await forgetUntilForgotten(s.client, content, { - tries: 3, - delayMs: 1000, - containerTag, - }).catch(() => {}) + try { + await Promise.all( + createdDocumentIds.map((documentId) => deleteDocument(documentId)), + ) + } finally { + await s?.close() } - await s?.close() - }) + }, 330_000) it("save → recall round-trips the saved memory", async () => { const marker = `rt-${randomUUID()}` const content = `e2e round-trip. token=${marker}. The test fruit is dragonfruit.` - created.push({ content }) - const save = await callTool(s.client, "memory", { content, action: "save" }) expect(save.isError).toBeFalsy() expect(textOf(save)).toMatch(/Saved memory/i) + const documentId = documentIdOf(save) + expect(documentId).toBeTruthy() + if (documentId) createdDocumentIds.push(documentId) const found = await recallUntil(s.client, "test fruit dragonfruit", marker) expect(found, `recall never returned marker ${marker}`).not.toBeNull() @@ -68,62 +68,90 @@ describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { it("forget accepts and removes a saved memory", async () => { const marker = `fg-${randomUUID()}` const content = `e2e forget target. token=${marker}. Secret animal is axolotl.` - created.push({ content }) - - await callTool(s.client, "memory", { content, action: "save" }) - const found = await recallUntil(s.client, "secret animal axolotl", marker) - expect(found, "memory should exist before forget").not.toBeNull() - - // Polls forget until it confirms real removal ("forgot"), past the extraction window. - const forgotten = await forgetUntilForgotten(s.client, content) + const created = await fetch(`${API_URL}/v4/memories`, { + method: "POST", + headers: { + Authorization: `Bearer ${API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + containerTag: "sm_project_default", + memories: [{ content }], + }), + }) expect( - forgotten, - `forget never confirmed removal for ${marker} (memory entry never extracted in time)`, - ).not.toBeNull() - - const gone = await recallUntilAbsent( - s.client, - "secret animal axolotl", - marker, - ) - if (!gone) { - console.warn( - `[e2e] forget confirmed but ${marker} still indexed after ~60s (eventual deletion)`, - ) + created.ok, + `memory fixture creation failed: ${created.status}`, + ).toBe(true) + const fixture = (await created.json()) as { + documentId?: string + memories?: Array<{ id: string }> } + expect(fixture.documentId).toBeTruthy() + expect(fixture.memories?.[0]?.id).toBeTruthy() + if (fixture.documentId) createdDocumentIds.push(fixture.documentId) + + const forgotten = await callTool(s.client, "memory", { + content, + action: "forget", + }) + expect(forgotten.isError).toBeFalsy() + expect(textOf(forgotten)).toMatch( + /Successfully forgot memory \(exact match\)/i, + ) + + const verify = await fetch(`${API_URL}/v4/memories`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: fixture.memories?.[0]?.id, + containerTag: "sm_project_default", + }), + }) + expect(verify.status).toBe(404) }, 240_000) it("forget previews similar memories and requires signed confirmation", async () => { const containerTag = "sm_e2e_safe_forget" const marker = `safe-fg-${randomUUID()}` const content = `e2e safe forget target. token=${marker}. Vault color is marigold.` - const scoped = await connect({ containerTag }) + let confirmForget = true + const scoped = await connect({ + containerTag, + onElicitation: () => + confirmForget + ? { action: "accept", content: { confirm: true } } + : { action: "decline" }, + }) let documentId: string | undefined try { - const saved = await callTool(scoped.client, "memory", { - content, - action: "save", + const created = await fetch(`${API_URL}/v4/memories`, { + method: "POST", + headers: { + Authorization: `Bearer ${API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + containerTag, + memories: [{ content }], + }), }) - documentId = textOf(saved).match(/id: ([^)]+)/)?.[1] - expect(documentId, "save should return a source document ID").toBeTruthy() - - let extractedMemory = "" - for (let i = 0; i < 18; i++) { - const listed = await callTool(scoped.client, "listMemories") - extractedMemory = - textOf(listed) - .split("\n") - .find((line) => line.startsWith("- ") && line.includes(marker)) - ?.slice(2) ?? "" - if (extractedMemory) break - await sleep(5000) - } expect( - extractedMemory, - "saved content should produce an extracted memory", + created.ok, + `memory fixture creation failed: ${created.status}`, + ).toBe(true) + documentId = ((await created.json()) as { documentId?: string }) + .documentId + expect( + documentId, + "fixture should return a source document ID", ).toBeTruthy() - const nearMatch = `${extractedMemory} please` + if (documentId) createdDocumentIds.push(documentId) + const nearMatch = `${content} please` let previewText = "" for (let i = 0; i < 18; i++) { @@ -139,8 +167,11 @@ describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { expect(previewText).toContain( "No exact memory matched. No changes were made.", ) - const confirmationToken = previewText.match( - /confirmationToken: (\S+)/, + const matchingCandidate = previewText + .split(/(?=- confirmationToken: )/) + .find((candidate) => candidate.includes(marker)) + const confirmationToken = matchingCandidate?.match( + /^- confirmationToken: (\S+)/, )?.[1] expect( confirmationToken, @@ -170,12 +201,24 @@ describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { "Invalid or expired forget confirmation. No changes were made.", ) + confirmForget = false + const declined = await callTool(scoped.client, "memory", { + content: nearMatch, + action: "forget", + confirmationToken, + }) + expect(declined.isError).toBeFalsy() + expect(textOf(declined)).toContain( + "Forget cancelled. No changes were made.", + ) + const afterPreview = await callTool(scoped.client, "listMemories") expect( textOf(afterPreview), "preview must not delete the candidate", ).toContain(marker) + confirmForget = true const confirmed = await callTool(scoped.client, "memory", { content: nearMatch, action: "forget", @@ -195,13 +238,6 @@ describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { } expect(removed, "confirmed candidate should be removed").toBe(true) } finally { - if (documentId) { - const cleanup = await fetch(`${API_URL}/v3/documents/${documentId}`, { - method: "DELETE", - headers: { Authorization: `Bearer ${API_KEY}` }, - }) - expect(cleanup.ok || cleanup.status === 404).toBe(true) - } await scoped.close() } }, 300_000) @@ -212,13 +248,14 @@ describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { const tagB = "sm_e2e_scope_b" const marker = `sc-${randomUUID()}` const content = `e2e scoping. token=${marker}. Project color is teal.` - created.push({ content, containerTag: tagA }) - - await callTool(s.client, "memory", { + const save = await callTool(s.client, "memory", { content, action: "save", containerTag: tagA, }) + const documentId = documentIdOf(save) + expect(documentId).toBeTruthy() + if (documentId) createdDocumentIds.push(documentId) const inA = await recallUntil(s.client, "project color teal", marker, { containerTag: tagA, diff --git a/apps/mcp/e2e/root-scope.test.ts b/apps/mcp/e2e/root-scope.test.ts index 000490a94..b8e7e8f35 100644 --- a/apps/mcp/e2e/root-scope.test.ts +++ b/apps/mcp/e2e/root-scope.test.ts @@ -1,6 +1,14 @@ import { randomUUID } from "node:crypto" -import { describe, expect, it } from "vitest" -import { API_KEY, callTool, connect, recallUntil, textOf } from "./helpers" +import { afterAll, describe, expect, it } from "vitest" +import { + API_KEY, + callTool, + connect, + deleteDocument, + documentIdOf, + recallUntil, + textOf, +} from "./helpers" type ToolLike = { name: string @@ -15,6 +23,14 @@ const SCOPE_TAG = "sm_e2e_root" // x-sm-project locks the connection to one project: strips containerTag from schemas and scopes every op — distinct from the per-call arg. describe.skipIf(!API_KEY)("MCP — x-sm-project root scoping", () => { + const createdDocumentIds: string[] = [] + + afterAll(async () => { + await Promise.all( + createdDocumentIds.map((documentId) => deleteDocument(documentId)), + ) + }, 330_000) + it("strips containerTag from tool schemas when x-sm-project is set", async () => { const scoped = await connect({ containerTag: SCOPE_TAG }) const plain = await connect() @@ -45,6 +61,9 @@ describe.skipIf(!API_KEY)("MCP — x-sm-project root scoping", () => { }) expect(save.isError).toBeFalsy() expect(textOf(save)).toContain(SCOPE_TAG) + const documentId = documentIdOf(save) + expect(documentId).toBeTruthy() + if (documentId) createdDocumentIds.push(documentId) const found = await recallUntil( rooted.client, @@ -53,10 +72,6 @@ describe.skipIf(!API_KEY)("MCP — x-sm-project root scoping", () => { ) expect(found, "marker not found within its root scope").not.toBeNull() } finally { - await callTool(rooted.client, "memory", { - content, - action: "forget", - }).catch(() => {}) await rooted.close() } diff --git a/apps/mcp/package.json b/apps/mcp/package.json index 2def592a7..64af41fbc 100644 --- a/apps/mcp/package.json +++ b/apps/mcp/package.json @@ -12,6 +12,7 @@ "test:e2e": "vitest run" }, "dependencies": { + "@cfworker/json-schema": "^4.1.1", "@cloudflare/workers-oauth-provider": "^0.2.2", "@modelcontextprotocol/ext-apps": "^1.0.0", "@modelcontextprotocol/sdk": "^1.25.2", diff --git a/apps/mcp/src/client.ts b/apps/mcp/src/client.ts index aaaa1a104..74913c1ed 100644 --- a/apps/mcp/src/client.ts +++ b/apps/mcp/src/client.ts @@ -115,6 +115,10 @@ function limitByChars(text: string, maxChars = MAX_CHARS): string { return text.length > maxChars ? `${text.slice(0, maxChars)}...` : text } +function previewByChars(text: string): string { + return text.length > 200 ? `${text.slice(0, 100)}...${text.slice(-97)}` : text +} + // Type for SDK search result item interface SDKResult { id: string @@ -360,7 +364,7 @@ export class SupermemoryClient { content, candidate.id, ) - return `- confirmationToken: ${confirmation}\n similarity: ${candidate.similarity.toFixed(2)}\n content: ${JSON.stringify(limitByChars(getMemoryText(candidate) || candidate.content || "", 200))}` + return `- confirmationToken: ${confirmation}\n similarity: ${candidate.similarity.toFixed(2)}\n content: ${JSON.stringify(previewByChars(getMemoryText(candidate) || candidate.content || ""))}` }), ) diff --git a/apps/mcp/src/forget.test.ts b/apps/mcp/src/forget.test.ts new file mode 100644 index 000000000..c2cc12074 --- /dev/null +++ b/apps/mcp/src/forget.test.ts @@ -0,0 +1,252 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" + +const sdk = vi.hoisted(() => ({ + add: vi.fn(), + forget: vi.fn(), + search: vi.fn(), +})) + +const analytics = vi.hoisted(() => ({ + memoryAdded: vi.fn(), + memoryForgot: vi.fn(), + memorySearch: vi.fn(), +})) + +vi.mock("supermemory", () => ({ + default: class { + add = sdk.add + memories = { forget: sdk.forget } + search = { memories: sdk.search } + }, +})) + +vi.mock("agents/mcp", () => ({ McpAgent: class {} })) +vi.mock("../dist/mcp-app.html", () => ({ default: "" })) +vi.mock("./posthog", () => ({ + initPosthog: vi.fn(), + posthog: { + ...analytics, + }, +})) + +import { SupermemoryClient } from "./client" +import { SupermemoryMCP } from "./server" + +type MemoryArgs = { + content: string + action?: "save" | "forget" + confirmationToken?: string + containerTag?: string +} + +type ToolResult = { + content: Array<{ type: "text"; text: string }> + isError?: boolean +} + +const handleMemory = Reflect.get(SupermemoryMCP.prototype, "handleMemory") as ( + args: MemoryArgs, +) => Promise + +function statusError(status: number): Error & { status: number } { + return Object.assign(new Error(`status ${status}`), { status }) +} + +function tokenFrom(message: string): string { + const token = message.match(/confirmationToken: (\S+)/)?.[1] + if (!token) throw new Error("preview did not contain a confirmation token") + return token +} + +function serverHarness(capabilities?: { elicitation?: { form?: object } }) { + const client = { + createMemory: vi.fn(), + forgetMemory: vi.fn(), + } + const elicitInput = vi.fn() + const getClient = vi.fn(() => client) + return { + client, + elicitInput, + getClient, + server: { + props: { userId: "user-1" }, + cachedContainerTags: [], + server: { + server: { + getClientCapabilities: () => capabilities, + elicitInput, + }, + }, + getClient, + getClientInfo: vi.fn(async () => null), + getMcpSessionId: vi.fn(() => undefined), + refreshContainerTags: vi.fn(async () => undefined), + }, + } +} + +beforeEach(() => { + vi.clearAllMocks() + vi.useRealTimers() + analytics.memoryAdded.mockResolvedValue(undefined) + analytics.memoryForgot.mockResolvedValue(undefined) + analytics.memorySearch.mockResolvedValue(undefined) +}) + +describe("signed forget confirmations", () => { + it("previews a long candidate unambiguously and accepts its valid token", async () => { + const content = "near-match query" + const candidate = `${"same-prefix ".repeat(20)}unique-tail` + sdk.forget.mockRejectedValueOnce(statusError(404)) + sdk.search.mockResolvedValueOnce({ + results: [{ id: "memory-1", memory: candidate, similarity: 0.97 }], + total: 1, + timing: 1, + }) + const client = new SupermemoryClient("secret", "project-a") + + const preview = await client.forgetMemory(content) + expect(preview.success).toBe(false) + expect(preview.message).toContain("unique-tail") + const token = tokenFrom(preview.message) + + sdk.forget.mockResolvedValueOnce({ id: "memory-1" }) + const confirmed = await client.forgetMemory(content, token) + sdk.forget.mockRejectedValueOnce(statusError(404)) + const replayed = await client.forgetMemory(content, token) + + expect(confirmed.success).toBe(true) + expect(replayed.success).toBe(false) + expect(replayed.message).toContain("No memory exists with ID memory-1") + expect(sdk.forget).toHaveBeenLastCalledWith({ + id: "memory-1", + containerTag: "project-a", + }) + }) + + it("rejects altered, cross-container, wrong-key, tampered, and expired tokens without deleting", async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date("2026-07-23T00:00:00Z")) + sdk.forget.mockRejectedValueOnce(statusError(404)) + sdk.search.mockResolvedValueOnce({ + results: [{ id: "memory-1", memory: "candidate", similarity: 0.99 }], + total: 1, + timing: 1, + }) + const client = new SupermemoryClient("secret", "project-a") + const preview = await client.forgetMemory("query") + const token = tokenFrom(preview.message) + const tampered = `${token.slice(0, -1)}${token.endsWith("A") ? "B" : "A"}` + + const rejected = [ + await client.forgetMemory("altered", token), + await new SupermemoryClient("secret", "project-b").forgetMemory( + "query", + token, + ), + await new SupermemoryClient("other-secret", "project-a").forgetMemory( + "query", + token, + ), + await client.forgetMemory("query", "malformed"), + await client.forgetMemory("query", tampered), + ] + vi.advanceTimersByTime(5 * 60 * 1000 + 1) + rejected.push(await client.forgetMemory("query", token)) + + expect(rejected.every((result) => !result.success)).toBe(true) + expect(sdk.forget).toHaveBeenCalledTimes(1) + }) + + it("keeps exact forget as one call and propagates non-404 failures", async () => { + const client = new SupermemoryClient("secret", "project-a") + sdk.forget.mockResolvedValueOnce({ id: "memory-1" }) + + const exact = await client.forgetMemory("exact content") + + expect(exact.success).toBe(true) + expect(sdk.forget).toHaveBeenCalledTimes(1) + expect(sdk.search).not.toHaveBeenCalled() + + sdk.forget.mockRejectedValueOnce(statusError(500)) + await expect(client.forgetMemory("failure")).rejects.toThrow( + /service may be temporarily unavailable/i, + ) + expect(sdk.search).not.toHaveBeenCalled() + }) +}) + +describe("memory tool confirmation contract", () => { + it("rejects a confirmation token outside the forget action without creating data", async () => { + const harness = serverHarness() + + const result = await handleMemory.call(harness.server, { + content: "query", + confirmationToken: "token", + }) + + expect(result.isError).toBe(true) + expect(result.content[0]?.text).toContain("requires action 'forget'") + expect(harness.getClient).not.toHaveBeenCalled() + expect(harness.client.createMemory).not.toHaveBeenCalled() + }) + + it("marks failed token confirmations as tool errors but keeps previews non-errors", async () => { + const harness = serverHarness() + harness.client.forgetMemory.mockResolvedValue({ + success: false, + message: "Invalid or expired forget confirmation. No changes were made.", + containerTag: "project-a", + }) + + const invalid = await handleMemory.call(harness.server, { + content: "query", + action: "forget", + confirmationToken: "invalid", + }) + const preview = await handleMemory.call(harness.server, { + content: "query", + action: "forget", + }) + + expect(invalid.isError).toBe(true) + expect(preview.isError).toBeUndefined() + expect(analytics.memoryForgot).not.toHaveBeenCalled() + }) + + it("requires an elicited yes before deleting when the client supports it", async () => { + const harness = serverHarness({ elicitation: { form: {} } }) + harness.elicitInput.mockResolvedValueOnce({ action: "decline" }) + + const declined = await handleMemory.call(harness.server, { + content: "query", + action: "forget", + confirmationToken: "token", + }) + + expect(declined.isError).toBeUndefined() + expect(declined.content[0]?.text).toContain("cancelled") + expect(harness.client.forgetMemory).not.toHaveBeenCalled() + + harness.elicitInput.mockResolvedValueOnce({ + action: "accept", + content: { confirm: true }, + }) + harness.client.forgetMemory.mockResolvedValueOnce({ + success: true, + message: "Successfully forgot memory with ID: memory-1", + containerTag: "project-a", + }) + + const accepted = await handleMemory.call(harness.server, { + content: "query", + action: "forget", + confirmationToken: "token", + }) + + expect(accepted.isError).toBeUndefined() + expect(harness.client.forgetMemory).toHaveBeenCalledOnce() + expect(analytics.memoryForgot).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/mcp/src/server.ts b/apps/mcp/src/server.ts index 1c67c9f5b..f305e3f44 100644 --- a/apps/mcp/src/server.ts +++ b/apps/mcp/src/server.ts @@ -1,5 +1,6 @@ import { McpAgent } from "agents/mcp" import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js" +import { CfWorkerJsonSchemaValidator } from "@modelcontextprotocol/sdk/validation/cfworker" import { registerAppTool, registerAppResource, @@ -48,10 +49,15 @@ export class SupermemoryMCP extends McpAgent { private cachedContainerTags: string[] = [] private containerTagsLastFetchedAt: number | null = null - server = new McpServer({ - name: "supermemory", - version: "4.0.0", - }) + server = new McpServer( + { + name: "supermemory", + version: "4.0.0", + }, + { + jsonSchemaValidator: new CfWorkerJsonSchemaValidator(), + }, + ) async init() { const storedClientInfo = await this.ctx.storage.get<{ @@ -154,7 +160,7 @@ export class SupermemoryMCP extends McpAgent { "memory", { description: - "DO NOT USE ANY OTHER MEMORY TOOL ONLY USE THIS ONE. Save or forget information about the user. Use 'save' when user shares preferences, facts, or asks to remember something. Forget removes only exact content. If no exact match exists, it returns similar candidates without changing them; ask the user to confirm one, then retry with the same content and its confirmationToken within 5 minutes.", + "DO NOT USE ANY OTHER MEMORY TOOL ONLY USE THIS ONE. Save or forget information about the user. Use 'save' when user shares preferences, facts, or asks to remember something. Forget removes only exact content. If no exact match exists, it returns similar candidates without changing them; ask the user to confirm one, then retry with action 'forget', the same content, and its confirmationToken within 5 minutes. Clients that support MCP elicitation will also show an explicit confirmation prompt before deletion.", inputSchema: memorySchema, annotations: MEMORY_TOOL_ANNOTATIONS, }, @@ -607,12 +613,57 @@ export class SupermemoryMCP extends McpAgent { }) { const { content, action = "save", confirmationToken, containerTag } = args const effectiveContainerTag = containerTag || this.props?.containerTag + if (confirmationToken && action !== "forget") { + return { + content: [ + { + type: "text" as const, + text: "Error: confirmationToken requires action 'forget'. No changes were made.", + }, + ], + isError: true, + } + } try { const client = this.getClient(effectiveContainerTag) const clientInfo = await this.getClientInfo() if (action === "forget") { + if ( + confirmationToken && + this.server.server.getClientCapabilities()?.elicitation?.form + ) { + const confirmation = await this.server.server.elicitInput({ + mode: "form", + message: + "Delete the similar memory selected in the previous preview? This cannot be undone.", + requestedSchema: { + type: "object", + properties: { + confirm: { + type: "boolean", + title: "Yes, forget this memory", + }, + }, + required: ["confirm"], + }, + }) + if ( + confirmation.action !== "accept" || + confirmation.content?.confirm !== true + ) { + return { + content: [ + { + type: "text" as const, + text: "Forget cancelled. No changes were made.", + }, + ], + } + } + } + const result = await client.forgetMemory(content, confirmationToken) if (result.success) { @@ -636,6 +687,7 @@ export class SupermemoryMCP extends McpAgent { text: `${result.message} in container ${result.containerTag}`, }, ], + ...(confirmationToken && !result.success ? { isError: true } : {}), } } diff --git a/bun.lock b/bun.lock index d6b4edf3a..6bcd3f86f 100644 --- a/bun.lock +++ b/bun.lock @@ -87,6 +87,7 @@ "name": "supermemory-mcp", "version": "4.0.0", "dependencies": { + "@cfworker/json-schema": "^4.1.1", "@cloudflare/workers-oauth-provider": "^0.2.2", "@modelcontextprotocol/ext-apps": "^1.0.0", "@modelcontextprotocol/sdk": "^1.25.2", From 1dd719f541269094c06ba3a30858882170c00725 Mon Sep 17 00:00:00 2001 From: DisturbedCrow Date: Thu, 23 Jul 2026 15:53:27 +0530 Subject: [PATCH 3/3] style(mcp): annotate parsed responses --- apps/mcp/e2e/memory.test.ts | 4 ++-- apps/mcp/src/client.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/mcp/e2e/memory.test.ts b/apps/mcp/e2e/memory.test.ts index 1b413e996..0df473883 100644 --- a/apps/mcp/e2e/memory.test.ts +++ b/apps/mcp/e2e/memory.test.ts @@ -144,8 +144,8 @@ describe.skipIf(!API_KEY)("MCP — memory behaviors", () => { created.ok, `memory fixture creation failed: ${created.status}`, ).toBe(true) - documentId = ((await created.json()) as { documentId?: string }) - .documentId + const body: { documentId?: string } = await created.json() + documentId = body.documentId expect( documentId, "fixture should return a source document ID", diff --git a/apps/mcp/src/client.ts b/apps/mcp/src/client.ts index 74913c1ed..aa92cd001 100644 --- a/apps/mcp/src/client.ts +++ b/apps/mcp/src/client.ts @@ -252,9 +252,9 @@ export class SupermemoryClient { ) if (!valid) return null - const confirmation = JSON.parse( + const confirmation: Record = JSON.parse( textDecoder.decode(this.fromBase64Url(payload)), - ) as Record + ) if ( confirmation.version !== 1 || typeof confirmation.memoryId !== "string" ||