From 8e1a654662cb3f069e9c6a69ad606089a7a4274c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 5 Sep 2026 09:55:07 -0700 Subject: [PATCH 1/9] feat(files): improve editor recovery and editing controls --- .../src/handlers/file-doc-store.test.ts | 567 ++++++++++++- apps/realtime/src/handlers/file-doc-store.ts | 524 ++++++++++-- .../handlers/file-doc.join-readiness.test.ts | 11 +- .../handlers/file-doc.multireplica.test.ts | 13 +- apps/realtime/src/handlers/file-doc.test.ts | 306 ++++++- apps/realtime/src/handlers/file-doc.ts | 338 +++++++- apps/realtime/src/routes/http.test.ts | 63 ++ apps/realtime/src/routes/http.ts | 31 +- .../app/api/webhooks/outbox/process/route.ts | 2 + .../components/find-bar/find-bar.test.tsx | 44 + .../components/find-bar/find-bar.tsx | 232 ++++-- .../collaboration/file-doc-provider.test.ts | 770 +++++++++++++++++- .../collaboration/file-doc-provider.ts | 572 ++++++++++++- .../pending-update-journal.test.ts | 189 +++++ .../collaboration/pending-update-journal.ts | 210 +++++ .../use-file-doc-collaboration.ts | 13 +- .../editor-lifecycle.test.tsx | 61 +- .../find/find-extension.test.ts | 138 +++- .../find/find-extension.ts | 94 ++- .../find/use-markdown-find.ts | 41 +- .../image-inspector.test.tsx | 102 +++ .../rich-markdown-editor/image-inspector.tsx | 130 +++ .../image-resize.test.tsx | 145 ++++ .../rich-markdown-editor/image-schema.ts | 86 +- .../rich-markdown-editor/image.tsx | 86 +- .../menus/bubble-menu-chrome.ts | 2 +- .../menus/bubble-menu.tsx | 36 +- .../menus/editor-toolbar-integration.test.tsx | 20 + .../menus/link-editing.tsx | 2 +- .../menus/toolbar-button.test.tsx | 14 + .../menus/toolbar-button.tsx | 6 +- .../rich-markdown-editor.tsx | 132 ++- .../round-trip-safety.test.ts | 14 + .../rich-markdown-editor/round-trip-safety.ts | 24 +- .../rich-markdown-editor/round-trip.test.ts | 46 ++ apps/sim/lib/collab-doc/converter.test.ts | 1 + apps/sim/lib/core/outbox/service.test.ts | 17 +- apps/sim/lib/core/outbox/service.ts | 12 +- apps/sim/lib/realtime/notify.test.ts | 116 ++- apps/sim/lib/realtime/notify.ts | 114 ++- .../workspace-file-live-doc-outbox.test.ts | 197 +++++ .../workspace-file-live-doc-outbox.ts | 116 +++ .../workspace/workspace-file-manager.ts | 56 +- .../workspace-file-storage-accounting.test.ts | 79 +- .../realtime-protocol/src/file-doc.test.ts | 2 + packages/realtime-protocol/src/file-doc.ts | 59 +- 46 files changed, 5352 insertions(+), 481 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx create mode 100644 apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts create mode 100644 apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index f5159c10cbc..b36588e550f 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -13,6 +13,7 @@ import * as Y from 'yjs' interface Backing { streams: Map }[]> kv: Map + dedupe: Map seq: number /** Number of upcoming xAdd calls to fail with a transient error (to exercise publish retry). */ failXAdd: number @@ -26,6 +27,15 @@ interface Backing { idleReads: number /** `connect()` calls, so a test can prove a closed reader is re-opened rather than abandoned. */ connects: number + /** Largest stream range response requested, proving replay is paginated. */ + maxRangeCount: number + /** Optional deterministic compaction hook invoked before each range page is read. */ + onRange?: (call: number, key: string) => void + rangeCalls: number + /** Largest multiplexed XREAD request and COUNT observed. */ + maxReadStreams: number + maxReadCount: number + onSnapshot?: () => Promise } const state = vi.hoisted(() => ({ backing: null as Backing | null })) @@ -57,7 +67,24 @@ function makeClient(): any { b().streams.set(key, arr) return id }, - xRange: async (key: string) => (b().streams.get(key) ?? []).map((e) => ({ ...e })), + xRange: async (key: string, start: string, end: string, options?: { COUNT?: number }) => { + b().rangeCalls++ + b().onRange?.(b().rangeCalls, key) + const startId = start.startsWith('(') ? start.slice(1) : start + const entries = (b().streams.get(key) ?? []).filter( + (entry) => + (start === '-' || seqOf(entry.id) > seqOf(startId)) && + (end === '+' || seqOf(entry.id) <= seqOf(end)) + ) + const count = options?.COUNT ?? entries.length + b().maxRangeCount = Math.max(b().maxRangeCount, count) + return entries.slice(0, count).map((entry) => ({ ...entry })) + }, + xRevRange: async (key: string, _start: string, _end: string, options?: { COUNT?: number }) => + [...(b().streams.get(key) ?? [])] + .reverse() + .slice(0, options?.COUNT) + .map((entry) => ({ ...entry })), xLen: async (key: string) => (b().streams.get(key) ?? []).length, xTrim: async (key: string, _strategy: string, minid: string) => { const arr = b().streams.get(key) ?? [] @@ -66,8 +93,13 @@ function makeClient(): any { arr.filter((e) => seqOf(e.id) >= seqOf(minid)) ) }, - xRead: async (streams: { key: string; id: string }[]) => { + xRead: async ( + streams: { key: string; id: string }[], + options?: { BLOCK?: number; COUNT?: number } + ) => { b().reads++ + b().maxReadStreams = Math.max(b().maxReadStreams, streams.length) + b().maxReadCount = Math.max(b().maxReadCount, options?.COUNT ?? 0) if (b().readerClosed) { b().failedReadTimes.push(Date.now()) client.isOpen = false @@ -76,7 +108,9 @@ function makeClient(): any { const res: { name: string; messages: { id: string; message: Record }[] }[] = [] for (const { key, id } of streams) { - const after = (b().streams.get(key) ?? []).filter((e) => seqOf(e.id) > seqOf(id)) + const after = (b().streams.get(key) ?? []) + .filter((e) => seqOf(e.id) > seqOf(id)) + .slice(0, options?.COUNT) if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) } if (res.length) { @@ -92,24 +126,114 @@ function makeClient(): any { b().kv.set(key, val) return 'OK' }, - del: async (key: string) => { - b().kv.delete(key) - return 1 + get: async (key: string) => b().kv.get(key) ?? null, + del: async (keys: string | string[]) => { + const targets = Array.isArray(keys) ? keys : [keys] + for (const key of targets) { + b().kv.delete(key) + b().streams.delete(key) + b().dedupe.delete(key) + } + return targets.length }, eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { const [key] = opts.keys + if (script.includes('return ARGV[1]')) { + const generation = b().kv.get(opts.keys[1]) + if (generation !== undefined) return generation + if (!b().streams.get(key)?.length) return false + b().kv.set(opts.keys[1], opts.arguments[0]) + return opts.arguments[0] + } + if (script.includes("redis.call('del', KEYS[1], KEYS[4], KEYS[5])")) { + const [, generationKey, versionKey, dedupeKey, agentKey] = opts.keys + const [version, , marker] = opts.arguments + const current = b().kv.get(versionKey) + if (current && Number(current) > Number(version)) return 0 + if (current === version && b().kv.get(generationKey) === marker) return 0 + b().kv.set(generationKey, marker) + b().kv.set(versionKey, version) + b().streams.delete(key) + b().dedupe.delete(dedupeKey) + b().kv.delete(agentKey) + return 1 + } + if (script.includes('zscore')) { + const [, dedupeKey, generationKey] = opts.keys + const [member, field, value, capacityText, , expectedGeneration] = opts.arguments + const generation = b().kv.get(generationKey) + if ((generation ?? '') !== expectedGeneration) return -1 + const members = b().dedupe.get(dedupeKey) ?? [] + if (members.includes(member)) return 0 + const id = `${++b().seq}-0` + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { [field]: value } }) + b().streams.set(key, arr) + members.push(member) + const capacity = Number(capacityText) + if (members.length > capacity) members.splice(0, members.length - capacity) + b().dedupe.set(dedupeKey, members) + return 1 + } // Atomic seed-if-empty (SEED_IF_EMPTY_SCRIPT): append the entry iff the stream is empty, in one // synchronous step — mirroring Redis's atomic Lua execution, so two concurrent evals can never both // append (the second sees a non-empty stream). if (script.includes('xlen')) { - const [field, value] = opts.arguments + const [, generationKey, versionKey] = opts.keys + const [field, value, generation, , generationField, version] = opts.arguments + if (Number(b().kv.get(versionKey) ?? 0) > Number(version)) return 0 const arr = b().streams.get(key) ?? [] if (arr.length > 0) return 0 + b().kv.set(generationKey, generation) + if (version !== '0') b().kv.set(versionKey, version) const id = `${++b().seq}-0` - arr.push({ id, message: { [field]: value } }) + arr.push({ id, message: { [field]: value, [generationField]: generation } }) b().streams.set(key, arr) return 1 } + if (script.includes('ARGV[5], ARGV[4]')) { + const [, generationKey] = opts.keys + const [field, value, marker, expectedGeneration, generationField] = opts.arguments + const generation = b().kv.get(generationKey) + if ((generation ?? '') !== expectedGeneration) return false + const id = `${++b().seq}-0` + const arr = b().streams.get(key) ?? [] + arr.push({ + id, + message: { + [field]: value, + [marker]: '1', + [generationField]: expectedGeneration, + }, + }) + b().streams.set(key, arr) + await b().onSnapshot?.() + return id + } + if (script.includes("ARGV[3] ~= ''")) { + const [, generationKey] = opts.keys + const generation = b().kv.get(generationKey) + const expectedGeneration = opts.arguments[3] + if ((generation ?? '') !== expectedGeneration) return false + if (b().failXAdd > 0) { + b().failXAdd-- + throw new Error('transient xAdd failure') + } + const [field, value, marker] = opts.arguments + const id = `${++b().seq}-0` + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { [field]: value, ...(marker ? { [marker]: '1' } : {}) } }) + b().streams.set(key, arr) + return id + } + if (script.includes('tonumber(c)')) { + const [value, , expectedGeneration] = opts.arguments + const generation = b().kv.get(opts.keys[1]) + if ((generation ?? '') !== expectedGeneration) return 0 + const current = b().kv.get(key) + if (current === undefined || Number(current) < Number(value)) b().kv.set(key, value) + return 1 + } // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. const [token] = opts.arguments if (b().kv.get(key) === token) { @@ -130,6 +254,28 @@ import { FileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN } from '@/handlers/file- const REDIS_URL = 'redis://fake' const NAME = 'workspace-file-doc:file-1' +interface StoreTestAccess { + localInvalidations: Map + rooms: Map< + string, + { + doc: Y.Doc + lastId: string + publishes: number + uncompactedDeltaBytes: number + compacting: boolean + seededObserved: boolean + realEdited: boolean + } + > + maybeCompact(name: string): Promise + appendUpdate(name: string, update: Uint8Array): Promise +} + +function storeInternals(store: FileDocStore): StoreTestAccess { + return store as unknown as StoreTestAccess +} + function docWithText(text: string): Y.Doc { const doc = new Y.Doc() doc.getText('body').insert(0, text) @@ -157,6 +303,7 @@ describe('FileDocStore', () => { state.backing = { streams: new Map(), kv: new Map(), + dedupe: new Map(), seq: 0, failXAdd: 0, readerClosed: false, @@ -164,6 +311,10 @@ describe('FileDocStore', () => { failedReadTimes: [], idleReads: 0, connects: 0, + maxRangeCount: 0, + rangeCalls: 0, + maxReadStreams: 0, + maxReadCount: 0, } stores = [] }) @@ -272,6 +423,28 @@ describe('FileDocStore', () => { expect(await b.shouldSeed(NAME)).toBeNull() }) + it('fences stale publishers after invalidation and lets the next authoritative seed start fresh', async () => { + const store = await newStore() + const original = updateFor('old generation') + await store.publishAndWait(NAME, original) + await store.invalidateDocument(NAME, 10) + + await expect(store.getStreamState(NAME)).resolves.toBeNull() + await expect(store.publishAndWait(NAME, updateFor('stale write'))).rejects.toThrow( + 'replaced by a newer durable version' + ) + await expect( + store.publishClientUpdateAndWait(NAME, 'stale-update', updateFor('stale acknowledged write')) + ).rejects.toThrow('replaced by a newer durable version') + + const fresh = updateFor('fresh generation') + await expect(store.seedIfEmpty(NAME, fresh, 11)).resolves.toBe(true) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('fresh generation') + recovered.destroy() + }) + it('getStreamState reconstructs the shared document from the stream', async () => { const a = await newStore() a.publish(NAME, updateFor('shared content')) @@ -286,6 +459,178 @@ describe('FileDocStore', () => { doc.destroy() }) + it('lets a headless replica append against the generation of its shared base', async () => { + const seeded = await newStore() + await seeded.seedIfEmpty(NAME, updateFor('shared'), 20) + const headless = await newStore() + const generation = await headless.getDocumentGeneration(NAME) + const doc = new Y.Doc() + Y.applyUpdate(doc, (await headless.getStreamState(NAME, generation))!) + const before = Y.encodeStateVector(doc) + doc.getText('body').insert(6, ' edit') + await headless.publishAndWait(NAME, Y.encodeStateAsUpdate(doc, before), generation) + const replay = new Y.Doc() + Y.applyUpdate(replay, (await seeded.getStreamState(NAME))!) + expect(replay.getText('body').toString()).toBe('shared edit') + doc.destroy() + replay.destroy() + }) + + it('keeps a newer seeded generation when an older invalidation arrives', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('newest'), 20) + const generation = await store.getDocumentGeneration(NAME) + await expect(store.invalidateDocument(NAME, 10)).resolves.toBe(false) + expect(await store.getDocumentGeneration(NAME)).toBe(generation) + expect(await store.getSyncedVersion(NAME)).toBe(20) + await expect(store.getStreamState(NAME)).resolves.not.toBeNull() + }) + + it('rejects old seeds and version callbacks after an invalidation', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('old'), 10) + const generation = await store.getDocumentGeneration(NAME) + await store.invalidateDocument(NAME, 20) + await expect(store.seedIfEmpty(NAME, updateFor('late stale seed'), 10)).resolves.toBe(false) + await store.setSyncedVersion(NAME, 30, generation) + expect(await store.getSyncedVersion(NAME)).toBe(20) + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + + it('does not resurrect a tracked stream with a dependency-only update after Redis loses it', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('base'), 10) + const generation = await store.getDocumentGeneration(NAME) + state.backing!.streams.delete(`filedoc:stream:${NAME}`) + state.backing!.kv.delete(`filedoc:generation:${NAME}`) + await expect(store.publishAndWait(NAME, updateFor('stale'), generation)).rejects.toThrow( + 'replaced' + ) + await expect( + store.publishClientUpdateAndWait(NAME, 'lost-stream-update', updateFor('stale'), generation) + ).rejects.toThrow('replaced') + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + + it('adopts the identity of a pre-upgrade stream before acknowledging its edits', async () => { + const store = await newStore() + const seed = new Y.Doc() + seed.getMap('config').set('initialContentLoaded', true) + seed.getMap('config').set('docId', 'legacy-document') + seed.getText('body').insert(0, 'legacy') + await store.publishAndWait(NAME, Y.encodeStateAsUpdate(seed)) + const attached = new Y.Doc() + await store.attachRoom(NAME, attached) + expect(await store.getDocumentGeneration(NAME)).toBe('legacy-document') + const before = Y.encodeStateVector(seed) + seed.getText('body').insert(6, ' edit') + await expect( + store.publishClientUpdateAndWait( + NAME, + 'legacy-edit', + Y.encodeStateAsUpdate(seed, before), + 'legacy-document' + ) + ).resolves.toBeUndefined() + store.detachRoom(NAME) + seed.destroy() + attached.destroy() + }) + + it('rejects a shared replay if the document generation changes between pages', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('old generation'), 10) + state.backing!.onRange = () => { + state.backing!.kv.set(`filedoc:generation:${NAME}`, 'new generation') + } + await expect(store.getStreamState(NAME)).rejects.toThrow('replaced') + }) + + it('replays stream history in bounded pages', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 40 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 40 + const store = await newStore() + + await expect(store.getStreamState(NAME)).resolves.not.toBeNull() + + expect(state.backing!.maxRangeCount).toBe(4) + }) + + it('fails safely when an uncompacted stream exceeds the replay entry budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 2_001 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 2_001 + const store = await newStore() + + await expect(store.getStreamState(NAME)).rejects.toThrow('replay exceeded its safety limit') + }) + + it('never exposes a partially replayed document when room attachment exceeds its budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 2_001 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 2_001 + const store = await newStore() + const doc = new Y.Doc() + + await expect(store.attachRoom(NAME, doc)).rejects.toThrow('replay exceeded its safety limit') + + expect(doc.getText('body').toString()).toBe('') + expect(storeInternals(store).rooms.has(NAME)).toBe(false) + doc.destroy() + }) + + it('recovers from compaction that trims unread pages during replay', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 8 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 8 + state.backing!.onRange = (call, key) => { + if (call !== 2 || key !== streamKey) return + state.backing!.streams.set(streamKey, [ + { + id: '9-0', + message: { u: Buffer.from(updateFor('compacted')).toString('base64'), s: '1' }, + }, + ]) + state.backing!.seq = 9 + state.backing!.onRange = undefined + } + const store = await newStore() + + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('compacted') + recovered.destroy() + }) + it('attachRoom catches a fresh task up to the current shared state', async () => { const a = await newStore() a.publish(NAME, updateFor('already here')) @@ -338,14 +683,16 @@ describe('FileDocStore', () => { const a = await newStore() // This task has integrated only up to entry 400 (all no-ops) — its local doc is empty and lags the // two peer entries. Inject that lagging room directly (a real edit was integrated → realEdited). - ;(a as any).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, seededObserved: true, realEdited: true, }) - await (a as any).maybeCompact(NAME) + await storeInternals(a).maybeCompact(NAME) // A fresh catch-up must still reconstruct the peer content — compaction must not have trimmed 401/402. const doc = new Y.Doc() @@ -391,11 +738,11 @@ describe('FileDocStore', () => { const a = await newStore() const doc = new Y.Doc() await a.attachRoom(NAME, doc) - const room = (a as any).rooms.get(NAME) + const room = storeInternals(a).rooms.get(NAME)! expect(room.realEdited).toBe(false) // Kick off a real (non-agent) append but do NOT await it: realEdited must already be true before the // xAdd/expire awaits resolve, so any compaction racing on the awaits sees the real edit. - const pending = (a as any).appendUpdate(NAME, updateFor('real user edit')) + const pending = storeInternals(a).appendUpdate(NAME, updateFor('real user edit')) expect(room.realEdited).toBe(true) await pending doc.destroy() @@ -414,14 +761,16 @@ describe('FileDocStore', () => { state.backing!.seq = 400 const a = await newStore() - ;(a as any).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: agentDoc, lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, seededObserved: true, realEdited: false, }) - await (a as any).maybeCompact(NAME) + await storeInternals(a).maybeCompact(NAME) // The snapshot must carry the AGENT marker, NOT the snapshot marker, so a peer catch-up applies it as // REDIS_AGENT_ORIGIN and never marks the doc edited — the no-persist guarantee survives compaction. @@ -452,6 +801,186 @@ describe('FileDocStore', () => { ) }) + it('deduplicates acknowledged client retries by update id', async () => { + const store = await newStore() + const update = updateFor('retry-safe') + + await store.publishClientUpdateAndWait(NAME, 'update-1', update) + await store.publishClientUpdateAndWait(NAME, 'update-1', update) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(1) + }) + + it('does not drop different payloads that reuse an acknowledged update id', async () => { + const store = await newStore() + + await store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('first')) + await store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('second')) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + }) + + it('uses unambiguous acknowledged-update deduplication keys', async () => { + const store = await newStore() + + await store.publishClientUpdateAndWait(NAME, 'a', new Uint8Array([0, 98])) + await store.publishClientUpdateAndWait(NAME, 'a\0', new Uint8Array([98])) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + }) + + it('bounds acknowledged-update deduplication independently of stream traffic', async () => { + const store = await newStore() + const update = updateFor('bounded') + + for (let index = 0; index <= 16_384; index += 1) { + await store.publishClientUpdateAndWait(NAME, `update-${index}`, update) + } + + expect(state.backing!.dedupe.get(`filedoc:updates:${NAME}`)).toHaveLength(16_384) + }) + + it('limits every multiplexed read to four streams and one entry per stream', async () => { + const store = await newStore() + const docs = Array.from({ length: 9 }, () => new Y.Doc()) + await Promise.all(docs.map((doc, index) => store.attachRoom(`${NAME}-${index}`, doc))) + state.backing!.maxReadStreams = 0 + state.backing!.maxReadCount = 0 + const readsBefore = state.backing!.reads + + await vi.waitFor(() => expect(state.backing!.reads).toBeGreaterThan(readsBefore)) + + expect(state.backing!.maxReadStreams).toBeLessThanOrEqual(4) + expect(state.backing!.maxReadCount).toBe(1) + docs.forEach((doc, index) => { + store.detachRoom(`${NAME}-${index}`) + doc.destroy() + }) + }) + + it('compacts on retained bytes before the entry-count threshold can exhaust replay', async () => { + const streamKey = `filedoc:stream:${NAME}` + const snapshot = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') + state.backing!.streams.set(streamKey, [{ id: '1-0', message: { u: snapshot } }]) + state.backing!.seq = 1 + const store = await newStore() + storeInternals(store).rooms.set(NAME, { + doc: new Y.Doc(), + lastId: '1-0', + publishes: 0, + uncompactedDeltaBytes: 12 * 1024 * 1024, + compacting: false, + seededObserved: true, + realEdited: true, + }) + + await storeInternals(store).maybeCompact(NAME) + + const stream = state.backing!.streams.get(streamKey)! + expect(stream).toHaveLength(2) + expect(stream.at(-1)?.message.s).toBe('1') + }) + + it('does not compact a large snapshot again while continuing to accept small edits', async () => { + const store = await newStore() + const source = docWithText('x'.repeat(9 * 1024 * 1024)) + source.getMap('config').set('initialContentLoaded', true) + source.getMap('config').set('docId', 'large-document') + const streamKey = `filedoc:stream:${NAME}` + state.backing!.kv.set(`filedoc:generation:${NAME}`, 'large-document') + state.backing!.streams.set(streamKey, [ + { + id: '1-0', + message: { + u: Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64'), + s: '1', + g: 'large-document', + }, + }, + ]) + state.backing!.seq = 1 + const loaded = new Y.Doc() + await store.attachRoom(NAME, loaded) + expect(storeInternals(store).rooms.get(NAME)!.uncompactedDeltaBytes).toBe(0) + + let deltaBytes = 0 + for (let index = 0; index < 30; index++) { + const before = Y.encodeStateVector(source) + source.getText('body').insert(source.getText('body').length, 'y') + const update = Y.encodeStateAsUpdate(source, before) + deltaBytes += Buffer.from(update).toString('base64').length + await store.publishClientUpdateAndWait(NAME, `small-${index}`, update, 'large-document') + await store.catchUp(NAME) + } + expect(state.backing!.streams.get(streamKey)?.filter((entry) => entry.message.s)).toHaveLength( + 1 + ) + expect(storeInternals(store).rooms.get(NAME)!.uncompactedDeltaBytes).toBe(deltaBytes) + expect(loaded.getText('body').length).toBe(9 * 1024 * 1024 + 30) + store.detachRoom(NAME) + source.destroy() + loaded.destroy() + }) + + it('preserves exactly the delta bytes observed after a compaction barrier', async () => { + const store = await newStore() + const doc = new Y.Doc() + await store.attachRoom(NAME, doc) + const room = storeInternals(store).rooms.get(NAME)! + room.uncompactedDeltaBytes = 12 * 1024 * 1024 + const lateUpdate = updateFor('concurrent edit') + state.backing!.onSnapshot = async () => { + await store.publishAndWait(NAME, lateUpdate) + await store.catchUp(NAME) + } + await storeInternals(store).maybeCompact(NAME) + expect(room.uncompactedDeltaBytes).toBe(Buffer.from(lateUpdate).toString('base64').length) + expect(doc.getText('body').toString()).toBe('concurrent edit') + store.detachRoom(NAME) + doc.destroy() + }) + + it('bounds single-replica invalidation markers to the lifetime of active rooms', async () => { + const store = new FileDocStore(undefined) + for (let index = 0; index < 100; index++) { + await store.invalidateDocument(`closed-${index}`, 10) + } + expect(storeInternals(store).localInvalidations.size).toBe(0) + const staleDoc = new Y.Doc() + await store.attachRoom(NAME, staleDoc) + await store.invalidateDocument(NAME, 20) + await expect(store.seedIfEmpty(NAME, updateFor('stale fetched seed'), 10)).resolves.toBe(false) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(false) + expect(storeInternals(store).localInvalidations.size).toBe(1) + + store.detachRoom(NAME) + expect(storeInternals(store).localInvalidations.size).toBe(0) + const freshDoc = new Y.Doc() + await store.attachRoom(NAME, freshDoc) + await expect(store.seedIfEmpty(NAME, updateFor('fresh authoritative seed'), 30)).resolves.toBe( + true + ) + await store.invalidateDocument(NAME, 40) + await store.shutdown() + expect(storeInternals(store).localInvalidations.size).toBe(0) + staleDoc.destroy() + freshDoc.destroy() + }) + + it('fails closed when a Redis-backed store has not initialized', async () => { + const store = new FileDocStore(REDIS_URL) + const doc = new Y.Doc() + + await expect(store.attachRoom(NAME, doc)).rejects.toThrow('not initialized') + await expect( + store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('x')) + ).rejects.toThrow('not initialized') + await expect(store.seedIfEmpty(NAME, updateFor('seed'))).rejects.toThrow('not initialized') + await expect(store.getStreamState(NAME)).rejects.toThrow('not initialized') + expect(await store.acquireMergeSlot(NAME, 1_000)).toBeNull() + doc.destroy() + }) + it('streamHasContent fences a seed apply against an already-seeded stream', async () => { const a = await newStore() expect(await a.streamHasContent(NAME)).toBe(false) @@ -580,21 +1109,25 @@ describe('FileDocStore', () => { const b = await newStore() const docA = new Y.Doc() Y.applyUpdate(docA, peerUpdates[0]) // A integrated up to 401 - ;(a as any).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: docA, lastId: '401-0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, seededObserved: true, realEdited: true, }) - ;(b as any).rooms.set(NAME, { + storeInternals(b).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, seededObserved: true, realEdited: true, }) - await Promise.all([(a as any).maybeCompact(NAME), (b as any).maybeCompact(NAME)]) + await Promise.all([storeInternals(a).maybeCompact(NAME), storeInternals(b).maybeCompact(NAME)]) const doc = new Y.Doc() Y.applyUpdate(doc, (await a.getStreamState(NAME))!) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 537f7f4db12..8e66bd465d7 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -34,8 +34,10 @@ * * @module */ + +import { createHash } from 'node:crypto' import { createLogger } from '@sim/logger' -import { FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_LIMITS, FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' @@ -61,7 +63,35 @@ const RELEASE_LOCK_SCRIPT = * Returns 1 if THIS call wrote the seed, 0 if the stream already had content. */ const SEED_IF_EMPTY_SCRIPT = - "if redis.call('xlen', KEYS[1]) == 0 then redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]); return 1 else return 0 end" + "local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[6]) then return 0 end; if redis.call('xlen', KEYS[1]) == 0 then redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[4]); if ARGV[6] ~= '0' then redis.call('set', KEYS[3], ARGV[6], 'EX', ARGV[4]) end; redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[5], ARGV[3]); redis.call('expire', KEYS[1], ARGV[4]); return 1 else return 0 end" + +/** Orders a durable replacement with seeds and merges, and fences publishers in the same transaction. */ +const INVALIDATE_DOCUMENT_SCRIPT = + "local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[1]) then return 0 end; if version == ARGV[1] and redis.call('get', KEYS[2]) == ARGV[3] then return 0 end; redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[2]); redis.call('set', KEYS[3], ARGV[1], 'EX', ARGV[2]); redis.call('del', KEYS[1], KEYS[4], KEYS[5]); return 1" + +/** Upgrades an existing pre-negotiation stream without ever resurrecting a missing stream. */ +const ADOPT_GENERATION_SCRIPT = + "local generation = redis.call('get', KEYS[2]); if generation then return generation end; if redis.call('xlen', KEYS[1]) == 0 then return false end; redis.call('set', KEYS[2], ARGV[1], 'EX', ARGV[2]); return ARGV[1]" + +/** + * Append an ordinary update only while the live-document generation is valid. A durable replacement + * that cannot be represented by the rich editor first sets the invalidation tombstone and then removes + * the stream; keeping the guard and XADD in one script prevents an old room from recreating that stream. + * Returns the new stream id, or `false` while invalidated. + */ +const APPEND_UPDATE_SCRIPT = + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] then return false end; if ARGV[3] ~= '' then return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1') else return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]) end" + +/** A compacted snapshot replaces the seed entry, so it must carry that seed's generation forward. */ +const APPEND_SNAPSHOT_SCRIPT = + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] then return false end; return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1', ARGV[5], ARGV[4])" + +/** + * Atomically deduplicate and append an acknowledged client update. Socket acknowledgements can be + * lost, so a retry with the same id must not inflate the stream or its compaction counters. + */ +const APPEND_CLIENT_UPDATE_SCRIPT = + "local generation = redis.call('get', KEYS[3]) or ''; if generation ~= ARGV[6] then return -1 end; if redis.call('zscore', KEYS[2], ARGV[1]) then redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 0 end; local id = redis.call('xadd', KEYS[1], '*', ARGV[2], ARGV[3]); local score = string.match(id, '^(%d+)'); redis.call('zadd', KEYS[2], score, ARGV[1]); local excess = redis.call('zcard', KEYS[2]) - tonumber(ARGV[4]); if excess > 0 then redis.call('zpopmin', KEYS[2], excess) end; if redis.call('ttl', KEYS[2]) < 0 then redis.call('expire', KEYS[2], ARGV[5]) end; redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 1" /** * Monotonic set of the synced-version token: overwrite ONLY when the new value is greater than the @@ -73,7 +103,7 @@ const SEED_IF_EMPTY_SCRIPT = * comfortably within a Lua double, so the numeric compare is exact. */ const SET_VERSION_IF_NEWER_SCRIPT = - "local c = redis.call('get', KEYS[1]); if c == false or tonumber(c) < tonumber(ARGV[1]) then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) else redis.call('expire', KEYS[1], ARGV[2]) end; return 1" + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[3] then return 0 end; local c = redis.call('get', KEYS[1]); if c == false or tonumber(c) < tonumber(ARGV[1]) then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) else redis.call('expire', KEYS[1], ARGV[2]) end; return 1" /** * The transaction origin the store stamps on updates it applies from the stream. The relay's @@ -103,6 +133,8 @@ export const REDIS_SNAPSHOT_ORIGIN = Symbol('file-doc-redis-snapshot') export const REDIS_AGENT_ORIGIN = Symbol('file-doc-redis-agent') const STREAM_PREFIX = 'filedoc:stream:' +const CLIENT_UPDATE_PREFIX = 'filedoc:updates:' +const GENERATION_PREFIX = 'filedoc:generation:' /** Cluster-wide "durable version the live doc is synced to" (the persist If-Match token). */ const SYNC_VERSION_PREFIX = 'filedoc:syncver:' const SEED_LOCK_PREFIX = 'filedoc:seedlock:' @@ -123,6 +155,9 @@ const SNAPSHOT_FIELD = 's' /** Marks a stream entry as an AGENT-STREAMED preview frame, so the tailer applies it with * {@link REDIS_AGENT_ORIGIN} (never marks the doc edited). Present only on agent-frame entries. */ const AGENT_FIELD = 'a' +/** Identifies a seed's document generation, allowing old rooms to reject every later update. */ +const GENERATION_FIELD = 'g' +const INVALIDATED_GENERATION = '__invalidated__' /** Sentinel token a DISABLED store returns from a lock acquire, so single-replica callers proceed * without special-casing; {@link FileDocStore.releaseLock} treats it as a no-op. Not a real UUID, so it @@ -136,8 +171,18 @@ const READ_BLOCK_MS = 1_000 /** Idle poll cadence when NO room is open on this task, so a freshly-attached room is picked up fast * without busy-spinning an empty task. */ const IDLE_POLL_MS = 250 -/** Max entries drained per stream per read. */ -const READ_COUNT = 200 +/** Max entries drained per stream per read, bounding one Redis response even for maximum-size edits. */ +const READ_COUNT = 1 +/** Maximum streams passed to one XREAD, bounding response memory independently of open-room count. */ +const READ_STREAM_BATCH_SIZE = 4 +/** Replay streams incrementally instead of materializing their complete history in one response. */ +const REPLAY_PAGE_COUNT = 4 +/** Compaction normally holds a stream near 400 entries; fail safely if that invariant is badly broken. */ +const REPLAY_MAX_ENTRIES = 2_000 +/** Base64 bytes accepted during one replay, including a full snapshot plus a bounded edit backlog. */ +const REPLAY_MAX_ENCODED_BYTES = FILE_DOC_LIMITS.updateBytes * 6 +/** Compact before a handful of individually valid large updates can exhaust the replay byte budget. */ +const COMPACT_ENCODED_BYTES = FILE_DOC_LIMITS.updateBytes * 2 /** Compact a stream once it exceeds this many entries (snapshot + trim). */ const COMPACT_THRESHOLD = 400 /** Check whether compaction is due only every Nth local publish, to avoid an XLEN per keystroke. */ @@ -166,8 +211,36 @@ const RECONNECT_MAX_DELAY_MS = 3_000 const READER_RETRY_MAX_MS = 10_000 /** After the first failure of a streak, log one reader failure in this many. */ const READER_ERROR_LOG_EVERY = 20 +const CLIENT_UPDATE_DEDUPE_CAPACITY = 16_384 const streamKey = (name: string) => `${STREAM_PREFIX}${name}` +const generationKey = (name: string) => `${GENERATION_PREFIX}${name}` + +export class FileDocInvalidatedError extends Error { + constructor() { + super('The live file document was replaced by a newer durable version') + this.name = 'FileDocInvalidatedError' + } +} + +function assertUpdateWithinLimit(update: Uint8Array): void { + if (update.byteLength === 0 || update.byteLength > FILE_DOC_LIMITS.updateBytes) { + throw new Error(`File document update is outside the ${FILE_DOC_LIMITS.updateBytes}-byte limit`) + } +} + +function generationOfSeed(update: Uint8Array): string { + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, update) + const docId = doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + return typeof docId === 'string' + ? docId + : `seed:${createHash('sha256').update(update).digest('hex')}` + } finally { + doc.destroy() + } +} /** * Decode one stream entry's base64 Yjs update and apply it to `doc`. A malformed entry is logged and @@ -218,6 +291,13 @@ interface StoreRoom { lastId: string /** Local publish count, to pace compaction checks. */ publishes: number + /** Non-snapshot bytes observed since this replica last compacted; peer compactions may reduce them. */ + uncompactedDeltaBytes: number + compacting: boolean + /** Document generation read from the seed entry; every later append is fenced against it. */ + generation: string | null + /** A newer seed was observed; this old room must ignore all entries until the relay replaces it. */ + generationInvalidated: boolean /** Set once the doc has been observed seeded, so the seed transition itself is never mistaken for an * edit (mirrors the relay's `seededObserved`). */ seededObserved: boolean @@ -239,6 +319,7 @@ export class FileDocStore { /** Dedicated connection for blocking XREAD (a blocking command monopolizes its connection). */ private read: RedisClientType | null = null private readonly rooms = new Map() + private readonly localInvalidations = new Map() private running = false private heartbeat: ReturnType | null = null @@ -262,7 +343,10 @@ export class FileDocStore { * connection it can rebuild is always worth rebuilding. */ reconnectStrategy: (retries: number) => - backoffWithJitter(retries + 1, null, { baseMs: 100, maxMs: RECONNECT_MAX_DELAY_MS }), + backoffWithJitter(retries + 1, null, { + baseMs: 100, + maxMs: RECONNECT_MAX_DELAY_MS, + }), }, } this.write = createClient(options) @@ -284,26 +368,38 @@ export class FileDocStore { await Promise.all([this.write?.quit().catch(() => {}), this.read?.quit().catch(() => {})]) this.write = null this.read = null + this.rooms.clear() + this.localInvalidations.clear() } /** * Register a locally-opened room and load the shared state into its doc ({@link catchUp}). A * brand-new file has an empty stream and loads nothing (it is seeded shortly after, via - * {@link shouldSeed}). No-op when disabled. + * {@link shouldSeed}). Single-replica rooms are tracked only for invalidation lifecycle. */ async attachRoom(name: string, doc: Y.Doc): Promise { - if (!this.enabled || !this.write) return + if (this.enabled && !this.write) throw new Error('FileDocStore is not initialized') // Register BEFORE the async read so a concurrent publish/tailer for this room can't be missed — // the tailer resumes from `lastId`, which the catch-up advances. const room: StoreRoom = { doc, lastId: '0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, + generation: null, + generationInvalidated: false, seededObserved: false, realEdited: false, } this.rooms.set(name, room) - await this.catchUp(name) + if (!this.enabled) return + try { + await this.catchUp(name) + } catch (error) { + if (this.rooms.get(name) === room) this.rooms.delete(name) + throw error + } } /** @@ -312,33 +408,47 @@ export class FileDocStore { * after it. This is the ONLY way a room loads shared state, so a caller that must not depend on the * tailer's asynchronous push — the join, which may not serve a client a half-assembled document — * can converge on demand. Idempotent and safe to call repeatedly; no-op when disabled or the room is - * not registered (a fast open→close detached it). Never throws. + * not registered (a fast open→close detached it). Throws without applying partial state when replay + * cannot complete, so a join never serves a prefix of the shared document. */ async catchUp(name: string): Promise { - if (!this.enabled || !this.write) return + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') const room = this.rooms.get(name) if (!room) return try { - const entries = await this.write.xRange(streamKey(name), '-', '+') - for (const entry of entries) { + const entries: Array<{ id: string; message: Record }> = [] + await this.replayEntries(name, room.lastId, (entry) => { // The room can be detached + its doc destroyed while the read is in flight (a fast // open→close); stop touching it the moment that happens. - if (this.rooms.get(name) !== room) return - // Applying a Yjs update twice is a no-op, but `applyEntry`'s bookkeeping is not: re-applying - // the SEED after `seededObserved` latched would count it as a post-seed edit and let a - // compaction snapshot claim content no user ever typed. Skip what this room already holds. - if (!isAfterStreamId(entry.id, room.lastId)) continue - this.applyEntry(room, entry.id, entry.message) + if (this.rooms.get(name) !== room) return false + entries.push(entry) + return true + }) + if (this.rooms.get(name) !== room) return + for (const entry of entries) this.applyEntry(name, room, entry.id, entry.message) + const docId = room.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + if (room.generation === null && typeof docId === 'string') { + const adopted = await this.write.eval(ADOPT_GENERATION_SCRIPT, { + keys: [streamKey(name), generationKey(name)], + arguments: [docId, String(STREAM_TTL_SEC)], + }) + if (adopted !== docId) throw new FileDocInvalidatedError() + room.generation = docId } await this.write.expire(streamKey(name), STREAM_TTL_SEC) } catch (error) { - logger.warn(`FileDocStore catch-up failed for ${name}`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore catch-up failed for ${name}`, { + error: getErrorMessage(error), + }) + throw error } } /** Deregister a room the relay is destroying, so the tailer stops touching its (about-to-be-destroyed) doc. */ detachRoom(name: string): void { this.rooms.delete(name) + this.localInvalidations.delete(name) } /** @@ -347,8 +457,14 @@ export class FileDocStore { * an edit from the shared log. Only the `xAdd` is retried; the TTL refresh + compaction check are * post-write best-effort and never re-trigger the append. Throws if the append ultimately fails. */ - private async appendUpdate(name: string, update: Uint8Array, agent = false): Promise { + private async appendUpdate( + name: string, + update: Uint8Array, + agent = false, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { if (!this.write) return + assertUpdateWithinLimit(update) // Latch realEdited SYNCHRONOUSLY — before the first await — for a real (non-agent) publish. The edit // already sits in room.doc (applied in doc.on('update') before publish was called), so if this set // were deferred past the xAdd/expire awaits a CONCURRENT agent-frame-triggered maybeCompact could read @@ -361,15 +477,21 @@ export class FileDocStore { if (editedRoom) editedRoom.realEdited = true } const encoded = Buffer.from(update).toString('base64') - const fields: Record = { [UPDATE_FIELD]: encoded } - if (agent) fields[AGENT_FIELD] = '1' + const marker = agent ? AGENT_FIELD : '' for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { - await this.write.xAdd(streamKey(name), '*', fields) + const id = await this.write.eval(APPEND_UPDATE_SCRIPT, { + keys: [streamKey(name), generationKey(name)], + arguments: [UPDATE_FIELD, encoded, marker, expectedGeneration], + }) + if (id === null || id === false) throw new FileDocInvalidatedError() break } catch (error) { + if (error instanceof FileDocInvalidatedError) throw error if (attempt === PUBLISH_MAX_RETRIES) { - logger.error(`FileDocStore append failed for ${name}`, { error: getErrorMessage(error) }) + logger.error(`FileDocStore append failed for ${name}`, { + error: getErrorMessage(error), + }) throw error } // Snappy backoff — a stream append is a fast op; a transient blip clears in tens of ms. @@ -378,8 +500,17 @@ export class FileDocStore { } } await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + await this.write.expire(generationKey(name), STREAM_TTL_SEC).catch(() => {}) const room = this.rooms.get(name) - if (room && ++room.publishes % COMPACT_CHECK_EVERY === 0) void this.maybeCompact(name) + if (room) { + room.publishes += 1 + if ( + room.uncompactedDeltaBytes >= COMPACT_ENCODED_BYTES || + room.publishes % COMPACT_CHECK_EVERY === 0 + ) { + await this.maybeCompact(name) + } + } } /** @@ -389,7 +520,11 @@ export class FileDocStore { */ publish(name: string, update: Uint8Array, agent = false): void { if (!this.enabled || !this.write) return - void this.appendUpdate(name, update, agent).catch(() => {}) // already logged inside appendUpdate + void this.appendUpdate(name, update, agent).catch((error) => { + logger.warn(`FileDocStore rejected a non-durable legacy update for ${name}`, { + error: getErrorMessage(error), + }) + }) } /** @@ -397,9 +532,77 @@ export class FileDocStore { * — the copilot merge, so the cross-task merge lock is not released before the diff is committed * (else the next task would diff a stale base). Throws on ultimate failure. No-op when disabled. */ - async publishAndWait(name: string, update: Uint8Array): Promise { - if (!this.enabled || !this.write) return - await this.appendUpdate(name, update) + async publishAndWait( + name: string, + update: Uint8Array, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') + await this.appendUpdate(name, update, false, expectedGeneration) + } + + /** + * Append a user update exactly once within the stream's bounded deduplication window and return only + * after Redis has accepted it. The client keeps the batch in IndexedDB until this promise succeeds + * and its socket acknowledgement arrives, so a relay restart or lost acknowledgement is safe to + * retry throughout that window. + */ + async publishClientUpdateAndWait( + name: string, + updateId: string, + update: Uint8Array, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') + assertUpdateWithinLimit(update) + const encoded = Buffer.from(update).toString('base64') + const dedupeMember = createHash('sha256') + .update(String(Buffer.byteLength(updateId))) + .update(':') + .update(updateId) + .update(update) + .digest('hex') + const room = this.rooms.get(name) + + for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { + try { + const appended = await this.write.eval(APPEND_CLIENT_UPDATE_SCRIPT, { + keys: [streamKey(name), `${CLIENT_UPDATE_PREFIX}${name}`, generationKey(name)], + arguments: [ + dedupeMember, + UPDATE_FIELD, + encoded, + String(CLIENT_UPDATE_DEDUPE_CAPACITY), + String(STREAM_TTL_SEC), + expectedGeneration, + ], + }) + if (appended === -1) throw new FileDocInvalidatedError() + if (appended === 1 && room) { + room.realEdited = true + room.publishes += 1 + if ( + room.uncompactedDeltaBytes >= COMPACT_ENCODED_BYTES || + room.publishes % COMPACT_CHECK_EVERY === 0 + ) { + await this.maybeCompact(name) + } + } + return + } catch (error) { + if (error instanceof FileDocInvalidatedError) throw error + if (attempt === PUBLISH_MAX_RETRIES) { + logger.error(`FileDocStore acknowledged append failed for ${name}`, { + updateId, + error: getErrorMessage(error), + }) + throw error + } + await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) + } + } } /** @@ -412,20 +615,38 @@ export class FileDocStore { * Retries a transient Redis error like {@link appendUpdate}; throws if it ultimately fails. Disabled → * true (single-replica: seed locally, no stream). */ - async seedIfEmpty(name: string, update: Uint8Array): Promise { - if (!this.enabled || !this.write) return true + async seedIfEmpty(name: string, update: Uint8Array, version = 0): Promise { + if (!this.enabled) { + if ((this.localInvalidations.get(name) ?? 0) > version) return false + this.localInvalidations.delete(name) + return true + } + if (!this.write) throw new Error('FileDocStore is not initialized') + assertUpdateWithinLimit(update) const encoded = Buffer.from(update).toString('base64') + const generation = generationOfSeed(update) for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { const wrote = await this.write.eval(SEED_IF_EMPTY_SCRIPT, { - keys: [streamKey(name)], - arguments: [UPDATE_FIELD, encoded], + keys: [streamKey(name), generationKey(name), `${SYNC_VERSION_PREFIX}${name}`], + arguments: [ + UPDATE_FIELD, + encoded, + generation, + String(STREAM_TTL_SEC), + GENERATION_FIELD, + String(version), + ], }) await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + const room = this.rooms.get(name) + if (wrote === 1 && room) room.generation = generation return wrote === 1 } catch (error) { if (attempt === PUBLISH_MAX_RETRIES) { - logger.error(`FileDocStore seed failed for ${name}`, { error: getErrorMessage(error) }) + logger.error(`FileDocStore seed failed for ${name}`, { + error: getErrorMessage(error), + }) throw error } await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) @@ -434,6 +655,47 @@ export class FileDocStore { return false } + /** + * Invalidate the current live-document generation after a durable replacement that the rich editor + * cannot represent. The invalid generation marker is written before deleting the stream, so stale + * room publishers cannot recreate it; the next authoritative seed atomically replaces the marker + * with its generation. The same TTL as the stream bounds abandoned markers. + */ + async invalidateDocument(name: string, version: number): Promise { + if (!this.enabled) { + if (!this.rooms.has(name)) return true + if ((this.localInvalidations.get(name) ?? 0) >= version) return false + this.localInvalidations.set(name, version) + return true + } + if (!this.write) throw new Error('FileDocStore is not initialized') + return ( + (await this.write.eval(INVALIDATE_DOCUMENT_SCRIPT, { + keys: [ + streamKey(name), + generationKey(name), + `${SYNC_VERSION_PREFIX}${name}`, + `${CLIENT_UPDATE_PREFIX}${name}`, + `${AGENT_STREAM_PREFIX}${name}`, + ], + arguments: [String(version), String(STREAM_TTL_SEC), INVALIDATED_GENERATION], + })) === 1 + ) + } + + async getDocumentGeneration(name: string): Promise { + if (!this.enabled) return '' + if (!this.write) throw new Error('FileDocStore is not initialized') + return (await this.write.get(generationKey(name))) ?? '' + } + + async isDocumentGenerationCurrent(name: string, generation?: string): Promise { + if (!this.enabled) return !this.localInvalidations.has(name) + if (!this.write) throw new Error('FileDocStore is not initialized') + const current = await this.write.get(generationKey(name)) + return current === null ? !generation : current === generation + } + /** * Whether the file's stream already holds content — an EFFICIENCY recheck in {@link shouldSeed} that * skips the seed fetch when a prior holder already seeded (the split-brain guard itself is the atomic @@ -460,12 +722,15 @@ export class FileDocStore { * disabled store return a truthy token so callers proceed single-replica without special-casing. */ private async acquireLock(key: string, ttlMs: number): Promise { - if (!this.enabled || !this.write) return DISABLED_LOCK_TOKEN + if (!this.enabled) return DISABLED_LOCK_TOKEN + if (!this.write) return null const token = generateId() try { return (await this.write.set(key, token, { NX: true, PX: ttlMs })) === 'OK' ? token : null } catch (error) { - logger.warn(`FileDocStore lock ${key} failed`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore lock ${key} failed`, { + error: getErrorMessage(error), + }) return null } } @@ -504,19 +769,69 @@ export class FileDocStore { * `null` when the stream is empty — i.e. no doc is (or was recently) live, so there is nothing to * merge into and the caller should fall back to a direct file write. Disabled → always null. */ - async getStreamState(name: string): Promise { - if (!this.enabled || !this.write) return null - const entries = await this.write.xRange(streamKey(name), '-', '+') - if (entries.length === 0) return null + async getStreamState(name: string, expectedGeneration?: string): Promise { + if (!this.enabled) return null + if (!this.write) throw new Error('FileDocStore is not initialized') const doc = new Y.Doc() try { - for (const entry of entries) applyEntryToDoc(doc, entry.id, entry.message) + const generation = await this.getDocumentGeneration(name) + if (expectedGeneration !== undefined && generation !== expectedGeneration) { + throw new FileDocInvalidatedError() + } + const count = await this.replayEntries(name, '0', (entry) => { + if (entry.message[GENERATION_FIELD] && entry.message[GENERATION_FIELD] !== generation) { + throw new FileDocInvalidatedError() + } + applyEntryToDoc(doc, entry.id, entry.message) + return true + }) + if ((await this.getDocumentGeneration(name)) !== generation) { + throw new FileDocInvalidatedError() + } + if (count === 0) return null return Y.encodeStateAsUpdate(doc) } finally { doc.destroy() } } + private async replayEntries( + name: string, + afterId: string, + visit: (entry: { id: string; message: Record }) => boolean + ): Promise { + if (!this.write) return 0 + const key = streamKey(name) + const tail = await this.write.xRevRange(key, '+', '-', { COUNT: 1 }) + if (tail.length === 0) return 0 + const endId = tail[0].id + let cursor = afterId.includes('-') ? afterId : `${afterId}-0` + let entriesRead = 0 + let encodedBytes = 0 + + while (isAfterStreamId(endId, cursor)) { + const page = await this.write.xRange(key, `(${cursor}`, '+', { + COUNT: REPLAY_PAGE_COUNT, + }) + if (page.length === 0) { + if (isAfterStreamId(endId, cursor)) { + throw new Error(`File document replay lost its completion barrier for ${name}`) + } + break + } + for (const entry of page) { + entriesRead += 1 + encodedBytes += entry.message[UPDATE_FIELD]?.length ?? 0 + if (entriesRead > REPLAY_MAX_ENTRIES || encodedBytes > REPLAY_MAX_ENCODED_BYTES) { + throw new Error(`File document replay exceeded its safety limit for ${name}`) + } + cursor = entry.id + if (!visit(entry)) return entriesRead + } + } + return entriesRead + } + /** Release the seed lock (compare-and-delete) once the seed has been published or a seed attempt failed. */ async releaseSeedLock(name: string, token: string): Promise { await this.releaseLock(`${SEED_LOCK_PREFIX}${name}`, token) @@ -580,7 +895,11 @@ export class FileDocStore { * new value exceeds the stored one ({@link SET_VERSION_IF_NEWER_SCRIPT}), so an out-of-order * fire-and-forget write can't regress the token. Best-effort; TTL-bounded like the stream so an idle * file's key can't outlive its room. No-op when disabled (single-pod fallback). */ - async setSyncedVersion(name: string, version: number): Promise { + async setSyncedVersion( + name: string, + version: number, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { if (!this.enabled || !this.write) return // Retry a transient failure (bounded) rather than swallow it: this token is the ONLY way a // peer-seeded task learns the durable version, so a dropped write would leave that peer's persists @@ -589,8 +908,8 @@ export class FileDocStore { for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { await this.write.eval(SET_VERSION_IF_NEWER_SCRIPT, { - keys: [`${SYNC_VERSION_PREFIX}${name}`], - arguments: [String(version), String(STREAM_TTL_SEC)], + keys: [`${SYNC_VERSION_PREFIX}${name}`, generationKey(name)], + arguments: [String(version), String(STREAM_TTL_SEC), expectedGeneration], }) return } catch (error) { @@ -637,8 +956,30 @@ export class FileDocStore { await this.releaseLock(`${MERGE_LOCK_PREFIX}${name}`, token) } - private applyEntry(room: StoreRoom, id: string, message: Record): void { + private applyEntry( + name: string, + room: StoreRoom, + id: string, + message: Record + ): void { + if (!isAfterStreamId(id, room.lastId)) return room.lastId = id + const generation = message[GENERATION_FIELD] + if (generation) { + if ( + room.generationInvalidated || + (room.generation !== null && room.generation !== generation) || + (room.generation === null && room.seededObserved) + ) { + room.generationInvalidated = true + return + } + room.generation = generation + } + if (room.generationInvalidated) return + const isSnapshot = + message[GENERATION_FIELD] !== undefined || message[SNAPSHOT_FIELD] !== undefined + if (!isSnapshot) room.uncompactedDeltaBytes += message[UPDATE_FIELD]?.length ?? 0 // A compaction snapshot folds seed + edits into one frame; stamp it so the relay's edit-tracker // treats a fresh catch-up from it as edited (a snapshot only exists once real edits accumulated). An // agent-streamed preview frame is stamped separately so the tracker NEVER marks it edited. @@ -657,6 +998,9 @@ export class FileDocStore { if (origin === REDIS_SNAPSHOT_ORIGIN || (origin === REDIS_ORIGIN && seededBefore)) { room.realEdited = true } + if (!isSnapshot && room.uncompactedDeltaBytes >= COMPACT_ENCODED_BYTES) { + void this.maybeCompact(name) + } } /** @@ -665,6 +1009,7 @@ export class FileDocStore { */ private async runReader(): Promise { let failures = 0 + let blockingBatchIndex = 0 while (this.running && this.read) { const snapshot = new Map(this.rooms) if (snapshot.size === 0) { @@ -672,26 +1017,40 @@ export class FileDocStore { continue } try { - const res = await this.read.xRead( - [...snapshot].map(([name, room]) => ({ key: streamKey(name), id: room.lastId })), - { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT } - ) - // The streak ends HERE, on the read returning at all — not further down once entries are - // applied. A blocking read that times out with nothing new is the idle steady state, and it - // proves the connection works just as well as one carrying messages; leaving the streak - // standing through it would keep an old outage's count alive indefinitely, so the next - // unrelated blip would open at the backoff cap and log a failure count it never earned. - failures = 0 - if (!res) continue - for (const stream of res) { - const name = stream.name.slice(STREAM_PREFIX.length) - const room = this.rooms.get(name) - // Skip if detached mid-read, OR replaced by a close→reopen (a DIFFERENT StoreRoom): applying - // entries read against the OLD room's lastId to the new one could regress its lastId (harmless - // but wasteful re-delivery). The new room caught itself up via xRange already. - if (!room || room !== snapshot.get(name)) continue - for (const entry of stream.messages) this.applyEntry(room, entry.id, entry.message) + const rooms = [...snapshot] + const batches: Array = [] + for (let index = 0; index < rooms.length; index += READ_STREAM_BATCH_SIZE) { + batches.push(rooms.slice(index, index + READ_STREAM_BATCH_SIZE)) + } + const applyResults = (results: Awaited>): boolean => { + if (!results) return false + for (const stream of results) { + const name = stream.name.slice(STREAM_PREFIX.length) + const room = this.rooms.get(name) + if (!room || room !== snapshot.get(name)) continue + for (const entry of stream.messages) + this.applyEntry(name, room, entry.id, entry.message) + } + return true + } + let received = false + for (const batch of batches) { + const streams = batch.map(([name, room]) => ({ + key: streamKey(name), + id: room.lastId, + })) + received = applyResults(await this.read.xRead(streams, { COUNT: READ_COUNT })) || received + } + if (!received) { + const batch = batches[blockingBatchIndex % batches.length] + blockingBatchIndex = (blockingBatchIndex + 1) % batches.length + const streams = batch.map(([name, room]) => ({ + key: streamKey(name), + id: room.lastId, + })) + applyResults(await this.read.xRead(streams, { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT })) } + failures = 0 } catch (error) { if (!this.running) break await this.recoverReader(++failures, error) @@ -718,7 +1077,12 @@ export class FileDocStore { error: getErrorMessage(error), }) } - await sleep(backoffWithJitter(failures, null, { baseMs: 500, maxMs: READER_RETRY_MAX_MS })) + await sleep( + backoffWithJitter(failures, null, { + baseMs: 500, + maxMs: READER_RETRY_MAX_MS, + }) + ) if (this.running && this.read && !this.read.isOpen) { await this.read.connect().catch((reconnectError) => { logger.warn('FileDocStore could not re-open the reader connection', { @@ -737,13 +1101,20 @@ export class FileDocStore { private async maybeCompact(name: string): Promise { if (!this.write) return const room = this.rooms.get(name) - if (!room) return + if (!room || room.compacting) return + room.compacting = true try { - if ((await this.write.xLen(streamKey(name))) < COMPACT_THRESHOLD) return + const streamLength = await this.write.xLen(streamKey(name)) + if (streamLength < COMPACT_THRESHOLD && room.uncompactedDeltaBytes < COMPACT_ENCODED_BYTES) { + return + } const key = `${COMPACT_LOCK_PREFIX}${name}` const token = await this.acquireLock(key, COMPACT_LOCK_TTL_MS) if (!token) return try { + /** Integrate the completed stream prefix before capturing the snapshot and compaction barrier. */ + await this.catchUp(name) + if (this.rooms.get(name) !== room) return // Capture the snapshot AND the id it covers in one synchronous step (no await between): the // snapshot is `room.doc`, which holds exactly what this task's tailer has integrated — every // entry up to `room.lastId`. Entries a peer task published AFTER that (id > lastId) are NOT in @@ -751,24 +1122,32 @@ export class FileDocStore { // them — only entries the snapshot provably subsumes (id <= lastId). Trimming to the freshly // appended snapshot id instead would silently drop those un-integrated peer entries. const upTo = room.lastId + const deltaBytesAtBarrier = room.uncompactedDeltaBytes const snapshot = Buffer.from(Y.encodeStateAsUpdate(room.doc)).toString('base64') // Stamp the snapshot by what it folds: a real edit → SNAPSHOT_FIELD (a fresh catch-up treats it // as edited content, not a bare seed). An agent-ONLY stream (no real edit yet) → AGENT_FIELD, so a // peer catching up applies it as REDIS_AGENT_ORIGIN and never marks the doc edited — preserving // the no-persist guarantee even when a long copilot stream alone crosses the compaction threshold. const marker = room.realEdited ? SNAPSHOT_FIELD : AGENT_FIELD - await this.write.xAdd(streamKey(name), '*', { - [UPDATE_FIELD]: snapshot, - [marker]: '1', + const snapshotId = await this.write.eval(APPEND_SNAPSHOT_SCRIPT, { + keys: [streamKey(name), generationKey(name)], + arguments: [UPDATE_FIELD, snapshot, marker, room.generation ?? '', GENERATION_FIELD], }) + if (typeof snapshotId !== 'string') return // MINID keeps entries with id >= upTo: the snapshot, any un-integrated peer entries, and // `upTo` itself (redundant with the snapshot, harmless); it drops only the folded older deltas. await this.write.xTrim(streamKey(name), 'MINID', upTo) + /** Deltas observed after the barrier survive MINID; snapshots never contribute to this count. */ + room.uncompactedDeltaBytes = Math.max(0, room.uncompactedDeltaBytes - deltaBytesAtBarrier) } finally { await this.releaseLock(key, token) } } catch (error) { - logger.warn(`FileDocStore compaction failed for ${name}`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore compaction failed for ${name}`, { + error: getErrorMessage(error), + }) + } finally { + room.compacting = false } } @@ -779,6 +1158,7 @@ export class FileDocStore { // Keep the synced-version key alive as long as its stream, so an open-but-idle doc's persist // If-Match token can't expire out from under it (which would force a needless reconcile). await this.write.expire(`${SYNC_VERSION_PREFIX}${name}`, STREAM_TTL_SEC).catch(() => {}) + await this.write.expire(generationKey(name), STREAM_TTL_SEC).catch(() => {}) } } } diff --git a/apps/realtime/src/handlers/file-doc.join-readiness.test.ts b/apps/realtime/src/handlers/file-doc.join-readiness.test.ts index 9b7b6a1c7ec..4938c0680f4 100644 --- a/apps/realtime/src/handlers/file-doc.join-readiness.test.ts +++ b/apps/realtime/src/handlers/file-doc.join-readiness.test.ts @@ -63,12 +63,19 @@ vi.mock('redis', () => { for (let i = 0; i < backing.readDelayTicks; i++) await Promise.resolve() return (backing.streams.get(key) ?? []).map((e) => ({ ...e })) }, + xRevRange: async (key: string) => + [...(backing.streams.get(key) ?? [])] + .reverse() + .slice(0, 1) + .map((entry) => ({ ...entry })), xLen: async (key: string) => (backing.streams.get(key) ?? []).length, - xRead: async (streams: { key: string; id: string }[]) => { + xRead: async (streams: { key: string; id: string }[], options?: { COUNT?: number }) => { const res: { name: string; messages: { id: string; message: Record }[] }[] = [] for (const { key, id } of streams) { - const after = (backing.streams.get(key) ?? []).filter((e) => seqOf(e.id) > seqOf(id)) + const after = (backing.streams.get(key) ?? []) + .filter((e) => seqOf(e.id) > seqOf(id)) + .slice(0, options?.COUNT) if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) } if (res.length) return res diff --git a/apps/realtime/src/handlers/file-doc.multireplica.test.ts b/apps/realtime/src/handlers/file-doc.multireplica.test.ts index cfffe81e857..cb24909618f 100644 --- a/apps/realtime/src/handlers/file-doc.multireplica.test.ts +++ b/apps/realtime/src/handlers/file-doc.multireplica.test.ts @@ -25,6 +25,7 @@ const fakeStore = { versions: new Map(), acquireMergeSlot: vi.fn(async () => 'token'), releaseMergeSlot: vi.fn(async () => {}), + getDocumentGeneration: vi.fn(async () => 'shared-generation'), getStreamState: vi.fn(async () => new Uint8Array([1])), publishAndWait: vi.fn(async () => {}), getSyncedVersion: vi.fn(async (name: string) => fakeStore.versions.get(name) ?? null), @@ -67,7 +68,13 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' expect(await applyMarkdownToLiveFileDoc('file-1', '# durable', { version: 100 })).toBe( 'applied' ) - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100) + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100, 'shared-generation') + expect(fakeStore.getStreamState).toHaveBeenCalledWith(ROOM_NAME, 'shared-generation') + expect(fakeStore.publishAndWait).toHaveBeenCalledWith( + ROOM_NAME, + expect.any(Uint8Array), + 'shared-generation' + ) mockFetchFileDocMerge.mockClear() // A durable write with an OLDER version than the SHARED synced version is stale — rejected under the @@ -81,7 +88,7 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' expect(await applyMarkdownToLiveFileDoc('file-1', '# durable again', { version: 150 })).toBe( 'applied' ) - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 150) + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 150, 'shared-generation') // setSyncedVersion fired only for the two applied durable writes, never for the stale one. expect(fakeStore.setSyncedVersion).toHaveBeenCalledTimes(2) }) @@ -98,7 +105,7 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' ).toBe('applied') expect(mockFetchFileDocMerge).not.toHaveBeenCalled() // content deferred to the client expect(fakeStore.publishAndWait).not.toHaveBeenCalled() - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100) // version still recorded + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100, 'shared-generation') // version still recorded // Once streaming stops the flag clears and the (now near-noop) durable merge resumes normally. fakeStore.isAgentStreaming.mockResolvedValue(false) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index c0878d90129..d3dc7193615 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -3,6 +3,7 @@ */ import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, FILE_DOC_SEED, } from '@sim/realtime-protocol/file-doc' @@ -40,9 +41,10 @@ import { flushAllFileDocRooms, setupWorkspaceFileDocHandlers, } from '@/handlers/file-doc' +import { FileDocInvalidatedError, getFileDocStore } from '@/handlers/file-doc-store' import { beginRoomPermissionRead, commitRoomPermission } from '@/middleware/permissions' -type Handler = (payload?: unknown) => Promise | void +type Handler = (...payload: unknown[]) => Promise | void const ROOM_NAME = 'workspace-file-doc:file-1' @@ -133,10 +135,11 @@ async function flushMicrotasks(): Promise { * An encoded Yjs update shaped like the server seed builder's output: some content in the shared * `default` type plus the {@link FILE_DOC_SEED} flag, so applying it marks the doc seeded. */ -function seedResult(content: string): { update: Uint8Array; version: number } { +function seedResult(content: string, docId?: string): { update: Uint8Array; version: number } { const doc = new Y.Doc() doc.getText(FILE_DOC_FIELD).insert(0, content) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + if (docId) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, docId) return { update: Y.encodeStateAsUpdate(doc), version: 1 } } @@ -217,6 +220,24 @@ describe('setupWorkspaceFileDocHandlers', () => { ) }) + it('fails closed when authorization does not resolve a workspace context', async () => { + mockAuthorizeRoom.mockResolvedValueOnce({ + allowed: true, + status: 200, + workspacePermission: 'write', + }) + const { io } = createIo() + const { socket, handlers } = setup('socket-no-workspace', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'JOIN_FAILED', retryable: true }) + ) + expect(socket.join).not.toHaveBeenCalled() + }) + it('rejects join with a retryable error when realtime is unavailable', async () => { const { io } = createIo() const { socket, handlers } = createSocket('socket-1') @@ -248,6 +269,196 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockAuthorizeRoom).not.toHaveBeenCalled() }) + it('rejects an incompatible collaborative-document schema before authorizing', async () => { + const { io } = createIo() + const { socket, handlers } = setup('socket-schema', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ + fileId: 'file-1', + clientId: 1, + schemaVersion: 99, + }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'SCHEMA_VERSION_MISMATCH', retryable: false }) + ) + expect(mockAuthorizeRoom).not.toHaveBeenCalled() + }) + + it('acknowledges user updates only after applying them to the joined document', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io, sent } = createIo() + const { handlers } = setup('socket-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'acknowledged edit') + const acknowledge = vi.fn() + sent.length = 0 + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-1', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ status: 'accepted', updateId: 'update-1' }) + expect(sent).toContainEqual( + expect.objectContaining({ + target: ROOM_NAME, + event: FILE_DOC_EVENTS.MESSAGE, + }) + ) + source.destroy() + }) + + it('rejects an update for a replaced document without applying it', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-current')) + const { io, sent } = createIo() + const { handlers } = setup('socket-replaced', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const acknowledge = vi.fn() + sent.length = 0 + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-stale', + updateId: 'update-stale', + update: Y.encodeStateAsUpdate(new Y.Doc()), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'DOCUMENT_REPLACED', + retryable: false, + updateId: 'update-stale', + }) + expect(sent).toHaveLength(0) + }) + + it('rejects malformed Yjs updates without retrying them', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io } = createIo() + const { handlers } = setup('socket-malformed-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const acknowledge = vi.fn() + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-malformed', + update: new Uint8Array([255]), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'INVALID_UPDATE', + retryable: false, + updateId: 'update-malformed', + }) + }) + + it('ignores an acknowledged-update event without a callable acknowledgement', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io } = createIo() + const { handlers } = setup('socket-missing-ack', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(() => + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-1', + update: Y.encodeStateAsUpdate(new Y.Doc()), + }, + { not: 'a function' } + ) + ).not.toThrow() + }) + + it('keeps a room alive until an acknowledged update finishes appending', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + let resolveAppend: () => void = () => {} + const append = new Promise((resolve) => { + resolveAppend = resolve + }) + const publish = vi + .spyOn(getFileDocStore(), 'publishClientUpdateAndWait') + .mockReturnValue(append) + const { io } = createIo() + const { handlers } = setup('socket-update-leave', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'accepted before leave') + const acknowledge = vi.fn() + + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-leave', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + await vi.waitFor(() => expect(publish).toHaveBeenCalledTimes(1)) + handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveAppend() + + await vi.waitFor(() => + expect(acknowledge).toHaveBeenCalledWith({ + status: 'accepted', + updateId: 'update-leave', + }) + ) + publish.mockRestore() + source.destroy() + }) + + it('rejects a generation-fenced update as a durable document replacement', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const publish = vi + .spyOn(getFileDocStore(), 'publishClientUpdateAndWait') + .mockRejectedValue(new FileDocInvalidatedError()) + const { io } = createIo() + const { handlers } = setup('socket-replaced-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'stale edit') + const acknowledge = vi.fn() + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-replaced', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'DOCUMENT_REPLACED', + retryable: false, + updateId: 'update-replaced', + }) + publish.mockRestore() + source.destroy() + }) + it('does not re-enter the room when access was revoked while the join was in flight', async () => { // The sweep records a revocation before it evicts, so a join whose authorize // completed just before that must not put the socket back in the document. @@ -550,6 +761,10 @@ describe('setupWorkspaceFileDocHandlers', () => { FILE_DOC_EVENTS.JOIN_SUCCESS, expect.objectContaining({ fileId: 'file-1', clientId: 1 }) ) + const joinSuccess = socket.emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.JOIN_SUCCESS + )?.[1] as Record + expect(joinSuccess).not.toHaveProperty('acknowledgedUpdates') // A binary sync-step-1 message (type tag 0) is sent to kick off the handshake. const syncMessage = socket.emit.mock.calls.find( @@ -575,6 +790,33 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# From server') }) + it('discards a fenced in-memory generation before serving the next join', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io, left } = createIo() + const first = setup('socket-old-generation', io) + await first.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + await getFileDocStore().invalidateDocument(ROOM_NAME, 1) + mockFetchFileDocSeed.mockResolvedValue(seedResult('# New', 'doc-new')) + const second = setup('socket-new-generation', io) + await second.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + expect(left).toContainEqual({ socketId: 'socket-old-generation', room: ROOM_NAME }) + second.socket.emit.mockClear() + second.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeSyncStep1(encoder, new Y.Doc()) + ) + ) + const reply = second.socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# New') + clientDoc.destroy() + }) + it('seeds once across concurrent joiners, and every one of them waits for that seed', async () => { // Keep the first seed fetch IN FLIGHT so the doc is still unseeded when the second socket joins: // that forces the dedup onto the in-flight seed rather than `isDocSeeded`. Both joins must WAIT @@ -1030,6 +1272,31 @@ describe('setupWorkspaceFileDocHandlers', () => { ).not.toThrow() }) + it('drops a legacy frame that cannot fit the durable stream budget', async () => { + const { io, sent } = createIo() + const a = setup('socket-oversized-legacy', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + sent.length = 0 + + expect(() => + a.handlers[FILE_DOC_EVENTS.MESSAGE](new Uint8Array(FILE_DOC_LIMITS.updateBytes + 65)) + ).not.toThrow() + expect(sent).toHaveLength(0) + }) + + it('preflights the inner legacy update before applying a framing-sized overflow', async () => { + const { io, sent } = createIo() + const a = setup('socket-inner-oversized-legacy', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + sent.length = 0 + const oversized = frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, new Uint8Array(FILE_DOC_LIMITS.updateBytes + 1)) + ) + + expect(() => a.handlers[FILE_DOC_EVENTS.MESSAGE](oversized)).not.toThrow() + expect(sent).toHaveLength(0) + }) + it('drops the document when the last editor leaves, re-seeding a fresh joiner from the server', async () => { const { io } = createIo() const a = setup('socket-a', io) @@ -1053,12 +1320,22 @@ describe('setupWorkspaceFileDocHandlers', () => { let resolveFirst: (v: unknown) => void = () => {} mockAuthorizeRoom .mockReturnValueOnce(new Promise((resolve) => (resolveFirst = resolve))) - .mockResolvedValueOnce({ allowed: true, status: 200, workspacePermission: 'write' }) + .mockResolvedValueOnce({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) const s = setup('socket-a', io) const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) - resolveFirst({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveFirst({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending // The socket is bound only to the newer file, never cross-bound to file-1. @@ -1075,7 +1352,12 @@ describe('setupWorkspaceFileDocHandlers', () => { const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) s.socket.disconnected = true cleanupFileDocForSocket('socket-a', io, true) // disconnect cleanup — no-op, nothing registered yet - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(s.socket.join).not.toHaveBeenCalled() @@ -1095,7 +1377,12 @@ describe('setupWorkspaceFileDocHandlers', () => { const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) // A stale leave for a DIFFERENT file must not invalidate the in-flight join. s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(joinSuccessFileId(s.socket)).toBe('file-2') @@ -1119,7 +1406,12 @@ describe('setupWorkspaceFileDocHandlers', () => { // map (`undefined !== generation`) and abort the join the client actually wants. s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(joinSuccessFileId(s.socket)).toBe('file-2') diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 28ef6686f7a..08d58b55451 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -27,10 +27,15 @@ import { createLogger } from '@sim/logger' import { ROOM_MEMBERSHIP_ACTIONS, satisfiesRoomMembership } from '@sim/platform-authz/room-policy' import { FILE_DOC_EVENTS, + FILE_DOC_LEGACY_SCHEMA_VERSION, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, FILE_DOC_TIMEOUTS, type FileDocPresenceUser, + type FileDocUpdateAck, + type FileDocUpdatePayload, type JoinFileDocPayload, type LeaveFileDocPayload, toFileDocBytes, @@ -47,6 +52,7 @@ import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' import { fetchFileDocMerge, fetchFileDocPersist, fetchFileDocSeed } from '@/handlers/file-doc-app' import { + FileDocInvalidatedError, getFileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN, @@ -176,7 +182,7 @@ interface FileDocRoom { agentStreamingUntil: number /** * Resolves once this room's doc reflects the file's shared stream (see {@link FileDocStore.catchUp}). - * Never rejects — the catch-up logs and gives up — so awaiting it can never fail a join. + * Rejects when replay cannot complete so the join fails closed rather than serving partial state. */ hydrated: Promise /** @@ -185,6 +191,8 @@ interface FileDocRoom { * document being assembled. A room with a join in flight is not idle. */ pendingJoins: number + /** Acknowledged updates currently waiting for their durable stream append. */ + pendingUpdates: number } /** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ @@ -223,8 +231,29 @@ const fileDocRoom = (fileId: string): RoomRef => ({ * `'timeout'`) for server-internal changes. Returns the socket id to exclude * from a relay, or `null` to broadcast to the whole room. */ +interface ClientUpdateOrigin { + kind: 'client-update' + socketId: string +} + +const MAX_CLIENT_UPDATE_ID_LENGTH = 128 + +function clientUpdateOrigin(socketId: string): ClientUpdateOrigin { + return { kind: 'client-update', socketId } +} + +function isClientUpdateOrigin(origin: unknown): origin is ClientUpdateOrigin { + return ( + typeof origin === 'object' && + origin !== null && + (origin as Partial).kind === 'client-update' && + typeof (origin as Partial).socketId === 'string' + ) +} + function originSocketId(origin: unknown): string | null { - return typeof origin === 'string' ? origin : null + if (typeof origin === 'string') return origin + return isClientUpdateOrigin(origin) ? origin.socketId : null } /** @@ -235,6 +264,29 @@ function originSocketId(origin: unknown): string | null { * on its own echo because the operations are already applied locally. */ const AGENT_SYNC_ORIGIN = Symbol('file-doc-agent-sync') +/** Maximum legacy framed message size: raw update budget plus small Yjs framing headroom. */ +const MAX_LEGACY_FRAME_BYTES = FILE_DOC_LIMITS.updateBytes + 64 + +/** + * Preflight the update-bearing inner Yjs message before `readSyncMessage` can mutate the room. Legacy + * clients use the unacknowledged sync channel, so the relay itself must ensure any applied update also + * fits the durable Redis stream; checking only the outer frame leaves a small framing-sized gap. + */ +function hasOversizedLegacyUpdate(bytes: Uint8Array): boolean { + const decoder = decoding.createDecoder(bytes) + const messageType = decoding.readVarUint(decoder) + if ( + messageType !== FILE_DOC_MESSAGE_TYPE.SYNC && + messageType !== FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST + ) { + return false + } + const syncType = decoding.readVarUint(decoder) + if (syncType !== syncProtocol.messageYjsSyncStep2 && syncType !== syncProtocol.messageYjsUpdate) { + return false + } + return decoding.readVarUint8Array(decoder).byteLength > FILE_DOC_LIMITS.updateBytes +} /** * Broadcast an AWARENESS frame to the room ACROSS tasks via the Socket.IO Redis adapter. Awareness @@ -299,6 +351,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr // Never project a doc no user actually edited back over the file (see {@link FileDocRoom.edited}). if (!room.edited || !room.workspaceId || !room.lastEditorUserId) return const store = getFileDocStore() + const generation = docIdOf(room.doc) const workspaceId = room.workspaceId const userId = room.lastEditorUserId // Synchronous fallback capture — before any await, since the caller may destroy `room.doc` the moment @@ -321,6 +374,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr try { return (await store.getStreamState(name)) ?? localState } catch (streamError) { + if (streamError instanceof FileDocInvalidatedError) throw streamError // A transient Redis read must NOT drop the write when we already hold a valid local snapshot — // else the last-disconnect flush loses the session's edits as the room is torn down. But once a // reconcile has run, `localState` is NULLED (it predates the merged-in out-of-band edit), so a @@ -344,6 +398,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr } try { + if (!(await store.isDocumentGenerationCurrent(name, generation))) return if (!final && !(await store.tryClaimPersistWindow(name, FILE_DOC_TIMEOUTS.persistRequestMs))) return @@ -367,6 +422,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr // out-of-band edit. A single attempt — on conflict we STOP rather than retry (see below). const docState = await captureState() if (!docState) return // nothing seeded/authoritative to persist yet + if (!(await store.isDocumentGenerationCurrent(name, generation))) return const result = await fetchFileDocPersist(workspaceId, room.fileId, userId, docState, ifMatch) if (result.status === 'missing') return // the file was deleted; nothing to write if (result.status === 'deferred') { @@ -382,12 +438,12 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr // here means a task that exits in the moments after a write comes back holding a version older // than the file's, and — since a conflict neither writes nor advances the token — never persists // that document again. One round trip after a blob write is not a cost worth that. - await store.setSyncedVersion(name, result.version) + await store.setSyncedVersion(name, result.version, generation) return } // status === 'conflict': the durable file advanced out-of-band since our If-Match token. We do NOT // re-persist against the current stream: an external write commits durable BEFORE its chokepoint merge - // (`mergeEditIntoLiveFileDoc`) reaches the stream, so a re-persist landing in that window would CAS-pass + // (`applyEditToLiveFileDoc`) reaches the stream, so a re-persist landing in that window would CAS-pass // with a stream that still lacks the external content and clobber the committed write. Instead leave the // durable content authoritative — the chokepoint merges the change into the stream and, ONLY once it is // actually there, advances the synced version (via the merge's own `recordVersion`); a later flush @@ -398,7 +454,9 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr `Persist conflict for file ${room.fileId}; durable content advanced out-of-band, left authoritative` ) } catch (error) { - logger.warn(`Persist failed for file ${room.fileId}`, { error: getErrorMessage(error) }) + logger.warn(`Persist failed for file ${room.fileId}`, { + error: getErrorMessage(error), + }) } } @@ -469,7 +527,7 @@ function awarenessUpdateClientIds(update: Uint8Array): number[] { */ function destroyRoomIfIdle(name: string) { const room = fileDocRooms.get(name) - if (!room || room.owners.size > 0 || room.pendingJoins > 0) return + if (!room || room.owners.size > 0 || room.pendingJoins > 0 || room.pendingUpdates > 0) return room.persistDeadline = null if (room.persistTimer) { clearTimeout(room.persistTimer) @@ -484,6 +542,27 @@ function destroyRoomIfIdle(name: string) { fileDocRooms.delete(name) } +/** + * Drop a seeded in-memory generation after an out-of-band durable replacement. It must not flush: the + * durable replacement is newer, and persisting this superseded document would only create a conflict. + * Existing clients are removed from the room before the next join creates and seeds a fresh document. + */ +function discardInvalidatedRoom(name: string, io: Server): void { + const room = fileDocRooms.get(name) + if (!room) return + room.persistDeadline = null + if (room.persistTimer) clearTimeout(room.persistTimer) + room.persistTimer = null + for (const socketId of room.owners.keys()) { + if (socketToRoomName.get(socketId) === name) socketToRoomName.delete(socketId) + io.in(socketId).socketsLeave(name) + } + getFileDocStore().detachRoom(name) + room.awareness.destroy() + room.doc.destroy() + fileDocRooms.delete(name) +} + /** * Flush every open, edited room's converged doc to durable markdown, AWAITING the writes. Called on * graceful shutdown (rolling deploy / scale-in) so edits since the last debounce aren't left only in the @@ -501,9 +580,9 @@ export async function flushAllFileDocRooms(): Promise { /** * Bring a room's document to its AUTHORITATIVE state — reflecting the file's shared stream and - * carrying its seed — so the join can attach a client to a document that is already whole. Never - * rejects: a room that cannot be seeded is served unseeded, which the client's readiness deadline - * turns into its read-only fallback, exactly as an unreachable relay does. + * carrying its seed — so the join can attach a client to a document that is already whole. Rejects + * when hydration or seeding cannot complete; serving an unseeded or partial room would make a client + * appear editable before the authoritative document exists. */ async function ensureRoomReady( name: string, @@ -513,8 +592,12 @@ async function ensureRoomReady( await room.hydrated // The room can be dropped and re-created while the catch-up is in flight (a fast open→close); the // join re-checks identity after this and abandons a stale room rather than serving from it. - if (fileDocRooms.get(name) !== room || !workspaceId) return + if (fileDocRooms.get(name) !== room) return + if (!workspaceId) throw new Error(`File document ${room.fileId} has no workspace context`) await ensureServerSeed(name, room, workspaceId) + if (fileDocRooms.get(name) === room && !isDocSeeded(room.doc)) { + throw new Error(`File document ${room.fileId} could not be seeded`) + } } /** @@ -588,7 +671,7 @@ async function seedUnderLock( // the doc unseeded and the stream empty for a clean retry. SEED_ORIGIN keeps `doc.on('update')` from // re-publishing it. const seedUpdate = seed?.update ?? emptySeedUpdate() - const didSeed = await store.seedIfEmpty(name, seedUpdate) + const didSeed = await store.seedIfEmpty(name, seedUpdate, seed?.version) // Record the durable version the moment THIS task's seed is in the stream — BEFORE the liveness/ // seeded guard below. Recording it only now that our seed WON (not from the fetch, before knowing who // won) keeps it in step with the stream's actual content: a newer own-fetch version could otherwise @@ -601,7 +684,6 @@ async function seedUnderLock( if (didSeed && seed) { const live = fileDocRooms.get(name) if (live) live.syncedVersion = Math.max(live.syncedVersion ?? 0, seed.version) - void store.setSyncedVersion(name, seed.version) } if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return if (didSeed) { @@ -671,18 +753,50 @@ export function applyMarkdownToLiveFileDoc( order: MergeOrder = {} ): Promise<'applied' | 'no-live-room' | 'merge-unavailable' | 'stale'> { const name = roomName(fileDocRoom(fileId)) + return serializeFileDocMutation(name, () => mergeMarkdownIntoRoom(name, fileId, markdown, order)) +} + +function serializeFileDocMutation(name: string, operation: () => Promise): Promise { const prior = fileDocMergeChains.get(name) ?? Promise.resolve() - // `.catch` so a failed prior merge doesn't reject this one — each merge is independent. - const run = prior.catch(() => {}).then(() => mergeMarkdownIntoRoom(name, fileId, markdown, order)) - fileDocMergeChains.set( - name, - run.finally(() => { + const run = prior + .catch(() => {}) + .then(operation) + .finally(() => { if (fileDocMergeChains.get(name) === run) fileDocMergeChains.delete(name) }) - ) + fileDocMergeChains.set(name, run) return run } +async function acquireFileDocMergeSlot(name: string): Promise { + const store = getFileDocStore() + let token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + for (let i = 0; !token && i < MERGE_LOCK_RETRIES; i++) { + await sleep(MERGE_LOCK_RETRY_MS) + token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + } + return token +} + +/** Serializes and version-orders an unsupported durable replacement with live Markdown merges. */ +export function invalidateLiveFileDocument( + fileId: string, + version: number +): Promise<'applied' | 'stale'> { + const name = roomName(fileDocRoom(fileId)) + return serializeFileDocMutation(name, async () => { + const store = getFileDocStore() + const token = await acquireFileDocMergeSlot(name) + if (!token) throw new Error('Live document invalidation slot is temporarily unavailable') + try { + if ((fileDocRooms.get(name)?.syncedVersion ?? 0) > version) return 'stale' + return (await store.invalidateDocument(name, version)) ? 'applied' : 'stale' + } finally { + await store.releaseMergeSlot(name, token) + } + }) +} + async function mergeMarkdownIntoRoom( name: string, fileId: string, @@ -695,14 +809,16 @@ async function mergeMarkdownIntoRoom( // in Redis for multi-task, plus this task's room) so the persist If-Match guard treats this write as // synced rather than an out-of-band conflict. AWAITED so the version is durable before the merge lock // releases, so the next lock holder's staleness check (below) reads a consistent value. - const recordVersion = async () => { + const recordVersion = async (generation?: string) => { if (version === undefined) return const room = fileDocRooms.get(name) // Never regress the token: merges/seeds/persists all write it, so a lower value arriving out of // order must not shadow a higher one the doc already incorporates (the Redis side is guarded // identically by SET_VERSION_IF_NEWER_SCRIPT). - if (room) room.syncedVersion = Math.max(room.syncedVersion ?? 0, version) - await store.setSyncedVersion(name, version) + if (room && (generation === undefined || docIdOf(room.doc) === generation)) { + room.syncedVersion = Math.max(room.syncedVersion ?? 0, version) + } + await store.setSyncedVersion(name, version, generation) } // Order this merge on the file's version line, where `current` is the durable version the doc already @@ -723,11 +839,7 @@ async function mergeMarkdownIntoRoom( // always releases (or its lock expires) first and we acquire — never merging against a shared base // while a peer holds the lock. If somehow still unavailable, skip the live merge (copilot's durable // file write stands) rather than race. - let token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) - for (let i = 0; !token && i < MERGE_LOCK_RETRIES; i++) { - await sleep(MERGE_LOCK_RETRY_MS) - token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) - } + const token = await acquireFileDocMergeSlot(name) if (!token) { logger.warn(`Merge lock unavailable for file ${fileId}; skipping live merge`) return 'merge-unavailable' @@ -738,13 +850,14 @@ async function mergeMarkdownIntoRoom( const shared = await store.getSyncedVersion(name) const current = Math.max(shared ?? 0, fileDocRooms.get(name)?.syncedVersion ?? 0) if (isStale(current)) return 'stale' + const generation = await store.getDocumentGeneration(name) // Defer to an actively-streaming client: it is applying this SAME agent edit into the shared doc // frame-by-frame, so also publishing a whole-document merge here would double-write the content (the // client's private shadow never observes this merge, so it re-inserts what we added → duplication). // Still record the durable version so the persist If-Match stays correct; the client owns the bytes, // and once streaming stops the flag clears and the final durable merge lands as a near-noop. if (await store.isAgentStreaming(name)) { - await recordVersion() + await recordVersion(generation) return 'applied' } // Compute the diff against the committed SHARED state and PUBLISH it — every task with the doc @@ -752,11 +865,11 @@ async function mergeMarkdownIntoRoom( // merge reaches the live doc no matter which task the apply-edit call landed on. An empty stream // means no doc is (or was recently) live → nothing to merge into. AWAIT the publish so the diff is // durably in the stream before we release the lock (else the next task would diff a stale base). - const base = await store.getStreamState(name) + const base = await store.getStreamState(name, generation) if (!base) return 'no-live-room' const diff = await fetchFileDocMerge(fileId, base, markdown) - await store.publishAndWait(name, diff) - await recordVersion() + await store.publishAndWait(name, diff, generation) + await recordVersion(generation) return 'applied' } finally { await store.releaseMergeSlot(name, token) @@ -815,6 +928,7 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { agentStreamingUntil: 0, hydrated, pendingJoins: 0, + pendingUpdates: 0, } // Register synchronously BEFORE the async catch-up so a concurrent join sees this room, not a second. fileDocRooms.set(name, room) @@ -839,7 +953,8 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { origin !== REDIS_ORIGIN && origin !== REDIS_SNAPSHOT_ORIGIN && origin !== REDIS_AGENT_ORIGIN && - origin !== SEED_ORIGIN + origin !== SEED_ORIGIN && + !isClientUpdateOrigin(origin) ) getFileDocStore().publish(name, update, origin === AGENT_SYNC_ORIGIN) // A locally-originated agent frame (this task's stream leader) means a client is applying this agent @@ -979,10 +1094,24 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { const bytes = toFileDocBytes(data) if (!bytes) return + if (bytes.byteLength > MAX_LEGACY_FRAME_BYTES) { + logger.warn('Dropping an oversized legacy file-doc frame', { + socketId: socket.id, + bytes: bytes.byteLength, + }) + return + } // A malformed frame from any client must never escape as a process-level // exception; drop it and keep the relay running. try { + if (hasOversizedLegacyUpdate(bytes)) { + logger.warn('Dropping a legacy file-doc update outside the durable stream budget', { + socketId: socket.id, + bytes: bytes.byteLength, + }) + return + } const decoder = decoding.createDecoder(bytes) const messageType = decoding.readVarUint(decoder) @@ -1027,7 +1156,9 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { // owned by this socket. const owned = room.owners.get(socket.id) if (owned === undefined || awarenessUpdateClientIds(update).some((id) => !owned.has(id))) { - logger.warn('Dropping awareness frame for an unowned client id', { socketId: socket.id }) + logger.warn('Dropping awareness frame for an unowned client id', { + socketId: socket.id, + }) return } awarenessProtocol.applyAwarenessUpdate(room.awareness, update, socket.id) @@ -1037,7 +1168,112 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { logger.warn('Unknown file-doc message type', { messageType }) } } catch (error) { - logger.warn('Dropping malformed file-doc frame', { socketId: socket.id, error }) + logger.warn('Dropping malformed file-doc frame', { + socketId: socket.id, + error, + }) + } +} + +async function handleClientUpdate( + socket: AuthenticatedSocket, + io: Server, + data: unknown, + acknowledge: (result: FileDocUpdateAck) => void +): Promise { + const reject = ( + code: Extract['code'], + retryable: boolean, + updateId?: string + ) => acknowledge({ status: 'rejected', code, retryable, updateId }) + + if (typeof data !== 'object' || data === null) { + reject('INVALID_UPDATE', false) + return + } + + const candidate = data as Partial + const update = toFileDocBytes(candidate.update) + if ( + typeof candidate.fileId !== 'string' || + candidate.fileId.length === 0 || + typeof candidate.docId !== 'string' || + candidate.docId.length === 0 || + typeof candidate.updateId !== 'string' || + candidate.updateId.length === 0 || + candidate.updateId.length > MAX_CLIENT_UPDATE_ID_LENGTH || + !update || + update.byteLength === 0 || + update.byteLength > FILE_DOC_LIMITS.updateBytes + ) { + reject('INVALID_UPDATE', false, candidate.updateId) + return + } + + const name = socketToRoomName.get(socket.id) + if (!name || name !== roomName(fileDocRoom(candidate.fileId))) { + reject('NOT_JOINED', true, candidate.updateId) + return + } + const room = fileDocRooms.get(name) + if (!room) { + reject('NOT_JOINED', true, candidate.updateId) + return + } + if (!isFileDocWriteAllowed(socket, io, name)) { + reject('ACCESS_REVOKED', false, candidate.updateId) + return + } + if (docIdOf(room.doc) !== candidate.docId) { + reject('DOCUMENT_REPLACED', false, candidate.updateId) + return + } + + const validationDoc = new Y.Doc() + try { + Y.applyUpdate(validationDoc, update) + } catch (error) { + logger.warn('Dropping malformed acknowledged file-doc update', { + socketId: socket.id, + fileId: candidate.fileId, + updateId: candidate.updateId, + error, + }) + reject('INVALID_UPDATE', false, candidate.updateId) + return + } finally { + validationDoc.destroy() + } + + const editor = room.owners.get(socket.id)?.values().next().value?.userId + if (editor) room.lastEditorUserId = editor + room.pendingUpdates += 1 + try { + await getFileDocStore().publishClientUpdateAndWait( + name, + candidate.updateId, + update, + candidate.docId + ) + Y.applyUpdate(room.doc, update, clientUpdateOrigin(socket.id)) + room.edited = true + schedulePersist(name, room) + acknowledge({ status: 'accepted', updateId: candidate.updateId }) + } catch (error) { + if (error instanceof FileDocInvalidatedError) { + reject('DOCUMENT_REPLACED', false, candidate.updateId) + return + } + logger.error('Failed to accept acknowledged file-doc update', { + socketId: socket.id, + fileId: candidate.fileId, + updateId: candidate.updateId, + error, + }) + reject('TEMPORARY_FAILURE', true, candidate.updateId) + } finally { + room.pendingUpdates -= 1 + destroyRoomIfIdle(name) } } @@ -1103,7 +1339,8 @@ export function setupWorkspaceFileDocHandlers( // leave for a DIFFERENT file must NOT cancel it (a document switch), mirroring workspace-files. let currentFileId: string | null = null - socket.on(FILE_DOC_EVENTS.JOIN, async ({ fileId, clientId }: JoinFileDocPayload) => { + socket.on(FILE_DOC_EVENTS.JOIN, async (payload: JoinFileDocPayload) => { + const { fileId, clientId } = payload // Hoisted so the catch can tell whether this join was superseded (a switch to another file) // before surfacing a retryable error for the abandoned one. let generation: number | undefined @@ -1142,6 +1379,17 @@ export function setupWorkspaceFileDocHandlers( emitJoinError(socket, fileId, clientId, 'Invalid join payload', 'INVALID_PAYLOAD', false) return } + if ((payload.schemaVersion ?? FILE_DOC_LEGACY_SCHEMA_VERSION) !== FILE_DOC_SCHEMA_VERSION) { + emitJoinError( + socket, + fileId, + clientId, + 'This document version is not supported', + 'SCHEMA_VERSION_MISMATCH', + false + ) + return + } // A generation represents the socket's intended FILE, not an individual provider. Co-mounted // providers for the same file must be allowed to join concurrently; switching files advances the @@ -1177,6 +1425,16 @@ export function setupWorkspaceFileDocHandlers( // awareness). Resolved here so the generation guard below also covers this await. const avatarUrl = await resolveAvatarUrl(socket, userId) + const store = getFileDocStore() + const existing = fileDocRooms.get(name) + if ( + existing && + isDocSeeded(existing.doc) && + !(await store.isDocumentGenerationCurrent(name, docIdOf(existing.doc))) + ) { + discardInvalidatedRoom(name, io) + } + const entry = getOrCreateRoom(io, room) // The workspace the server-side persist writes back to — and what the seed is built from, so it // must be captured BEFORE the room is prepared below. @@ -1295,6 +1553,8 @@ export function setupWorkspaceFileDocHandlers( fileId, clientId, docId: docIdOf(entry.doc), + schemaVersion: FILE_DOC_SCHEMA_VERSION, + ...(store.enabled ? { acknowledgedUpdates: true as const } : {}), }) // Server-authenticated roster → everyone in the room, including this joiner. broadcastFileDocPresence(io, name, entry) @@ -1354,6 +1614,16 @@ export function setupWorkspaceFileDocHandlers( socket.on(FILE_DOC_EVENTS.MESSAGE, (data: unknown) => handleMessage(socket, io, data)) + socket.on( + FILE_DOC_EVENTS.UPDATE, + (data: unknown, acknowledge?: (result: FileDocUpdateAck) => void) => { + if (typeof acknowledge !== 'function') return + void handleClientUpdate(socket, io, data, acknowledge).catch((error) => { + logger.error('Unhandled acknowledged file-doc update failure:', error) + }) + } + ) + socket.on(FILE_DOC_EVENTS.LEAVE, (payload?: LeaveFileDocPayload) => { try { // Cancel an in-flight join whose file the client is now leaving (or an unscoped leave): a diff --git a/apps/realtime/src/routes/http.test.ts b/apps/realtime/src/routes/http.test.ts index 725341deac9..952d9ff75d5 100644 --- a/apps/realtime/src/routes/http.test.ts +++ b/apps/realtime/src/routes/http.test.ts @@ -1,6 +1,14 @@ import type { IncomingMessage, ServerResponse } from 'http' import { describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' + +const { mockInvalidateDocument } = vi.hoisted(() => ({ mockInvalidateDocument: vi.fn() })) + +vi.mock('@/handlers/file-doc', () => ({ + applyMarkdownToLiveFileDoc: vi.fn(), + invalidateLiveFileDocument: mockInvalidateDocument, +})) + import { createHttpHandler } from '@/routes/http' function createMocks(req: Partial) { @@ -11,6 +19,7 @@ function createMocks(req: Partial) { const roomManager = { getTotalActiveConnections: vi.fn().mockResolvedValue(0), isReady: vi.fn().mockReturnValue(true), + emitToRoom: vi.fn(), } as unknown as IRoomManager return { @@ -20,9 +29,25 @@ function createMocks(req: Partial) { setHeader, writeHead, end, + roomManager, } } +function requestWithBody(url: string, body: unknown): Partial { + const text = JSON.stringify(body) + const request = { + method: 'POST', + url, + headers: { 'x-api-key': 'test-internal-api-secret-at-least-32-chars' }, + on(event: string, callback: (value?: Buffer) => void) { + if (event === 'data') callback(Buffer.from(text)) + if (event === 'end') callback() + return request + }, + } + return request as unknown as Partial +} + describe('createHttpHandler', () => { /** * `/health` is the only route on this server that returns 200 with a body, so @@ -58,4 +83,42 @@ describe('createHttpHandler', () => { expect(writeHead).toHaveBeenCalledWith(200, { 'Content-Type': 'application/json' }) }) + + it('invalidates the shared generation before notifying every open editor', async () => { + mockInvalidateDocument.mockResolvedValueOnce('applied') + const { handler, req, res, writeHead, roomManager } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1', version: 100 }) + ) + + await handler(req, res) + + expect(mockInvalidateDocument).toHaveBeenCalledWith('file-1', 100) + expect(roomManager.emitToRoom).toHaveBeenCalledWith( + { type: 'workspace-file-doc', id: 'file-1' }, + 'file-doc-invalidated', + expect.objectContaining({ fileId: 'file-1' }) + ) + expect(mockInvalidateDocument.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(roomManager.emitToRoom).mock.invocationCallOrder[0] + ) + expect(writeHead).toHaveBeenCalledWith(200, { 'Content-Type': 'application/json' }) + }) + + it('does not evict editors for a superseded invalidation', async () => { + mockInvalidateDocument.mockResolvedValueOnce('stale') + const { handler, req, res, end, roomManager } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1', version: 100 }) + ) + await handler(req, res) + expect(roomManager.emitToRoom).not.toHaveBeenCalled() + expect(end).toHaveBeenCalledWith(JSON.stringify({ status: 'stale' })) + }) + + it('requires a durable version for invalidation', async () => { + const { handler, req, res, writeHead } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1' }) + ) + await handler(req, res) + expect(writeHead).toHaveBeenCalledWith(400, { 'Content-Type': 'application/json' }) + }) }) diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 19d2401e37e..aaff9839ce5 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -1,8 +1,9 @@ import type { IncomingMessage, ServerResponse } from 'http' -import { WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { FILE_DOC_EVENTS, type FileDocInvalidated } from '@sim/realtime-protocol/file-doc' +import { ROOM_TYPES, WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { safeCompare } from '@sim/security/compare' import { env } from '@/env' -import { applyMarkdownToLiveFileDoc } from '@/handlers/file-doc' +import { applyMarkdownToLiveFileDoc, invalidateLiveFileDocument } from '@/handlers/file-doc' import { type IRoomManager, WorkflowRoomService } from '@/rooms' interface Logger { @@ -207,7 +208,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { version: typeof version === 'number' ? version : undefined, }) res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ applied: result === 'applied' })) + res.end(JSON.stringify({ applied: result === 'applied', status: result })) } catch (error) { logger.error('Error applying copilot edit to live file-doc:', error) sendError(res, 'Failed to apply edit to live document') @@ -215,6 +216,30 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } + if (req.method === 'POST' && req.url === '/api/file-doc/invalidate') { + try { + const body = await readRequestBody(req) + const { fileId, version } = JSON.parse(body) + if (!isNonEmptyString(fileId)) return sendError(res, 'Invalid fileId', 400) + if (!Number.isSafeInteger(version) || version <= 0) { + return sendError(res, 'Invalid version', 400) + } + const room = { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: fileId } as const + const status = await invalidateLiveFileDocument(fileId, version) + const payload: FileDocInvalidated = { + fileId, + message: 'This file changed outside the editor. Reload to continue editing.', + } + if (status === 'applied') roomManager.emitToRoom(room, FILE_DOC_EVENTS.INVALIDATED, payload) + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ status })) + } catch (error) { + logger.error('Error invalidating live file-doc:', error) + sendError(res, 'Failed to invalidate live document') + } + return + } + res.writeHead(404, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ error: 'Not found' })) } diff --git a/apps/sim/app/api/webhooks/outbox/process/route.ts b/apps/sim/app/api/webhooks/outbox/process/route.ts index ea6746ff8f8..b4acb132c12 100644 --- a/apps/sim/app/api/webhooks/outbox/process/route.ts +++ b/apps/sim/app/api/webhooks/outbox/process/route.ts @@ -14,6 +14,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { directGrantOutboxHandlers } from '@/lib/invitations/direct-grant' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' +import { workspaceFileLiveDocOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' import { workspaceFileStorageCleanupOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox' import { workflowDeploymentOutboxHandlers } from '@/lib/workflows/deployment-outbox' import { invitationMigrationOutboxHandlers } from '@/lib/workspaces/admin-move' @@ -34,6 +35,7 @@ const handlers = { ...invitationMigrationOutboxHandlers, ...directGrantOutboxHandlers, ...knowledgeDocumentProcessingOutboxHandlers, + ...workspaceFileLiveDocOutboxHandlers, ...workspaceFileStorageCleanupOutboxHandlers, ...workflowDeploymentOutboxHandlers, } as const diff --git a/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx b/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx index 1a4c4db0f15..180dde64feb 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx @@ -10,6 +10,7 @@ import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/emcn', () => ({ + cn: (...classes: Array) => classes.filter(Boolean).join(' '), Button: ({ children, ...props }: { children: ReactNode } & Record) => ( - ) : undefined - } - /> - {/* Always mounted, reserving its width: rendering it only once there is a +
+
+ {replace && ( + + )} + onQueryChange(e.target.value)} + onKeyDown={handleKeyDown} + /** Whitespace may not match, but the user must still be able to clear it. */ + endAdornment={ + query.length > 0 ? ( + + ) : undefined + } + /> + {/* Always mounted, reserving its width: rendering it only once there is a query would resize the bar on the first keystroke, and a live region inserted together with its text is announced unreliably. */} - - {counterContent()} - - - - + + {counterContent()} + + + + +
+ {replace && showReplace && ( +
+ + replace.onChange(event.target.value)} + onKeyDown={(event) => { + if (event.nativeEvent.isComposing || event.keyCode === 229) return + if (event.key === 'Enter' && replace.canReplace) { + event.preventDefault() + replace.onReplace() + } else if (event.key === 'Escape') { + event.preventDefault() + onClose() + } + }} + /> + + +
+ )}
) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index 7d420e6da3f..8640f6f88ca 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -3,17 +3,37 @@ */ import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, + FILE_DOC_TIMEOUTS, + type FileDocUpdateAck, } from '@sim/realtime-protocol/file-doc' +import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' import type { Socket } from 'socket.io-client' import { describe, expect, it, vi } from 'vitest' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' -import { AGENT_STREAM_ORIGIN } from './apply-streamed-markdown' -import { FileDocProvider } from './file-doc-provider' +import { AGENT_STREAM_ORIGIN } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown' +import { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' +import { PendingFileDocUpdateJournal } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' + +const journalStorage = vi.hoisted(() => new Map()) + +vi.mock('idb-keyval', () => ({ + get: vi.fn((key: string) => journalStorage.get(key)), + update: vi.fn((key: string, updater: (value: unknown) => unknown) => { + journalStorage.set(key, updater(journalStorage.get(key))) + }), + del: vi.fn((key: string) => { + journalStorage.delete(key) + }), +})) + +const UPDATE_BATCH_TEST_WINDOW_MS = 100 /** A minimal fake Socket.IO client whose server→client events can be fired in tests. */ function createSocket(connected = true) { @@ -53,9 +73,15 @@ function createProvider(connected = true) { function acceptJoin( fire: (event: string, ...args: unknown[]) => void, clientId: number, - docId?: string + docId?: string, + acknowledgedUpdates = true ) { - fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1', clientId, docId }) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId, + docId, + acknowledgedUpdates: acknowledgedUpdates ? true : undefined, + }) } /** Messages emitted to the server, decoded to their `{ type, bytes }`. */ @@ -67,12 +93,20 @@ function emittedMessages(emit: ReturnType) { .map(([, payload]) => payload as Uint8Array) } +function syncStep1Frame(doc: Y.Doc): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep1(encoder, doc) + return encoding.toUint8Array(encoder) +} + describe('FileDocProvider', () => { it('joins immediately with its client id when the socket is already connected', () => { const { doc, emit } = createProvider(true) expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, { fileId: 'file-1', clientId: doc.clientID, + schemaVersion: FILE_DOC_SCHEMA_VERSION, }) }) @@ -206,6 +240,17 @@ describe('FileDocProvider', () => { expect(joinError).toHaveBeenCalledTimes(1) }) + it('fails closed when a seeded legacy tab has no identity but the server does', () => { + const { provider, doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + emit.mockClear() + + acceptJoin(fire, doc.clientID, 'doc-current') + + expect(emittedMessages(emit)).toHaveLength(0) + expect(provider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED', retryable: false }) + }) + it('syncs when the room holds the document it already has', () => { const { doc, emit, fire } = createProvider(true) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-original') @@ -247,16 +292,698 @@ describe('FileDocProvider', () => { expect(synced).toHaveBeenCalledWith(true) }) - it('sends local document edits to the server as sync updates', () => { + it('routes local differences through the acknowledged channel instead of the sync handshake', async () => { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getText('default').insert(0, 'local') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc) + ) + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + + const syncReplies = emittedMessages(emit).filter((message) => { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + return decoding.readVarUint(decoder) === syncProtocol.messageYjsSyncStep2 + }) + expect(syncReplies).toHaveLength(0) + await vi.waitFor(() => { + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(true) + }) + const updatePayload = emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.UPDATE + )?.[1] as { + update: Uint8Array + } + const serverDoc = new Y.Doc() + Y.applyUpdate(serverDoc, updatePayload.update) + expect(serverDoc.getText('default').toString()).toBe('local') + serverDoc.destroy() + provider.destroy() + }) + + it('keeps standard Yjs sync behavior with an older relay during a rolling deployment', () => { const { doc, emit, fire } = createProvider(true) - acceptJoin(fire, doc.clientID) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getText('default').insert(0, 'local') + acceptJoin(fire, doc.clientID, 'doc-1', false) emit.mockClear() - doc.getText('default').insert(0, 'x') + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + doc.getText('default').insert(5, ' edit') const messages = emittedMessages(emit) - expect(messages.length).toBe(1) - expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + expect(messages.length).toBeGreaterThanOrEqual(2) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + }) + + it('does not enable acknowledged updates unless the relay also supplies a document identity', () => { + const { doc, emit, fire } = createProvider(true) + doc.getText('default').insert(0, 'local') + acceptJoin(fire, doc.clientID, undefined, true) + emit.mockClear() + + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + + expect(emittedMessages(emit).length).toBeGreaterThan(0) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + }) + + it('recovers an unacknowledged edit after a tab restart and clears it only after acceptance', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const serverDoc = new Y.Doc() + const serverConfig = serverDoc.getMap(FILE_DOC_SEED.configMap) + serverConfig.set(FILE_DOC_SEED.docIdKey, 'doc-1') + serverConfig.set(FILE_DOC_SEED.flag, true) + serverDoc.getText('default').insert(0, 'base') + + const firstSocket = createSocket(true) + const firstDoc = new Y.Doc() + Y.applyUpdate(firstDoc, Y.encodeStateAsUpdate(serverDoc)) + const firstProvider = new FileDocProvider( + firstSocket.socket, + 'file-1', + firstDoc, + new awarenessProtocol.Awareness(firstDoc), + scope + ) + acceptJoin(firstSocket.fire, firstDoc.clientID, 'doc-1') + firstSocket.emit.mockClear() + firstDoc.getText('default').insert(4, ' local') + await vi.waitFor(() => { + expect(firstSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe( + true + ) + }) + firstProvider.destroy() + + const secondSocket = createSocket(true) + const secondDoc = new Y.Doc() + const secondProvider = new FileDocProvider( + secondSocket.socket, + 'file-1', + secondDoc, + new awarenessProtocol.Awareness(secondDoc), + scope + ) + acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') + const syncEncoder = encoding.createEncoder() + encoding.writeVarUint(syncEncoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(syncEncoder, serverDoc) + secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(syncEncoder)) + + await vi.waitFor(() => { + expect(secondDoc.getText('default').toString()).toBe('base local') + expect(secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe( + true + ) + }) + const updateCall = secondSocket.emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.UPDATE + ) + const payload = updateCall?.[1] as { updateId: string } + const acknowledge = updateCall?.[2] as (ack: FileDocUpdateAck) => void + Y.applyUpdate(serverDoc, (updateCall?.[1] as { update: Uint8Array }).update) + acknowledge({ status: 'accepted', updateId: payload.updateId }) + + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + await expect(journal.load()).resolves.toBeNull() + }) + + vi.useFakeTimers() + try { + secondSocket.fire('disconnect') + secondSocket.fire('connect') + secondSocket.emit.mockClear() + acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') + secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe( + false + ) + } finally { + vi.useRealTimers() + } + secondProvider.destroy() + serverDoc.destroy() + }) + + it('batches local document edits into the acknowledged update channel', async () => { + const { doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + doc.getText('default').insert(0, 'x') + + await vi.waitFor(() => { + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.UPDATE, + expect.objectContaining({ fileId: 'file-1', docId: 'doc-1' }), + expect.any(Function) + ) + }) + expect(emittedMessages(emit)).toHaveLength(0) + }) + + it('serializes journal flushes so an edit made during storage never becomes stranded', async () => { + vi.useFakeTimers() + const firstSave = Promise.withResolvers<{ + pendingUpdate: Uint8Array + status: 'saved' + }>() + let saveCalls = 0 + let firstPendingUpdate: Uint8Array | null = null + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => { + saveCalls += 1 + if (saveCalls === 1) { + firstPendingUpdate = pendingUpdate + return firstSave.promise + } + return { pendingUpdate, status: 'saved' } + }) + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + { workspaceId: 'workspace-1', userId: 'user-1' } + ) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + emit.mockClear() + + doc.getText('default').insert(0, 'first') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + doc.getText('default').insert(5, ' second') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(save).toHaveBeenCalledOnce() + + firstSave.resolve({ + pendingUpdate: firstPendingUpdate!, + status: 'saved', + }) + await vi.advanceTimersByTimeAsync(0) + const firstUpdate = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const firstPayload = firstUpdate?.[1] as { updateId: string } + const acknowledge = firstUpdate?.[2] as (ack: FileDocUpdateAck) => void + acknowledge({ status: 'accepted', updateId: firstPayload.updateId }) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + + expect(save).toHaveBeenCalledTimes(2) + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toHaveLength(2) + provider.destroy() + } finally { + save.mockRestore() + vi.useRealTimers() + } + }) + + it('retries an unacknowledged update with the same idempotency key', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + doc.getText('default').insert(0, 'kept until acknowledged') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(first).toBeDefined() + + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 2_000) + const updates = emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(updates.length).toBeGreaterThan(1) + expect((updates[1][1] as { updateId: string }).updateId).toBe( + (first?.[1] as { updateId: string }).updateId + ) + provider.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('rejoins before retrying an update rejected because the room membership went stale', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + doc.getText('default').insert(0, 'edit') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const payload = first?.[1] as { updateId: string } + const acknowledge = first?.[2] as (ack: FileDocUpdateAck) => void + + acknowledge({ + status: 'rejected', + updateId: payload.updateId, + code: 'NOT_JOINED', + retryable: true, + }) + await vi.advanceTimersByTimeAsync(1_000) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.JOIN)).toBe(true) + + emit.mockClear() + acceptJoin(fire, doc.clientID, 'doc-1') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(true) + provider.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('journals an edit made while disconnected before page teardown', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + fire('disconnect') + + doc.getText('default').insert(0, 'offline edit') + + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + await expect(journal.load('doc-1')).resolves.not.toBeNull() + }) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + }) + + it('does not recreate a discarded recovery record during page teardown or destroy', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + fire('disconnect') + doc.getText('default').insert(0, 'discard me') + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => expect(await journal.load('doc-1')).not.toBeNull()) + + await provider.discardPendingChanges() + ;(provider as unknown as { handlePageHide: () => void }).handlePageHide() + provider.destroy() + + await expect(journal.load('doc-1')).resolves.toBeNull() + }) + + it('hydrates the complete local draft before reporting a replaced document', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const oldDoc = new Y.Doc() + oldDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'old-doc') + oldDoc.getText('default').insert(0, 'complete draft') + const recoverySnapshot = Y.encodeStateAsUpdate(oldDoc) + const stateVector = Y.encodeStateVector(oldDoc) + oldDoc.getText('default').insert('complete draft'.length, ' plus pending') + const pendingUpdate = Y.encodeStateAsUpdate(oldDoc, stateVector) + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'old-doc', + pendingUpdate, + recoverySnapshot + ) + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + + acceptJoin(fire, doc.clientID, 'current-doc') + + await vi.waitFor(() => expect(provider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED' })) + expect(doc.getText('default').toString()).toBe('complete draft plus pending') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + oldDoc.destroy() + }) + + it('never falls back to a different document identity when loading local recovery', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const oldDoc = new Y.Doc() + oldDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-old') + oldDoc.getText('default').insert(0, 'old draft') + const oldSnapshot = Y.encodeStateAsUpdate(oldDoc) + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'doc-old', + oldSnapshot, + oldSnapshot + ) + + const { socket, fire } = createSocket(true) + const currentDoc = new Y.Doc() + currentDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-current') + currentDoc.getText('default').insert(0, 'current content') + const provider = new FileDocProvider( + socket, + 'file-1', + currentDoc, + new awarenessProtocol.Awareness(currentDoc), + scope + ) + acceptJoin(fire, currentDoc.clientID, 'doc-current') + + await vi.waitFor(() => expect(provider.joinError).toBeNull()) + expect(currentDoc.getText('default').toString()).toBe('current content') + await expect( + new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-old') + ).resolves.not.toBeNull() + provider.destroy() + currentDoc.destroy() + oldDoc.destroy() + }) + + it('fails terminally without partially applying a malformed local recovery record', async () => { + const load = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'load').mockResolvedValue({ + docId: 'doc-1', + recoverySnapshot: null, + pendingUpdate: new Uint8Array([255]), + updatedAt: Date.now(), + }) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const discard = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'discard').mockResolvedValue() + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + + await vi.waitFor(() => expect(provider.joinError).toMatchObject({ code: 'INVALID_UPDATE' })) + expect(doc.getText('default').toString()).toBe('') + await provider.discardPendingChanges() + expect(discard).toHaveBeenCalledWith('doc-1') + provider.destroy() + doc.destroy() + load.mockRestore() + discard.mockRestore() + }) + + it('ignores an obsolete schema rejection when recovery finishes after reconnecting', async () => { + const recovery = Promise.withResolvers() + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(recovery.promise) + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, { + workspaceId: 'workspace-1', + userId: 'user-1', + }) + try { + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + schemaVersion: FILE_DOC_SCHEMA_VERSION + 1, + }) + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID) + emit.mockClear() + recovery.resolve(null) + + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + expect(provider.joinError).toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + load.mockRestore() + } + }) + + it('fails closed when hydration buffers more than its bounded message count', async () => { + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(new Promise(() => {})) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + const message = syncStep1Frame(new Y.Doc()) + + for (let index = 0; index < 129; index += 1) { + fire(FILE_DOC_EVENTS.MESSAGE, message) + } + + expect(provider.joinError).toMatchObject({ code: 'HYDRATION_BUFFER_OVERFLOW' }) + provider.destroy() + load.mockRestore() + }) + + it('fails closed when hydration buffers more than its bounded byte budget', () => { + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(new Promise(() => {})) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + + fire(FILE_DOC_EVENTS.MESSAGE, new Uint8Array(FILE_DOC_LIMITS.updateBytes * 2 + 1)) + + expect(provider.joinError).toMatchObject({ code: 'HYDRATION_BUFFER_OVERFLOW' }) + provider.destroy() + doc.destroy() + load.mockRestore() + }) + + it('makes an older different-file provider terminal before unscoped frames can cross documents', async () => { + const { socket, emit, fire } = createSocket(true) + const firstDoc = new Y.Doc() + const firstProvider = new FileDocProvider( + socket, + 'file-1', + firstDoc, + new awarenessProtocol.Awareness(firstDoc) + ) + acceptJoin(fire, firstDoc.clientID) + + const secondDoc = new Y.Doc() + const secondProvider = new FileDocProvider( + socket, + 'file-2', + secondDoc, + new awarenessProtocol.Awareness(secondDoc) + ) + expect(firstProvider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED' }) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-2', + clientId: secondDoc.clientID, + acknowledgedUpdates: true, + }) + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + + const remote = new Y.Doc() + remote.getText('default').insert(0, 'second-file content') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(remote)) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + expect(firstDoc.getText('default').toString()).toBe('') + expect(secondDoc.getText('default').toString()).toBe('second-file content') + firstProvider.destroy() + secondProvider.destroy() + firstDoc.destroy() + secondDoc.destroy() + remote.destroy() + }) + + it('stops editing while the complete local recovery snapshot cannot be persisted', async () => { + vi.useFakeTimers() + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => ({ + pendingUpdate, + status: 'limit-exceeded', + })) + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + { workspaceId: 'workspace-1', userId: 'user-1' } + ) + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + doc.getText('default').insert(0, 'must remain downloadable') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + + expect(provider.joinError).toMatchObject({ code: 'PENDING_UPDATE_LIMIT' }) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + } finally { + save.mockRestore() + vi.useRealTimers() + } + }) + + it.each(['saved', 'unavailable'] as const)( + 'warns before unloading pending edits and continues acknowledged saves when storage is %s', + async (status) => { + vi.useFakeTimers() + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => ({ pendingUpdate, status })) + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, { + workspaceId: 'workspace-1', + userId: 'user-1', + }) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + expect(unloadIsPrevented()).toBe(false) + + doc.getText('default').insert(0, 'pending edit') + expect(unloadIsPrevented()).toBe(true) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(provider.joinError).toBeNull() + const update = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(update).toBeDefined() + const payload = update?.[1] as { updateId: string } + const acknowledge = update?.[2] as (ack: FileDocUpdateAck) => void + expect(unloadIsPrevented()).toBe(true) + acknowledge({ status: 'accepted', updateId: payload.updateId }) + expect(unloadIsPrevented()).toBe(false) + + doc.getText('default').insert(0, 'another ') + expect(unloadIsPrevented()).toBe(true) + provider.destroy() + expect(unloadIsPrevented()).toBe(false) + awareness.destroy() + doc.destroy() + } finally { + save.mockRestore() + vi.unstubAllGlobals() + vi.useRealTimers() + } + } + ) + + it('keeps retrying sync without fatally timing out a previously healthy reconnect', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + acceptJoin(fire, doc.clientID, 'doc-1') + const serverDoc = new Y.Doc() + const config = serverDoc.getMap(FILE_DOC_SEED.configMap) + config.set(FILE_DOC_SEED.docIdKey, 'doc-1') + config.set(FILE_DOC_SEED.flag, true) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + + emit.mockClear() + fire('disconnect') + fire('connect') + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.JOIN)).toHaveLength(1) + acceptJoin(fire, doc.clientID, 'doc-1') + + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.readinessDeadlineMs) + expect(provider.joinError).toBeNull() + expect(emittedMessages(emit).length).toBeGreaterThan(1) + provider.destroy() + serverDoc.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('retries an accepted sync handshake that never receives a response', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + acceptJoin(fire, doc.clientID) + emit.mockClear() + + await vi.advanceTimersByTimeAsync(6_000) + + expect(emittedMessages(emit).length).toBeGreaterThan(1) + expect(provider.joinError).toBeNull() + provider.destroy() + } finally { + vi.useRealTimers() + } }) it('tags agent-streamed edits as SYNC_NO_PERSIST so the relay skips the durable persist', () => { @@ -359,6 +1086,31 @@ describe('FileDocProvider', () => { expect(provider.joinError).toEqual(error) }) + it('becomes terminal when a durable replacement invalidates its document generation', () => { + const { provider, doc, emit, fire } = createProvider(true) + const onError = vi.fn() + provider.on('join-error', onError) + + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + message: 'This file changed outside the editor. Reload to continue editing.', + }) + + expect(provider.joinError).toMatchObject({ + code: 'DOCUMENT_REPLACED', + retryable: false, + }) + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ code: 'DOCUMENT_REPLACED', retryable: false }) + ) + + emit.mockClear() + fire('connect') + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + expect(doc.getText('default').toString()).toBe('') + }) + it('scopes join errors to the matching provider on a shared socket', () => { const { socket, fire } = createSocket(true) const firstDoc = new Y.Doc() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index b3a04d50b96..f1cce26a7bc 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -4,14 +4,20 @@ import { } from '@sim/realtime-protocol/events' import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, FILE_DOC_TIMEOUTS, + type FileDocInvalidated, + type FileDocUpdateAck, + type FileDocUpdatePayload, type JoinFileDocError, type JoinFileDocSuccess, toFileDocBytes, } from '@sim/realtime-protocol/file-doc' import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { generateShortId } from '@sim/utils/id' import { backoffWithJitter } from '@sim/utils/retry' import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' @@ -19,8 +25,9 @@ import { ObservableV2 } from 'lib0/observable' import type { Socket } from 'socket.io-client' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' -import type * as Y from 'yjs' -import { AGENT_STREAM_ORIGIN } from './apply-streamed-markdown' +import * as Y from 'yjs' +import { AGENT_STREAM_ORIGIN } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown' +import { PendingFileDocUpdateJournal } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' /** * Events emitted by {@link FileDocProvider}. @@ -48,6 +55,27 @@ interface FileDocProviderEvents { const READINESS_DEADLINE_MS = FILE_DOC_TIMEOUTS.readinessDeadlineMs const JOIN_RETRY_BASE_MS = 500 const JOIN_RETRY_MAX_MS = 5_000 +const UPDATE_BATCH_MS = 50 +const UPDATE_RETRY_BASE_MS = 250 +const UPDATE_RETRY_MAX_MS = 5_000 +const MAX_HYDRATION_MESSAGES = 128 +const MAX_HYDRATION_BYTES = FILE_DOC_LIMITS.updateBytes * 2 +const RECOVERY_ORIGIN = Symbol('file-doc-recovery') + +function hasYjsUpdateContent(update: Uint8Array): boolean { + const decoded = Y.decodeUpdate(update) + return decoded.structs.length > 0 || decoded.ds.clients.size > 0 +} + +interface FileDocProviderScope { + workspaceId: string + userId: string +} + +interface PendingClientUpdate { + updateId: string + update: Uint8Array +} /** * Live-provider counts per file, per shared socket. Two surfaces in one tab (the Files editor and the @@ -101,6 +129,13 @@ function releaseRoomMembership(socket: Socket, fileId: string): boolean { * reconnect) without discarding local edits. */ export class FileDocProvider extends ObservableV2 { + /** Socket.IO carries unscoped Yjs frames, so opening a different file terminalizes providers for the + * previous file; multiple providers for the same file may coexist. */ + private static readonly activeProviders = new WeakMap< + Socket, + { fileId: string; providers: Set } + >() + synced = false /** * The latched non-retryable join rejection, or `null`. The `join-error` event is @@ -116,18 +151,45 @@ export class FileDocProvider extends ObservableV2 { /** Deadline for reaching readiness (synced + seeded); fires the fallback if it is never reached. */ private readinessTimer: ReturnType | null = null private joinAccepted = false + private acknowledgedUpdates = false private joinPending = false private joinRetryAttempt = 0 private joinRetryTimer: ReturnType | null = null + private joinAckTimer: ReturnType | null = null + private syncRetryTimer: ReturnType | null = null + private syncRetryAttempt = 0 + private joinHydrating = false + private connectionGeneration = 0 + private bufferedMessages: Uint8Array[] = [] + private bufferedMessageBytes = 0 + private pendingUpdateBatch: Uint8Array[] = [] + private inFlightUpdate: PendingClientUpdate | null = null + private updateBatchTimer: ReturnType | null = null + private updateAckTimer: ReturnType | null = null + private updateRetryTimer: ReturnType | null = null + private updateRetryAttempt = 0 + private updateFlushInProgress = false + private recoveryApplied = false + private recoveryQueued = false + private recoveryDocId: string | null = null + private pendingChangesDiscarded = false + private beforeUnloadProtected = false + private readonly journal: PendingFileDocUpdateJournal | null + private readonly journalLoad: ReturnType constructor( private readonly socket: Socket, private readonly fileId: string, readonly doc: Y.Doc, - readonly awareness: awarenessProtocol.Awareness + readonly awareness: awarenessProtocol.Awareness, + scope?: FileDocProviderScope ) { super() + this.journal = scope ? new PendingFileDocUpdateJournal({ ...scope, fileId: this.fileId }) : null + this.journalLoad = this.journal?.load(this.docId()) ?? Promise.resolve(null) + this.registerActiveProvider() + // Restore an empty local awareness state if it has been cleared. A fresh // Awareness starts with `{}`, but a *reused* one whose local state was removed // (a prior provider's `destroy()` clears it, and so does `Awareness.destroy()`) @@ -143,11 +205,13 @@ export class FileDocProvider extends ObservableV2 { socket.on(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) socket.on(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) socket.on(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + socket.on(FILE_DOC_EVENTS.INVALIDATED, this.handleInvalidated) socket.on(ROOM_ACCESS_REVOKED_EVENT, this.handleAccessRevoked) socket.on('connect', this.handleConnect) socket.on('disconnect', this.handleDisconnect) doc.on('update', this.handleDocUpdate) awareness.on('update', this.handleAwarenessUpdate) + if (typeof window !== 'undefined') window.addEventListener('pagehide', this.handlePageHide) // Watch the seed flag so reaching "seeded" (server seed applied) can clear the readiness deadline. doc.getMap(FILE_DOC_SEED.configMap).observe(this.handleConfigChange) @@ -158,7 +222,7 @@ export class FileDocProvider extends ObservableV2 { if (socket.connected) this.join() // Arm the fallback: if we don't reach readiness (synced + seeded) before the deadline, give up. - this.readinessTimer = setTimeout(this.handleReadinessDeadline, READINESS_DEADLINE_MS) + this.armReadinessDeadline() } /** Whether the server seed has recorded the initial content on the doc. */ @@ -169,6 +233,9 @@ export class FileDocProvider extends ObservableV2 { /** Clear the readiness deadline once the editor is usable (synced AND seeded). */ private handleConfigChange = () => { if (this.synced && this.isSeeded()) this.clearReadinessTimer() + if (this.acknowledgedUpdates && this.docId() && this.pendingUpdateBatch.length > 0) { + this.scheduleUpdateFlush(0) + } } /** @@ -196,6 +263,11 @@ export class FileDocProvider extends ObservableV2 { } } + private armReadinessDeadline() { + this.clearReadinessTimer() + this.readinessTimer = setTimeout(this.handleReadinessDeadline, READINESS_DEADLINE_MS) + } + private clearJoinRetryTimer() { if (this.joinRetryTimer !== null) { clearTimeout(this.joinRetryTimer) @@ -203,11 +275,47 @@ export class FileDocProvider extends ObservableV2 { } } + private clearJoinAckTimer() { + if (this.joinAckTimer !== null) { + clearTimeout(this.joinAckTimer) + this.joinAckTimer = null + } + } + + private clearSyncRetryTimer() { + if (this.syncRetryTimer !== null) { + clearTimeout(this.syncRetryTimer) + this.syncRetryTimer = null + } + } + + private clearUpdateTimers() { + if (this.updateBatchTimer !== null) clearTimeout(this.updateBatchTimer) + if (this.updateAckTimer !== null) clearTimeout(this.updateAckTimer) + if (this.updateRetryTimer !== null) clearTimeout(this.updateRetryTimer) + this.updateBatchTimer = null + this.updateAckTimer = null + this.updateRetryTimer = null + } + /** Join the room, binding our client id so the server only accepts awareness we own. */ private join = () => { if (this.fatal || this.disposed || !this.socket.connected || this.joinPending) return this.joinPending = true - this.socket.emit(FILE_DOC_EVENTS.JOIN, { fileId: this.fileId, clientId: this.doc.clientID }) + this.clearJoinAckTimer() + this.joinAckTimer = setTimeout(() => { + this.joinAckTimer = null + if (!this.joinPending || this.fatal || this.disposed) return + this.joinPending = false + this.joinAccepted = false + this.setSynced(false) + this.scheduleJoinRetry() + }, FILE_DOC_TIMEOUTS.joinAckMs) + this.socket.emit(FILE_DOC_EVENTS.JOIN, { + fileId: this.fileId, + clientId: this.doc.clientID, + schemaVersion: FILE_DOC_SCHEMA_VERSION, + }) } private scheduleJoinRetry() { @@ -232,7 +340,10 @@ export class FileDocProvider extends ObservableV2 { */ private handleConnect = () => { if (this.fatal) return + this.connectionGeneration += 1 this.clearJoinRetryTimer() + this.clearSyncRetryTimer() + this.syncRetryAttempt = 0 this.joinAccepted = false this.joinPending = false this.joinRetryAttempt = 0 @@ -241,8 +352,17 @@ export class FileDocProvider extends ObservableV2 { } private handleDisconnect = () => { + this.connectionGeneration += 1 + this.clearBufferedMessages() this.clearJoinRetryTimer() + this.clearJoinAckTimer() + this.clearSyncRetryTimer() + if (this.updateAckTimer !== null) clearTimeout(this.updateAckTimer) + if (this.updateRetryTimer !== null) clearTimeout(this.updateRetryTimer) + this.updateAckTimer = null + this.updateRetryTimer = null this.joinAccepted = false + this.joinHydrating = false this.joinPending = false this.setSynced(false) @@ -277,20 +397,108 @@ export class FileDocProvider extends ObservableV2 { (data.clientId !== undefined && data.clientId !== this.doc.clientID) ) return + this.clearJoinAckTimer() this.joinPending = false this.joinRetryAttempt = 0 this.clearJoinRetryTimer() + this.acknowledgedUpdates = data.acknowledgedUpdates === true && data.docId !== undefined + this.joinHydrating = true + const generation = this.connectionGeneration + if (!this.journal) { + this.finishAcceptJoin(data, generation, null) + return + } + void this.journalLoad.then((recovered) => { + this.finishAcceptJoin(data, generation, recovered) + }) + } + + private finishAcceptJoin( + data: JoinFileDocSuccess, + generation: number, + recovered: Awaited> + ): void { + if ( + this.disposed || + this.fatal || + !this.socket.connected || + generation !== this.connectionGeneration || + !this.joinHydrating + ) + return + + const serverSchemaVersion = data.schemaVersion ?? 1 + if (serverSchemaVersion !== FILE_DOC_SCHEMA_VERSION) { + this.failFatally( + 'This document version is not supported; refresh to continue editing', + 'SCHEMA_VERSION_MISMATCH' + ) + return + } + + if (recovered !== null && !this.recoveryApplied) { + this.recoveryDocId = recovered.docId + const validationDoc = new Y.Doc() + try { + if (recovered.recoverySnapshot) { + Y.applyUpdate(validationDoc, recovered.recoverySnapshot) + } + Y.applyUpdate(validationDoc, recovered.pendingUpdate) + } catch { + this.failFatally( + 'The local recovery copy is damaged. Download the current draft before discarding it.', + 'INVALID_UPDATE' + ) + return + } finally { + validationDoc.destroy() + } + try { + if (recovered.recoverySnapshot) { + Y.applyUpdate(this.doc, recovered.recoverySnapshot, RECOVERY_ORIGIN) + } + Y.applyUpdate(this.doc, recovered.pendingUpdate, RECOVERY_ORIGIN) + } catch { + this.failFatally( + 'The local recovery copy could not be restored. Download the current draft before discarding it.', + 'INVALID_UPDATE' + ) + return + } + this.recoveryApplied = true + } + const local = this.docId() - if (local !== undefined && data.docId !== undefined && data.docId !== local) { + if ( + (data.docId !== undefined && + ((local !== undefined && data.docId !== local) || + (local === undefined && this.isSeeded()))) || + (recovered !== null && data.docId !== recovered.docId) + ) { this.failFatally( 'This document was reloaded on the server; refresh to continue editing', 'DOCUMENT_REPLACED' ) return } + + if (recovered !== null && this.acknowledgedUpdates && !this.recoveryQueued) { + this.queuePendingUpdate(recovered.pendingUpdate) + this.recoveryQueued = true + } + + this.joinHydrating = false this.joinAccepted = true this.sendSyncStep1() + this.scheduleSyncRetry() this.sendLocalAwareness() + const bufferedMessages = this.bufferedMessages + this.clearBufferedMessages() + for (const message of bufferedMessages) this.applyMessage(message) + if (this.acknowledgedUpdates) { + if (this.inFlightUpdate) this.sendInFlightUpdate() + else if (this.pendingUpdateBatch.length > 0) this.scheduleUpdateFlush(0) + } } /** The identity of the document we hold, once the server seed has named one. */ @@ -313,15 +521,48 @@ export class FileDocProvider extends ObservableV2 { retryable: false, } this.fatal = true + this.clearBufferedMessages() this.joinError = error + void this.persistPendingSnapshot() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearUpdateTimers() + this.clearSyncRetryTimer() this.joinAccepted = false this.joinPending = false + this.joinHydrating = false + this.clearJoinAckTimer() this.setSynced(false) this.emit('join-error', [error]) } + private registerActiveProvider(): void { + const active = FileDocProvider.activeProviders.get(this.socket) + if (active?.fileId === this.fileId) { + active.providers.add(this) + return + } + if (active) { + for (const provider of active.providers) { + provider.failFatally( + 'Another file was opened in this tab. Reload this file to resume editing it.', + 'DOCUMENT_REPLACED' + ) + } + } + FileDocProvider.activeProviders.set(this.socket, { + fileId: this.fileId, + providers: new Set([this]), + }) + } + + private unregisterActiveProvider(): void { + const active = FileDocProvider.activeProviders.get(this.socket) + if (active?.fileId !== this.fileId) return + active.providers.delete(this) + if (active.providers.size === 0) FileDocProvider.activeProviders.delete(this.socket) + } + /** * Handle a join rejection. A non-retryable rejection (access denied, invalid) * won't succeed on retry, so latch {@link fatal} to stop (re)joining and let the @@ -336,11 +577,16 @@ export class FileDocProvider extends ObservableV2 { return this.joinAccepted = false this.joinPending = false + this.joinHydrating = false + this.clearJoinAckTimer() if (data.retryable === false) { this.fatal = true this.joinError = data + void this.persistPendingSnapshot() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearUpdateTimers() + this.clearSyncRetryTimer() this.setSynced(false) } else { this.setSynced(false) @@ -362,6 +608,11 @@ export class FileDocProvider extends ObservableV2 { this.failFatally(data.message, 'ACCESS_REVOKED') } + private handleInvalidated = (data: FileDocInvalidated) => { + if (data.fileId !== this.fileId) return + this.failFatally(data.message, 'DOCUMENT_REPLACED') + } + private handleMessage = (data: unknown) => { // Once we've given up (a non-retryable rejection, or the connect deadline lapsed and the editor // fell back to a read-only local seed), ignore ALL inbound frames. A late SyncStep2 arriving @@ -369,7 +620,30 @@ export class FileDocProvider extends ObservableV2 { // duplicating content — and flip `synced` true, which un-gates autosave and would persist the // duplicate back to the real file. `fatal` guarding (re)join alone is not enough; it must also // stop applying sync here. - if (this.fatal || !this.joinAccepted) return + if (this.fatal) return + if (this.joinHydrating) { + const bytes = toFileDocBytes(data) + if (!bytes) return + if ( + this.bufferedMessages.length >= MAX_HYDRATION_MESSAGES || + this.bufferedMessageBytes + bytes.byteLength > MAX_HYDRATION_BYTES + ) { + this.failFatally( + 'Realtime document hydration exceeded its safety limit', + 'HYDRATION_BUFFER_OVERFLOW' + ) + return + } + const buffered = new Uint8Array(bytes) + this.bufferedMessages.push(buffered) + this.bufferedMessageBytes += buffered.byteLength + return + } + if (!this.joinAccepted) return + this.applyMessage(data) + } + + private applyMessage(data: unknown) { const bytes = toFileDocBytes(data) if (!bytes) return @@ -384,7 +658,19 @@ export class FileDocProvider extends ObservableV2 { // re-sending updates we just applied from the server. const syncType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this) if (encoding.length(encoder) > 1) { - this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + const response = encoding.toUint8Array(encoder) + if (this.acknowledgedUpdates && syncType === syncProtocol.messageYjsSyncStep1) { + const responseDecoder = decoding.createDecoder(response) + decoding.readVarUint(responseDecoder) + decoding.readVarUint(responseDecoder) + const update = new Uint8Array(decoding.readVarUint8Array(responseDecoder)) + if (hasYjsUpdateContent(update)) { + this.queuePendingUpdate(update) + this.scheduleUpdateFlush(0) + } + } else { + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, response) + } } if (syncType === syncProtocol.messageYjsSyncStep2 && !this.synced) this.setSynced(true) break @@ -405,20 +691,236 @@ export class FileDocProvider extends ObservableV2 { // the stored content into the doc locally as its read-only fallback. Never relay those local // writes — the server never seeded this doc, so echoing them would push unseeded content to peers // (and each fallen-back client would do so, union-duplicating). A fatal client is fully local. - if (this.fatal || !this.joinAccepted || !this.socket.connected) return - // Updates we applied from the server carry `this` as origin — don't echo them. - if (origin === this) return + if (this.fatal || origin === this || origin === RECOVERY_ORIGIN) return // Agent-streamed frames must reach peers (so a collaborator sees the stream live) but must NOT be // treated by the server as a durable user edit — the copilot's final `edit_content` write is the // authoritative persist. Tag them so the relay applies + fans out but skips persist bookkeeping. - const messageType = - origin === AGENT_STREAM_ORIGIN - ? FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST - : FILE_DOC_MESSAGE_TYPE.SYNC - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageType) - syncProtocol.writeUpdate(encoder, update) - this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + if (origin === AGENT_STREAM_ORIGIN) { + if (!this.joinAccepted || !this.socket.connected) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST) + syncProtocol.writeUpdate(encoder, update) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + return + } + + if (!this.joinAccepted) { + this.queuePendingUpdate(update) + if (this.acknowledgedUpdates) this.scheduleUpdateFlush(UPDATE_BATCH_MS) + return + } + + if (!this.acknowledgedUpdates) { + if (!this.socket.connected) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, update) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + return + } + + this.queuePendingUpdate(update) + this.scheduleUpdateFlush(UPDATE_BATCH_MS) + } + + private queuePendingUpdate(update: Uint8Array): void { + this.pendingUpdateBatch.push(update) + this.updateBeforeUnloadProtection() + } + + private scheduleUpdateFlush(delay: number) { + if (this.updateBatchTimer !== null || this.updateFlushInProgress || this.disposed || this.fatal) + return + this.updateBatchTimer = setTimeout(() => { + this.updateBatchTimer = null + void this.flushPendingUpdates() + }, delay) + } + + private async flushPendingUpdates(): Promise { + if ( + !this.acknowledgedUpdates || + this.pendingUpdateBatch.length === 0 || + this.disposed || + this.fatal + ) + return + const docId = this.docId() + if (!docId) return + + this.updateFlushInProgress = true + try { + const update = Y.mergeUpdates(this.pendingUpdateBatch) + this.pendingUpdateBatch = [] + const journalUpdate = this.inFlightUpdate + ? Y.mergeUpdates([this.inFlightUpdate.update, update]) + : update + const saved = await this.journal?.save(docId, journalUpdate, Y.encodeStateAsUpdate(this.doc)) + if (this.disposed || this.fatal || this.pendingChangesDiscarded) { + if (!this.pendingChangesDiscarded) this.queuePendingUpdate(update) + return + } + if (saved?.status === 'limit-exceeded') { + this.queuePendingUpdate(update) + this.failFatally( + 'Local edits exceeded the safe recovery limit; download your draft before reloading', + 'PENDING_UPDATE_LIMIT' + ) + return + } + const durableUpdate = saved?.pendingUpdate ?? update + + if (this.inFlightUpdate) { + this.queuePendingUpdate(update) + return + } + this.inFlightUpdate = { updateId: generateShortId(), update: durableUpdate } + this.updateRetryAttempt = 0 + this.sendInFlightUpdate() + } finally { + this.updateFlushInProgress = false + this.updateBeforeUnloadProtection() + if (this.pendingUpdateBatch.length > 0 && !this.inFlightUpdate) { + this.scheduleUpdateFlush(0) + } + } + } + + private sendInFlightUpdate() { + const pending = this.inFlightUpdate + const docId = this.docId() + if ( + !pending || + !docId || + !this.acknowledgedUpdates || + this.disposed || + this.fatal || + !this.socket.connected || + !this.joinAccepted + ) + return + + if (this.updateAckTimer !== null) clearTimeout(this.updateAckTimer) + this.updateAckTimer = setTimeout(() => { + this.updateAckTimer = null + this.scheduleUpdateRetry() + }, FILE_DOC_TIMEOUTS.updateAckMs) + + const payload: FileDocUpdatePayload = { + fileId: this.fileId, + docId, + updateId: pending.updateId, + update: pending.update, + } + this.socket.emit(FILE_DOC_EVENTS.UPDATE, payload, (ack: FileDocUpdateAck) => { + this.handleUpdateAck(ack) + }) + } + + private handleUpdateAck(ack: FileDocUpdateAck) { + const pending = this.inFlightUpdate + if (!pending || ack.updateId !== pending.updateId || this.disposed || this.fatal) return + if (this.updateAckTimer !== null) clearTimeout(this.updateAckTimer) + this.updateAckTimer = null + + if (ack.status === 'accepted') { + const docId = this.docId() + this.inFlightUpdate = null + this.updateRetryAttempt = 0 + this.updateBeforeUnloadProtection() + if (this.pendingUpdateBatch.length > 0) this.scheduleUpdateFlush(0) + else if (!this.updateFlushInProgress && docId) void this.journal?.clear(docId, pending.update) + return + } + + if (!ack.retryable) { + const message = + ack.code === 'ACCESS_REVOKED' + ? 'Your access to this document has been revoked' + : 'This document changed while this tab was disconnected; refresh to continue editing' + this.failFatally(message, ack.code) + return + } + if (ack.code === 'NOT_JOINED') { + this.setSynced(false) + this.joinAccepted = false + this.joinPending = false + this.clearSyncRetryTimer() + this.scheduleJoinRetry() + return + } + this.scheduleUpdateRetry() + } + + private scheduleUpdateRetry() { + if (this.updateRetryTimer !== null || this.disposed || this.fatal || !this.socket.connected) + return + this.updateRetryAttempt += 1 + this.updateRetryTimer = setTimeout( + () => { + this.updateRetryTimer = null + this.sendInFlightUpdate() + }, + backoffWithJitter(this.updateRetryAttempt, null, { + baseMs: UPDATE_RETRY_BASE_MS, + maxMs: UPDATE_RETRY_MAX_MS, + }) + ) + } + + private pendingJournalUpdate(): Uint8Array | null { + const updates = [ + ...(this.inFlightUpdate ? [this.inFlightUpdate.update] : []), + ...this.pendingUpdateBatch, + ] + return updates.length > 0 ? Y.mergeUpdates(updates) : null + } + + private persistPendingSnapshot(): Promise | undefined { + if (this.pendingChangesDiscarded) return + const update = this.pendingJournalUpdate() + const docId = this.docId() + if (!update || !docId || !this.journal) return + return this.journal.save(docId, update, Y.encodeStateAsUpdate(this.doc)).then(() => undefined) + } + + private handlePageHide = () => { + void this.persistPendingSnapshot() + } + + private handleBeforeUnload = (event: BeforeUnloadEvent) => { + event.preventDefault() + event.returnValue = '' + } + + private updateBeforeUnloadProtection(): void { + if (typeof window === 'undefined') return + const shouldProtect = + !this.disposed && + !this.pendingChangesDiscarded && + (this.pendingUpdateBatch.length > 0 || + this.inFlightUpdate !== null || + this.updateFlushInProgress) + if (shouldProtect === this.beforeUnloadProtected) return + this.beforeUnloadProtected = shouldProtect + if (shouldProtect) window.addEventListener('beforeunload', this.handleBeforeUnload) + else window.removeEventListener('beforeunload', this.handleBeforeUnload) + } + + /** Remove the stale recovery record before deliberately loading a replacement document. */ + discardPendingChanges(): Promise { + this.pendingChangesDiscarded = true + this.clearUpdateTimers() + this.pendingUpdateBatch = [] + this.inFlightUpdate = null + this.updateBeforeUnloadProtection() + const docId = this.recoveryDocId ?? this.docId() + return docId ? (this.journal?.discard(docId) ?? Promise.resolve()) : Promise.resolve() + } + + private clearBufferedMessages(): void { + this.bufferedMessages = [] + this.bufferedMessageBytes = 0 } private handleAwarenessUpdate = ( @@ -452,6 +954,25 @@ export class FileDocProvider extends ObservableV2 { this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) } + private scheduleSyncRetry() { + this.clearSyncRetryTimer() + if (this.synced || this.fatal || this.disposed || !this.socket.connected || !this.joinAccepted) + return + this.syncRetryAttempt += 1 + this.syncRetryTimer = setTimeout( + () => { + this.syncRetryTimer = null + if (this.synced || this.fatal || this.disposed || !this.joinAccepted) return + this.sendSyncStep1() + this.scheduleSyncRetry() + }, + backoffWithJitter(this.syncRetryAttempt, null, { + baseMs: 1_000, + maxMs: JOIN_RETRY_MAX_MS, + }) + ) + } + private sendLocalAwareness() { if (this.awareness.getLocalState() === null) return const encoder = encoding.createEncoder() @@ -466,6 +987,10 @@ export class FileDocProvider extends ObservableV2 { private setSynced(synced: boolean) { if (this.synced === synced) return this.synced = synced + if (synced) { + this.clearSyncRetryTimer() + this.syncRetryAttempt = 0 + } // Readiness needs synced AND seeded; only clear the deadline when both hold (the seed may have // arrived first, or may still be pending — `handleConfigChange` clears it if seeded arrives later). if (synced && this.isSeeded()) this.clearReadinessTimer() @@ -482,9 +1007,16 @@ export class FileDocProvider extends ObservableV2 { super.destroy() return } + void this.persistPendingSnapshot() this.disposed = true + this.updateBeforeUnloadProtection() + this.unregisterActiveProvider() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearJoinAckTimer() + this.clearSyncRetryTimer() + this.clearUpdateTimers() + this.clearBufferedMessages() this.joinPending = false // Publish our final awareness removal while this provider is still admitted. A co-mounted sibling @@ -500,12 +1032,14 @@ export class FileDocProvider extends ObservableV2 { this.socket.off(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) this.socket.off(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) this.socket.off(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + this.socket.off(FILE_DOC_EVENTS.INVALIDATED, this.handleInvalidated) this.socket.off(ROOM_ACCESS_REVOKED_EVENT, this.handleAccessRevoked) this.socket.off('connect', this.handleConnect) this.socket.off('disconnect', this.handleDisconnect) this.doc.off('update', this.handleDocUpdate) this.doc.getMap(FILE_DOC_SEED.configMap).unobserve(this.handleConfigChange) this.awareness.off('update', this.handleAwarenessUpdate) + if (typeof window !== 'undefined') window.removeEventListener('pagehide', this.handlePageHide) super.destroy() } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts new file mode 100644 index 00000000000..618bc443c67 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts @@ -0,0 +1,189 @@ +/** + * @vitest-environment node + */ +import { FILE_DOC_LIMITS } from '@sim/realtime-protocol/file-doc' +import { update as updateValue } from 'idb-keyval' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +const storage = vi.hoisted(() => new Map()) + +vi.mock('idb-keyval', () => ({ + update: vi.fn((key: string, updater: (value: unknown) => unknown) => { + storage.set(key, updater(storage.get(key))) + }), +})) + +import { PendingFileDocUpdateJournal } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' + +function journal(): PendingFileDocUpdateJournal { + return new PendingFileDocUpdateJournal({ + workspaceId: 'workspace-1', + fileId: 'file-1', + userId: 'user-1', + }) +} + +function updateWith(text: string): Uint8Array { + const doc = new Y.Doc() + doc.getText('body').insert(0, text) + return Y.encodeStateAsUpdate(doc) +} + +describe('PendingFileDocUpdateJournal', () => { + beforeEach(() => { + storage.clear() + }) + + it('stores a full recovery snapshot separately from the pending wire update', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + const recoverySnapshot = updateWith('complete local draft') + + await subject.save('doc-1', pendingUpdate, recoverySnapshot) + + await expect(subject.load('doc-1')).resolves.toEqual( + expect.objectContaining({ docId: 'doc-1', pendingUpdate, recoverySnapshot }) + ) + }) + + it('reports when the current full recovery snapshot cannot be stored', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + + const result = await subject.save( + 'doc-1', + pendingUpdate, + new Uint8Array(FILE_DOC_LIMITS.updateBytes * 2 + 1) + ) + + expect(result).toMatchObject({ status: 'limit-exceeded' }) + }) + + it('distinguishes unavailable browser storage from a configured size limit', async () => { + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Storage denied')) + const pendingUpdate = updateWith('pending') + + await expect(journal().save('doc-1', pendingUpdate, pendingUpdate)).resolves.toEqual({ + pendingUpdate, + status: 'unavailable', + }) + }) + + it('atomically preserves concurrent providers until their aggregate is acknowledged', async () => { + const first = journal() + const second = journal() + const firstUpdate = updateWith('a') + const secondUpdate = updateWith('b') + + await first.save('doc-1', firstUpdate, firstUpdate) + const combined = await second.save('doc-1', secondUpdate, secondUpdate) + await first.clear('doc-1', firstUpdate) + await expect(first.load('doc-1')).resolves.not.toBeNull() + + const recovered = new Y.Doc() + Y.applyUpdate(recovered, combined.pendingUpdate) + expect(recovered.getText('body').toString()).toHaveLength(2) + + await second.clear('doc-1', combined.pendingUpdate) + await expect(first.load('doc-1')).resolves.toBeNull() + }) + + it('preserves the snapshot dependencies of pending edits from concurrent tabs', async () => { + const first = journal() + const second = journal() + const base = new Y.Doc() + base.getText('body').insert(0, 'base') + const firstDoc = new Y.Doc() + const secondDoc = new Y.Doc() + Y.applyUpdate(firstDoc, Y.encodeStateAsUpdate(base)) + Y.applyUpdate(secondDoc, Y.encodeStateAsUpdate(base)) + + firstDoc.getText('body').insert(4, ' acknowledged') + const firstVector = Y.encodeStateVector(firstDoc) + firstDoc.getText('body').insert(17, ' pending-first') + await first.save( + 'doc-1', + Y.encodeStateAsUpdate(firstDoc, firstVector), + Y.encodeStateAsUpdate(firstDoc) + ) + + const secondVector = Y.encodeStateVector(secondDoc) + secondDoc.getText('body').insert(4, ' pending-second') + await second.save( + 'doc-1', + Y.encodeStateAsUpdate(secondDoc, secondVector), + Y.encodeStateAsUpdate(secondDoc) + ) + + const stored = await journal().load('doc-1') + expect(stored).not.toBeNull() + const recovered = new Y.Doc() + Y.applyUpdate(recovered, stored!.recoverySnapshot!) + Y.applyUpdate(recovered, stored!.pendingUpdate) + + const expected = new Y.Doc() + Y.applyUpdate(expected, Y.encodeStateAsUpdate(firstDoc)) + Y.applyUpdate(expected, Y.encodeStateAsUpdate(secondDoc)) + expect(recovered.getText('body').toString()).toBe(expected.getText('body').toString()) + expect(recovered.getText('body').toString()).toContain('pending-first') + expect(recovered.getText('body').toString()).toContain('pending-second') + for (const doc of [base, firstDoc, secondDoc, recovered, expected]) doc.destroy() + }) + + it('bounds the combined snapshots without overwriting the previous recovery copy', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + const firstSnapshot = updateWith('a'.repeat(FILE_DOC_LIMITS.updateBytes)) + const secondSnapshot = updateWith('b'.repeat(FILE_DOC_LIMITS.updateBytes)) + await subject.save('doc-1', pendingUpdate, firstSnapshot) + + await expect(subject.save('doc-1', pendingUpdate, secondSnapshot)).resolves.toMatchObject({ + status: 'limit-exceeded', + }) + const recovered = await subject.load('doc-1') + expect(recovered?.recoverySnapshot).toBeInstanceOf(Uint8Array) + expect(Buffer.from(recovered!.recoverySnapshot!).equals(Buffer.from(firstSnapshot))).toBe(true) + }) + + it('retains bounded recovery records for separate document identities', async () => { + const subject = journal() + for (const docId of ['doc-1', 'doc-2', 'doc-3', 'doc-4']) { + const update = updateWith(docId) + await subject.save(docId, update, update) + } + + await expect(subject.load('doc-4')).resolves.toMatchObject({ docId: 'doc-4' }) + await expect(subject.load('doc-2')).resolves.toMatchObject({ docId: 'doc-2' }) + await expect(subject.load('doc-1')).resolves.toBeNull() + await expect(subject.load()).resolves.toMatchObject({ docId: 'doc-4' }) + }) + + it('discards only the selected document identity', async () => { + const subject = journal() + const oldUpdate = updateWith('old') + const currentUpdate = updateWith('current') + await subject.save('old-doc', oldUpdate, oldUpdate) + await subject.save('current-doc', currentUpdate, currentUpdate) + + await subject.discard('old-doc') + + await expect(subject.load('old-doc')).resolves.toBeNull() + await expect(subject.load('current-doc')).resolves.toMatchObject({ docId: 'current-doc' }) + }) + + it('isolates records by user, workspace, and file', async () => { + const first = journal() + const otherUser = new PendingFileDocUpdateJournal({ + workspaceId: 'workspace-1', + fileId: 'file-1', + userId: 'user-2', + }) + const update = updateWith('draft') + + await first.save('doc-1', update, update) + + await expect(first.load()).resolves.not.toBeNull() + await expect(otherUser.load()).resolves.toBeNull() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts new file mode 100644 index 00000000000..df2b92cf214 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts @@ -0,0 +1,210 @@ +'use client' + +import { createLogger } from '@sim/logger' +import { FILE_DOC_LIMITS } from '@sim/realtime-protocol/file-doc' +import { update as updateValue } from 'idb-keyval' +import * as Y from 'yjs' + +const logger = createLogger('PendingFileDocUpdateJournal') +const JOURNAL_VERSION = 1 +const JOURNAL_TTL_MS = 7 * 24 * 60 * 60 * 1_000 +const MAX_DOCUMENTS = 3 +const RECOVERY_SNAPSHOT_MAX_BYTES = FILE_DOC_LIMITS.updateBytes * 2 + +export interface PendingDocumentRecovery { + docId: string + pendingUpdate: Uint8Array + recoverySnapshot: Uint8Array | null + updatedAt: number +} + +interface PendingUpdateJournalRecord { + version: typeof JOURNAL_VERSION + documents: PendingDocumentRecovery[] +} + +interface PendingUpdateJournalScope { + workspaceId: string + fileId: string + userId: string +} + +interface JournalSaveResult { + pendingUpdate: Uint8Array + status: 'saved' | 'limit-exceeded' | 'unavailable' +} + +function isRecovery(value: unknown): value is PendingDocumentRecovery { + if (typeof value !== 'object' || value === null) return false + const candidate = value as Partial + return ( + typeof candidate.docId === 'string' && + candidate.docId.length > 0 && + candidate.pendingUpdate instanceof Uint8Array && + candidate.pendingUpdate.byteLength > 0 && + candidate.pendingUpdate.byteLength <= FILE_DOC_LIMITS.updateBytes && + (candidate.recoverySnapshot === null || + (candidate.recoverySnapshot instanceof Uint8Array && + candidate.recoverySnapshot.byteLength > 0 && + candidate.recoverySnapshot.byteLength <= RECOVERY_SNAPSHOT_MAX_BYTES)) && + typeof candidate.updatedAt === 'number' && + Number.isFinite(candidate.updatedAt) + ) +} + +function liveDocuments(value: unknown, now: number): PendingDocumentRecovery[] { + if (typeof value !== 'object' || value === null) return [] + const candidate = value as Partial + if (candidate.version !== JOURNAL_VERSION || !Array.isArray(candidate.documents)) return [] + return candidate.documents + .filter(isRecovery) + .filter((document) => now - document.updatedAt <= JOURNAL_TTL_MS) + .sort((left, right) => right.updatedAt - left.updatedAt) + .slice(0, MAX_DOCUMENTS) +} + +function record(documents: PendingDocumentRecovery[]): PendingUpdateJournalRecord { + return { version: JOURNAL_VERSION, documents } +} + +function sameUpdate(left: Uint8Array, right: Uint8Array): boolean { + if (left.byteLength !== right.byteLength) return false + return left.every((byte, index) => byte === right[index]) +} + +/** + * A bounded crash-recovery journal for user edits the relay has not acknowledged. One atomic + * file-scoped envelope retains up to three recent Yjs document identities, so rebuilding a live + * document cannot overwrite an older local draft. The pending delta is wire-bounded separately from + * the full recovery snapshot: only the delta is ever replayed to a matching server document. + */ +export class PendingFileDocUpdateJournal { + private readonly key: string + private mutationQueue = Promise.resolve() + + constructor({ workspaceId, fileId, userId }: PendingUpdateJournalScope) { + const origin = typeof location === 'undefined' ? 'server' : location.origin + this.key = [ + 'sim', + 'file-doc-pending', + JOURNAL_VERSION, + origin, + userId, + workspaceId, + fileId, + ].join(':') + } + + async load(preferredDocId?: string): Promise { + try { + await this.mutationQueue + let recovered: PendingDocumentRecovery | null = null + await updateValue(this.key, (value) => { + const documents = liveDocuments(value, Date.now()) + recovered = preferredDocId + ? (documents.find((document) => document.docId === preferredDocId) ?? null) + : (documents[0] ?? null) + return record(documents) + }) + return recovered + } catch (error) { + logger.warn('Failed to load pending file edits', { error }) + return null + } + } + + save( + docId: string, + pendingUpdate: Uint8Array, + recoverySnapshot: Uint8Array + ): Promise { + const pendingWithinLimit = + pendingUpdate.byteLength > 0 && pendingUpdate.byteLength <= FILE_DOC_LIMITS.updateBytes + const snapshotWithinLimit = + recoverySnapshot.byteLength > 0 && recoverySnapshot.byteLength <= RECOVERY_SNAPSHOT_MAX_BYTES + const limited: JournalSaveResult = { pendingUpdate, status: 'limit-exceeded' } + if (!pendingWithinLimit || !snapshotWithinLimit) return Promise.resolve(limited) + + return this.enqueue( + async () => { + let result = limited + await updateValue(this.key, (value) => { + const now = Date.now() + const documents = liveDocuments(value, now) + const existing = documents.find((document) => document.docId === docId) + const merged = existing + ? Y.mergeUpdates([existing.pendingUpdate, pendingUpdate]) + : pendingUpdate + if (merged.byteLength > FILE_DOC_LIMITS.updateBytes) return record(documents) + + const mergedSnapshot = existing?.recoverySnapshot + ? Y.mergeUpdates([existing.recoverySnapshot, recoverySnapshot]) + : recoverySnapshot + if (mergedSnapshot.byteLength > RECOVERY_SNAPSHOT_MAX_BYTES) return record(documents) + + const next: PendingDocumentRecovery = { + docId, + pendingUpdate: merged, + recoverySnapshot: mergedSnapshot, + updatedAt: now, + } + const retained = [ + next, + ...documents.filter((document) => document.docId !== docId), + ].slice(0, MAX_DOCUMENTS) + result = { + pendingUpdate: merged, + status: 'saved', + } + return record(retained) + }) + if (result.status === 'limit-exceeded') { + logger.warn('Pending file edits exceeded the crash-recovery journal limit') + } + return result + }, + { pendingUpdate, status: 'unavailable' } + ) + } + + clear(docId: string, acknowledgedUpdate: Uint8Array): Promise { + return this.enqueue( + () => + updateValue(this.key, (value) => { + const documents = liveDocuments(value, Date.now()) + return record( + documents.filter( + (document) => + document.docId !== docId || !sameUpdate(document.pendingUpdate, acknowledgedUpdate) + ) + ) + }), + undefined + ) + } + + /** Deliberately abandon one recovery identity after the user has preserved its local draft. */ + discard(docId: string): Promise { + return this.enqueue( + () => + updateValue(this.key, (value) => + record(liveDocuments(value, Date.now()).filter((document) => document.docId !== docId)) + ), + undefined, + true + ) + } + + private enqueue(operation: () => Promise, fallback: T, rethrow = false): Promise { + const result = this.mutationQueue.then(operation, operation) + this.mutationQueue = result.then( + () => undefined, + () => undefined + ) + return result.catch((error) => { + logger.warn('Failed to persist pending file edits', { error }) + if (rethrow) throw error + return fallback + }) + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts index 7dc9eaaa530..2bad83b13da 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts @@ -5,9 +5,9 @@ import { FILE_DOC_EVENTS, type FileDocPresence } from '@sim/realtime-protocol/fi import { Awareness } from 'y-protocols/awareness' import * as Y from 'yjs' import { getUserColor } from '@/lib/workspaces/colors' +import { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' +import { useReportFileDocOthers } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context' import { useSocket } from '@/app/workspace/providers/socket-provider' -import { FileDocProvider } from './file-doc-provider' -import { useReportFileDocOthers } from './file-doc-room-context' /** The live collaboration binding the editor wires into TipTap's Collaboration * (the {@link Y.Doc}) and CollaborationCaret (the awareness). */ @@ -32,6 +32,7 @@ export interface FileDocCollaboration { } interface UseFileDocCollaborationParams { + workspaceId: string fileId: string userId: string userName: string @@ -51,6 +52,7 @@ interface UseFileDocCollaborationParams { * realtime relay over the shared socket. Returns `null` while disabled. */ export function useFileDocCollaboration({ + workspaceId, fileId, userId, userName, @@ -102,13 +104,16 @@ export function useFileDocCollaboration({ // (see above), so this always binds the same doc/awareness the editor froze at mount. const doc = docRef.current as Y.Doc const awareness = awarenessRef.current as Awareness - const fileProvider = new FileDocProvider(socket, fileId, doc, awareness) + const fileProvider = new FileDocProvider(socket, fileId, doc, awareness, { + workspaceId, + userId, + }) setProvider(fileProvider) return () => { fileProvider.destroy() setProvider(null) } - }, [enabled, socket, fileId]) + }, [enabled, socket, fileId, workspaceId, userId]) const reportOthers = useReportFileDocOthers() const reportOthersRef = useRef(reportOthers) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx index 914a0643d63..70bcd21f71c 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx @@ -17,17 +17,22 @@ import { } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/paste-admission' import { LoadedRichMarkdownEditor } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor' -const { collaborationRef, uploadFile } = vi.hoisted(() => ({ +const { collaborationRef, uploadFile, saveBlob } = vi.hoisted(() => ({ collaborationRef: { current: null as unknown }, uploadFile: vi.fn(), + saveBlob: vi.fn(), })) -vi.mock('next/navigation', () => ({ useRouter: () => ({ push: vi.fn() }) })) +vi.mock('next/navigation', () => ({ + usePathname: () => '/workspace/workspace-1/files', + useRouter: () => ({ push: vi.fn() }), +})) vi.mock('@/lib/auth/auth-client', () => ({ useSession: () => ({ data: null, isPending: false }) })) vi.mock('@/hooks/queries/workspace-files', () => ({ useUploadWorkspaceFile: () => ({ mutateAsync: uploadFile }), })) vi.mock('@/hooks/use-add-to-chat', () => ({ useAddToChat: () => vi.fn() })) +vi.mock('@/lib/uploads/client/download', () => ({ saveBlob })) vi.mock('@/hooks/use-file-content-source', () => ({ useFileContentSource: () => ({ resolveImageSrc: (src: string) => src }), })) @@ -98,6 +103,7 @@ const onChange = vi.fn() const onEditSource = vi.fn() const onClientAutosaveChange = vi.fn() const onSaveShortcut = vi.fn() +const onDownloadDraft = vi.fn() const onSuspendedRender = vi.fn() const pendingRender = new Promise(() => {}) @@ -116,6 +122,7 @@ function SuspendAfterEditor({ active }: SuspendAfterEditorProps) { class FakeFileDocProvider { synced = false joinError: JoinFileDocError | null = null + discardPendingChanges = vi.fn(() => Promise.resolve()) private readonly listeners = new Map void>>() on(event: string, listener: (value: unknown) => void) { @@ -175,6 +182,7 @@ async function render( onEditSource={onEditSource} onClientAutosaveChange={onClientAutosaveChange} onSaveShortcut={options.onSaveShortcut ?? onSaveShortcut} + onDownloadDraft={onDownloadDraft} /> @@ -255,7 +263,7 @@ describe('loaded rich editor lifecycle', () => { expect(container.textContent).not.toContain('Reconnecting…') }) - it('keeps the live document visible and read-only after a fatal collaboration error', async () => { + it('keeps a revoked unacknowledged draft visible, read-only, and downloadable', async () => { const provider = new FakeFileDocProvider() const doc = new Y.Doc() doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) @@ -275,7 +283,7 @@ describe('loaded rich editor lifecycle', () => { provider.fail({ fileId: 'file-1', error: 'Access denied', - code: 'ACCESS_DENIED', + code: 'ACCESS_REVOKED', retryable: false, }) ) @@ -286,6 +294,7 @@ describe('loaded rich editor lifecycle', () => { expect(editor.view.dom.closest('.hidden')).toBeNull() expect(container.textContent).not.toContain('stale opening snapshot') expect(container.textContent).not.toContain('Reconnecting…') + expect(container.textContent).toContain('Download local draft') }) it('shows stored content read-only when collaboration fails before the first sync', async () => { @@ -316,6 +325,50 @@ describe('loaded rich editor lifecycle', () => { expect(container.textContent).not.toContain('Reconnecting…') }) + it.each(['DOCUMENT_REPLACED', 'PENDING_UPDATE_LIMIT', 'INVALID_UPDATE'])( + 'keeps a local draft downloadable before offering a destructive reload for %s', + async (code) => { + const provider = new FakeFileDocProvider() + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + collaborationRef.current = { + doc, + awareness: new Awareness(doc), + provider, + user: { name: 'User', color: '#000000', clientId: doc.clientID }, + } + await render('stored body', 'stored body', true, { collaborative: true }) + + await act(async () => provider.setSynced(true)) + await act(async () => getEditor().commands.insertContent('preserved local change')) + await act(async () => + provider.fail({ + fileId: 'file-1', + error: 'Local recovery required', + code, + retryable: false, + }) + ) + + const buttons = [...container.querySelectorAll('button')] + const download = buttons.find((button) => button.textContent === 'Download local draft') + expect(download).toBeDefined() + expect(buttons.some((button) => button.textContent === 'Discard draft')).toBe(true) + await act(async () => download?.click()) + expect(onDownloadDraft).not.toHaveBeenCalled() + expect(saveBlob).toHaveBeenCalledOnce() + const downloaded = saveBlob.mock.calls[0][0] as Blob + const downloadedText = await new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(String(reader.result)) + reader.onerror = () => reject(reader.error) + reader.readAsText(downloaded) + }) + expect(downloadedText).toContain('preserved local change') + expect(getEditor().getText()).toContain('preserved local change') + } + ) + it('explains a picker selection whose insertion anchor was invalidated', async () => { await render('before TARGET after') const editor = getEditor() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts index 2f730b40771..3bba8143644 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts @@ -1,11 +1,26 @@ /** * @vitest-environment jsdom */ + +import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import { Editor } from '@tiptap/core' -import { undoDepth } from '@tiptap/pm/history' -import { afterEach, describe, expect, it } from 'vitest' -import { createMarkdownContentExtensions } from '../extensions' -import { getFindTally, RichMarkdownFind, setFindQuery, stepFindMatch } from './find-extension' +import { redoDepth, undoDepth } from '@tiptap/pm/history' +import { yUndoPluginKey } from '@tiptap/y-tiptap' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import type * as Y from 'yjs' +import { markdownToYDoc } from '@/lib/collab-doc/converter' +import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + getFindTally, + RichMarkdownFind, + replaceActiveFindMatch, + replaceAllFindMatches, + setFindQuery, + stepFindMatch, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension' +import { FIND_MATCH_LIMIT } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' let editor: Editor | null = null afterEach(() => { @@ -129,4 +144,119 @@ describe('RichMarkdownFind', () => { // instead of their real last edit. expect(undoDepth(instance.state)).toBe(undoBefore) }) + + it('replaces the active match while preserving its inline marks', () => { + const instance = mountEditor('**alpha** and alpha') + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'beta')).toBe(true) + expect(instance.getMarkdown()).toBe('**beta** and alpha') + expect(getFindTally(instance.state).matches).toHaveLength(1) + }) + + it('uses the matched text formatting instead of an unrelated typing mark', () => { + const instance = mountEditor('alpha and beta') + instance.commands.setTextSelection(instance.state.doc.content.size - 1) + instance.commands.toggleBold() + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'gamma')).toBe(true) + expect(instance.getMarkdown()).toBe('gamma and beta') + }) + + it('preserves each matched range formatting during Replace All', () => { + const instance = mountEditor('**alpha** and alpha and *alpha*') + instance.commands.setTextSelection(instance.state.doc.content.size - 1) + instance.commands.toggleStrike() + setFindQuery(instance, 'alpha') + + expect(replaceAllFindMatches(instance, 'beta')).toBe(3) + expect(instance.getMarkdown()).toBe('**beta** and beta and *beta*') + }) + + it('supports deleting matches with an empty replacement', () => { + const instance = mountEditor('alpha beta alpha') + setFindQuery(instance, 'alpha ') + + expect(replaceActiveFindMatch(instance, '')).toBe(true) + expect(instance.getMarkdown()).toBe('beta alpha') + }) + + it('rejects oversized individual and aggregate replacements before dispatching a transaction', () => { + const instance = mountEditor(Array.from({ length: FIND_MATCH_LIMIT }, () => 'x').join(' ')) + setFindQuery(instance, 'x') + const onLimitExceeded = vi.fn() + const dispatch = vi.spyOn(instance.view, 'dispatch') + const documentBefore = instance.state.doc + + expect( + replaceActiveFindMatch( + instance, + 'y'.repeat(PASTE_RENDER_THRESHOLDS.ENHANCED_TEXT_CHARACTERS), + onLimitExceeded + ) + ).toBe(false) + expect(replaceAllFindMatches(instance, 'y'.repeat(600), onLimitExceeded)).toBe(0) + + expect(onLimitExceeded).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + expect(instance.state.doc).toBe(documentBefore) + }) + + it('advances past a replacement that still contains the search term', () => { + const instance = mountEditor('alpha alpha') + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'alphaX')).toBe(true) + expect(replaceActiveFindMatch(instance, 'alphaX')).toBe(true) + + expect(instance.getMarkdown()).toBe('alphaX alphaX') + }) + + it('keeps each collaborative replacement as a separate undo item', () => { + const doc = markdownToYDoc('alpha alpha') + const awareness = new Awareness(doc) + editor = new Editor({ + extensions: createMarkdownEditorExtensions({ + placeholder: '', + collaboration: { doc, awareness, user: { name: 'User', color: '#fff' } }, + }), + }) + const history = yUndoPluginKey.getState(editor.state) as { undoManager: Y.UndoManager } + history.undoManager.clear() + setFindQuery(editor, 'alpha') + + replaceActiveFindMatch(editor, 'beta') + replaceActiveFindMatch(editor, 'gamma') + expect(editor.getMarkdown()).toBe('beta gamma') + + expect(editor.commands.undo()).toBe(true) + expect(editor.getMarkdown()).toBe('beta alpha') + editor.destroy() + editor = null + awareness.destroy() + doc.destroy() + }) + + it('replaces every match in one undo step', () => { + const instance = mountEditor('alpha alpha alpha') + setFindQuery(instance, 'alpha') + const undoBefore = undoDepth(instance.state) + + expect(replaceAllFindMatches(instance, 'beta')).toBe(3) + expect(instance.getMarkdown()).toBe('beta beta beta') + expect(undoDepth(instance.state)).toBe(undoBefore + 1) + expect(redoDepth(instance.state)).toBe(0) + expect(instance.commands.undo()).toBe(true) + expect(instance.getMarkdown()).toBe('alpha alpha alpha') + }) + + it('refuses to label a capped partial replacement as replace all', () => { + const instance = mountEditor(Array.from({ length: FIND_MATCH_LIMIT + 1 }, () => 'x').join(' ')) + setFindQuery(instance, 'x') + expect(getFindTally(instance.state).truncated).toBe(true) + + expect(replaceAllFindMatches(instance, 'y')).toBe(0) + expect(instance.getMarkdown().startsWith('x x x')).toBe(true) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts index e2ac3f0a20a..93121d6b4d9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts @@ -1,9 +1,16 @@ +import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import type { Editor } from '@tiptap/core' import { Extension } from '@tiptap/core' +import { closeHistory } from '@tiptap/pm/history' import type { EditorState } from '@tiptap/pm/state' import { Plugin, PluginKey } from '@tiptap/pm/state' import { Decoration, DecorationSet } from '@tiptap/pm/view' -import { EMPTY_FIND_RESULT, type FindMatch, findMatches } from './find-matches' +import { yUndoPluginKey } from '@tiptap/y-tiptap' +import { + EMPTY_FIND_RESULT, + type FindMatch, + findMatches, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' /** Class on every match. The active one carries {@link ACTIVE_MATCH_CLASS} as well. */ const MATCH_CLASS = 'rich-find-match' @@ -149,3 +156,88 @@ export function setFindQuery(editor: Editor, query: string): void { export function stepFindMatch(editor: Editor, delta: number): void { dispatchFindMeta(editor, { activeIndex: getFindTally(editor.state).activeIndex + delta }) } + +function stopUndoCapture(editor: Editor): void { + const state = yUndoPluginKey.getState(editor.state) as + | { undoManager?: { stopCapturing: () => void } } + | undefined + state?.undoManager?.stopCapturing() +} + +function isolateReplacement(editor: Editor, transaction: EditorState['tr']): void { + stopUndoCapture(editor) + editor.view.dispatch(closeHistory(transaction).scrollIntoView()) + stopUndoCapture(editor) + editor.view.dispatch(closeHistory(editor.state.tr)) +} + +/** Bounds aggregate growth before Replace All can materialize hundreds of large insertions. */ +function replacementExceedsLimit( + editor: Editor, + matches: readonly FindMatch[], + replacement: string +): boolean { + const currentSize = editor.state.doc.content.size + const nextSize = matches.reduce( + (size, match) => size + replacement.length - (match.to - match.from), + currentSize + ) + return nextSize > Math.max(currentSize, PASTE_RENDER_THRESHOLDS.ENHANCED_TEXT_CHARACTERS) +} + +/** Uses the target range's marks, independent of formatting armed at the editor's caret. */ +function replaceMatch(transaction: EditorState['tr'], match: FindMatch, replacement: string): void { + const marks = transaction.doc.resolve(match.from).marksAcross(transaction.doc.resolve(match.to)) + transaction.replaceWith( + match.from, + match.to, + replacement ? transaction.doc.type.schema.text(replacement, marks) : [] + ) +} + +/** Replaces the active match as one ordinary editor transaction. */ +export function replaceActiveFindMatch( + editor: Editor, + replacement: string, + onLimitExceeded?: () => void +): boolean { + if (!editor.isEditable) return false + const findState = RICH_FIND_PLUGIN_KEY.getState(editor.state) ?? INITIAL_STATE + const { matches, activeIndex } = findState + const match = matches[activeIndex] + if (!match) return false + if (replacementExceedsLimit(editor, [match], replacement)) { + onLimitExceeded?.() + return false + } + const transaction = editor.state.tr + replaceMatch(transaction, match, replacement) + const remaining = findMatches(transaction.doc, findState.query).matches + const insertionEnd = match.from + replacement.length + const nextIndex = remaining.findIndex((candidate) => candidate.from >= insertionEnd) + transaction.setMeta(RICH_FIND_PLUGIN_KEY, { activeIndex: nextIndex === -1 ? 0 : nextIndex }) + isolateReplacement(editor, transaction) + return true +} + +/** Replaces every collected match in one undo step; capped searches must first be narrowed. */ +export function replaceAllFindMatches( + editor: Editor, + replacement: string, + onLimitExceeded?: () => void +): number { + if (!editor.isEditable) return 0 + const { matches, truncated } = getFindTally(editor.state) + if (truncated || matches.length === 0) return 0 + if (replacementExceedsLimit(editor, matches, replacement)) { + onLimitExceeded?.() + return 0 + } + const transaction = closeHistory(editor.state.tr) + for (let index = matches.length - 1; index >= 0; index -= 1) { + const match = matches[index] + replaceMatch(transaction, match, replacement) + } + isolateReplacement(editor, transaction) + return matches.length +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts index b0a97d5a6f6..d3a103a69c1 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts @@ -2,9 +2,17 @@ import type React from 'react' import { useCallback, useEffect, useRef, useState } from 'react' +import { toast } from '@sim/emcn' import type { Editor } from '@tiptap/react' import { useFindShortcut } from '@/app/workspace/[workspaceId]/components' -import { ACTIVE_MATCH_CLASS, getFindTally, setFindQuery, stepFindMatch } from './find-extension' +import { + ACTIVE_MATCH_CLASS, + getFindTally, + replaceActiveFindMatch, + replaceAllFindMatches, + setFindQuery, + stepFindMatch, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension' /** What the surface hands `FindBar`, plus the open state the shortcut drives. */ export interface MarkdownFindController { @@ -14,9 +22,13 @@ export interface MarkdownFindController { currentIndex: number truncated: boolean inputRef: React.RefObject + replacement: string setQuery: (query: string) => void + setReplacement: (replacement: string) => void next: () => void prev: () => void + replaceCurrent: () => void + replaceAll: () => void close: () => void } @@ -29,6 +41,12 @@ interface FindTally { const EMPTY_TALLY: FindTally = { count: 0, currentIndex: 0, truncated: false } +function warnReplacementLimit(): void { + toast.warning('Replacement is too large', { + description: 'Use the source editor for changes that exceed the rich-text editing limit.', + }) +} + interface UseMarkdownFindOptions { editor: Editor | null /** @@ -55,6 +73,7 @@ export function useMarkdownFind({ }: UseMarkdownFindOptions): MarkdownFindController { const [isOpen, setIsOpen] = useState(false) const [query, setQueryState] = useState('') + const [replacement, setReplacement] = useState('') const [tally, setTally] = useState(EMPTY_TALLY) const inputRef = useRef(null) const editorRef = useRef(editor) @@ -141,13 +160,29 @@ export function useMarkdownFind({ const next = useCallback(() => step(1), [step]) const prev = useCallback(() => step(-1), [step]) + const replaceCurrent = useCallback(() => { + const current = editorRef.current + if (!current || !replaceActiveFindMatch(current, replacement, warnReplacementLimit)) return + revealActiveMatch() + }, [replacement, revealActiveMatch]) + + const replaceAll = useCallback(() => { + const current = editorRef.current + if (!current) return + replaceAllFindMatches(current, replacement, warnReplacementLimit) + }, [replacement]) + /** Closing ends the search: term, highlights and active match all go. */ const close = useCallback(() => { setIsOpen(false) setQueryState('') + setReplacement('') setTally(EMPTY_TALLY) const current = editorRef.current if (current) setFindQuery(current, '') + requestAnimationFrame(() => { + if (current && !current.isDestroyed) current.commands.focus() + }) }, []) const open = useCallback(() => setIsOpen(true), []) @@ -156,13 +191,17 @@ export function useMarkdownFind({ return { isOpen, query, + replacement, count: tally.count, currentIndex: tally.currentIndex, truncated: tally.truncated, inputRef, setQuery, + setReplacement, next, prev, + replaceCurrent, + replaceAll, close, } } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.test.tsx new file mode 100644 index 00000000000..22eaa427505 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.test.tsx @@ -0,0 +1,102 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { Tooltip } from '@sim/emcn' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ImageInspector } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector' + +let host: HTMLDivElement +let root: Root + +beforeEach(() => { + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) +}) + +afterEach(() => { + act(() => root.unmount()) + host.remove() +}) + +function button(label: string): HTMLButtonElement { + const element = host.querySelector(`button[aria-label="${label}"]`) + if (!element) throw new Error(`Missing ${label} button`) + return element +} + +function change(input: HTMLInputElement, value: string): void { + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(input, value) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) +} + +describe('ImageInspector', () => { + it('validates and applies accessible image details', async () => { + const onApply = vi.fn() + const onReturnFocus = vi.fn() + act(() => { + root.render( + + + + ) + }) + + act(() => button('Edit image details').click()) + expect(host.firstElementChild).toHaveClass('left-0') + expect(host.firstElementChild).not.toHaveClass('sm:left-1/2', 'sm:-translate-x-1/2') + const alt = host.querySelector('input[aria-label="Image alt text"]') + const href = host.querySelector('input[aria-label="Image link URL"]') + expect(alt?.value).toBe('Diagram') + expect(href?.value).toBe('https://sim.ai/original') + if (!alt || !href) return + + change(alt, 'Updated diagram') + href.focus() + change(href, 'javascript:alert(1)') + expect(document.activeElement).toBe(href) + expect(href).toHaveAttribute('aria-invalid', 'true') + expect(host.querySelector('[role="alert"]')?.textContent).toContain('valid link') + const apply = Array.from(host.querySelectorAll('button')).find((candidate) => + candidate.textContent?.includes('Apply') + ) + expect(apply?.disabled).toBe(true) + + change(href, 'https://sim.ai/updated') + act(() => apply?.click()) + expect(onApply).toHaveBeenCalledWith({ + alt: 'Updated diagram', + href: 'https://sim.ai/updated', + }) + await vi.waitFor(() => expect(onReturnFocus).toHaveBeenCalledTimes(1)) + }) + + it('offers size reset only for explicitly sized images', () => { + const onResetSize = vi.fn() + act(() => { + root.render( + + + + ) + }) + act(() => button('Reset image size').click()) + expect(onResetSize).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx new file mode 100644 index 00000000000..677423eb73e --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx @@ -0,0 +1,130 @@ +import { type KeyboardEvent, useId, useState } from 'react' +import { Button, ChipInput } from '@sim/emcn' +import { Check, RefreshCw, Settings, X } from '@sim/emcn/icons' +import { normalizeLinkHref } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { ToolbarButton } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button' + +interface ImageDetails { + alt: string + href: string +} + +interface ImageInspectorProps extends ImageDetails { + hasCustomSize: boolean + onApply: (details: ImageDetails) => void + onResetSize: () => void + onReturnFocus: () => void +} + +/** Selection-local controls for accessible image text, links, and explicit sizing. */ +export function ImageInspector({ + alt, + href, + hasCustomSize, + onApply, + onResetSize, + onReturnFocus, +}: ImageInspectorProps) { + const [draft, setDraft] = useState(null) + const errorId = useId() + const normalizedHref = draft ? normalizeLinkHref(draft.href.trim()) : '' + const invalidHref = Boolean(draft?.href.trim()) && !normalizedHref + + const close = () => { + setDraft(null) + queueMicrotask(onReturnFocus) + } + + const apply = () => { + if (!draft || invalidHref) return + onApply({ alt: draft.alt, href: normalizedHref }) + close() + } + + const handleKeyDown = (event: KeyboardEvent) => { + event.stopPropagation() + if (event.nativeEvent.isComposing) return + if (event.key === 'Enter') { + event.preventDefault() + apply() + } else if (event.key === 'Escape') { + event.preventDefault() + close() + } + } + + return ( +
event.stopPropagation()} + > + {draft ? ( +
+ + setDraft((current) => ({ ...(current ?? draft), alt: event.target.value })) + } + onKeyDown={handleKeyDown} + /> + + setDraft((current) => ({ ...(current ?? draft), href: event.target.value })) + } + onKeyDown={handleKeyDown} + /> + {invalidHref && ( + + )} +
+ + +
+
+ ) : ( +
+ setDraft({ alt, href })} + /> + {hasCustomSize && ( + { + onResetSize() + queueMicrotask(onReturnFocus) + }} + /> + )} +
+ )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx new file mode 100644 index 00000000000..ba8b33fd482 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx @@ -0,0 +1,145 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import type { ReactNodeViewProps } from '@tiptap/react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@tiptap/react', () => ({ + NodeViewWrapper: 'div', + ReactNodeViewRenderer: vi.fn(), +})) + +vi.mock('@/hooks/use-file-content-source', () => ({ + useFileContentSource: () => ({ + resolveImageSrc: (src: string) => src, + getImageDimensions: () => null, + }), +})) + +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/use-editor-editable', + () => ({ useEditorEditable: () => true }) +) + +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector', + () => ({ ImageInspector: () => null }) +) + +import { ResizableImageView } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image' + +let host: HTMLDivElement +let root: Root +const editor = { isEditable: true, isDestroyed: false, commands: { focus: vi.fn() } } + +beforeEach(() => { + editor.isEditable = true + editor.isDestroyed = false + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) +}) + +afterEach(() => { + act(() => root.unmount()) + host.remove() +}) + +function pointerEvent( + type: string, + { pointerId, clientX = 0, button = 0 }: { pointerId: number; clientX?: number; button?: number } +): Event { + const event = new Event(type, { bubbles: true, cancelable: true }) + Object.defineProperties(event, { + pointerId: { value: pointerId }, + clientX: { value: clientX }, + button: { value: button }, + pointerType: { value: 'touch' }, + }) + return event +} + +function renderImage(updateAttributes: ReturnType): HTMLButtonElement { + const props = { + node: { + attrs: { + src: '/image.png', + alt: '', + title: null, + width: null, + height: '100', + href: null, + }, + }, + updateAttributes, + selected: true, + editor, + } as unknown as ReactNodeViewProps + act(() => root.render()) + const image = host.querySelector('img') + const handle = host.querySelector('button[aria-label="Resize image"]') + if (!image || !handle) throw new Error('Resizable image did not render') + Object.defineProperty(image, 'offsetWidth', { configurable: true, value: 200 }) + Object.assign(handle, { + setPointerCapture: vi.fn(), + hasPointerCapture: vi.fn(() => true), + releasePointerCapture: vi.fn(), + }) + return handle +} + +describe('ResizableImageView', () => { + it('keeps the width automatic when only an explicit height is stored', () => { + renderImage(vi.fn()) + const image = host.querySelector('img') + if (!image) throw new Error('Missing image') + Object.defineProperties(image, { + naturalWidth: { configurable: true, value: 400 }, + naturalHeight: { configurable: true, value: 200 }, + }) + act(() => image.dispatchEvent(new Event('load'))) + + expect(image.style.height).toBe('100px') + expect(image.style.width).toBe('') + }) + + it('commits one proportional width change and clears a stale explicit height', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 7, clientX: 160 }))) + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 7, clientX: 160 }))) + + expect(updateAttributes).toHaveBeenCalledOnce() + expect(updateAttributes).toHaveBeenCalledWith({ width: '260', height: null }) + }) + + it('ignores unrelated pointers and cancels without mutating document attributes', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 8, clientX: 180 }))) + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 8, clientX: 180 }))) + act(() => window.dispatchEvent(pointerEvent('pointercancel', { pointerId: 7 }))) + expect(updateAttributes).not.toHaveBeenCalled() + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 9, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 9, clientX: 140 }))) + act(() => window.dispatchEvent(new Event('blur'))) + expect(updateAttributes).not.toHaveBeenCalled() + }) + + it('does not commit a resize after live editing becomes unavailable', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 7, clientX: 160 }))) + editor.isEditable = false + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 7, clientX: 160 }))) + + expect(updateAttributes).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts index dccc926d6b4..38f39113f25 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts @@ -1,5 +1,6 @@ import type { JSONContent } from '@tiptap/core' import { Image } from '@tiptap/extension-image' +import { Lexer, Tokenizer } from 'marked' /** * React-free schema half of the image node. Lives apart from {@link ./image} (its React resize node @@ -16,9 +17,6 @@ import { Image } from '@tiptap/extension-image' * the whole construct ourselves and hang the link target on the image node's `href` attribute, so it * round-trips losslessly (and the file stays editable rather than opening read-only). */ -const LINKED_IMAGE_RE = - /^\[!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/ - /** Escape a value for safe interpolation into a double-quoted HTML attribute. */ function escapeAttr(value: string): string { return value @@ -28,16 +26,32 @@ function escapeAttr(value: string): string { .replace(/>/g, '>') } +function decodeAttr(value: string): string { + return value + .replace(/"/g, '"') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&') +} + +function imageAttrsFromHtml(raw: string): Record | null { + if (!/^ = {} + const attributePattern = /([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g + for (const match of raw.matchAll(attributePattern)) { + attrs[match[1].toLowerCase()] = decodeAttr(match[2] ?? match[3] ?? match[4] ?? '') + } + return typeof attrs.src === 'string' ? attrs : null +} + /** * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to * preserve its dimensions. Unsized images stay clean `![alt](src)`. An image with an `href` is * wrapped in a markdown link so a linked badge round-trips as `[![alt](src)](href)`. * - * A *sized **and** linked* image is the one case markdown can't represent: the linked-image tokenizer - * only recognizes `[![alt](src)](href)`, so emitting `[](href)` would silently drop the link on - * reparse (and the round-trip-safety probe wouldn't catch it). We keep the link and fall back to the - * unsized `[![alt](src)](href)` form — the link matters more than the exact dimensions for a badge. + * Sized linked images use the standard `[](href)` combination, preserving both dimensions and + * link semantics while remaining readable by ordinary Markdown renderers. */ function imageMarkdown(node: JSONContent): string { const attrs = node.attrs ?? {} @@ -49,9 +63,8 @@ function imageMarkdown(node: JSONContent): string { const width = attrs.width const height = attrs.height let image: string - if ((width || height) && !href) { - const parts = [`src="${escapeAttr(src)}"`] - if (alt) parts.push(`alt="${escapeAttr(alt)}"`) + if (width || height) { + const parts = [`src="${escapeAttr(src)}"`, `alt="${escapeAttr(alt)}"`] if (title) parts.push(`title="${escapeAttr(title)}"`) if (width) parts.push(`width="${escapeAttr(String(width))}"`) if (height) parts.push(`height="${escapeAttr(String(height))}"`) @@ -67,7 +80,8 @@ function imageMarkdown(node: JSONContent): string { // Escape `"`/`\` so an href title can't break out of the `[…](href "title")` syntax (mirrors the // image title escaping above). const hrefTitlePart = hrefTitle ? ` "${hrefTitle.replace(/["\\]/g, '\\$&')}"` : '' - return `[${image}](${href}${hrefTitlePart})` + const safeHref = /[\s()]/.test(href) ? `<${href}>` : href + return `[${image}](${safeHref}${hrefTitlePart})` } interface MarkdownImageToken { @@ -78,6 +92,8 @@ interface MarkdownImageToken { /** Built-in image token holds the source URL here; our linked token holds the link target. */ href?: string hrefTitle?: string | null + width?: string | null + height?: string | null /** Built-in image token holds the alt text here. */ text?: string } @@ -94,6 +110,8 @@ function parseImageToken(token: MarkdownImageToken): JSONContent { title: token.title ?? null, href: token.href ?? null, hrefTitle: token.hrefTitle ?? null, + width: token.width ?? null, + height: token.height ?? null, } : { src: token.href ?? '', @@ -101,6 +119,8 @@ function parseImageToken(token: MarkdownImageToken): JSONContent { title: token.title ?? null, href: null, hrefTitle: null, + width: null, + height: null, }, } } @@ -141,18 +161,44 @@ export const MarkdownImage = Image.extend({ markdownTokenizer: { name: 'image', level: 'inline', - start: (src: string) => src.indexOf('[!['), + start: (src: string) => { + const markdown = src.indexOf('[![') + const html = src.search(/\[ { - const match = LINKED_IMAGE_RE.exec(src) - if (!match) return undefined + if (!src.startsWith('[![') && !/^\[`. */ -function ResizableImageView({ node, updateAttributes, selected, editor }: ReactNodeViewProps) { +export function ResizableImageView({ + node, + updateAttributes, + selected, + editor, +}: ReactNodeViewProps) { const source = useFileContentSource() const imageRef = useRef(null) const dragAbortRef = useRef(null) @@ -37,6 +43,7 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN alt?: string title?: string width?: string | null + height?: string | null href?: string | null } @@ -54,8 +61,12 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN const startResize = (event: React.PointerEvent) => { event.preventDefault() + if (event.button !== 0 || dragging) return const image = imageRef.current if (!image) return + const handle = event.currentTarget + const pointerId = event.pointerId + handle.setPointerCapture(pointerId) const startX = event.clientX const startWidth = image.offsetWidth setDragging(true) @@ -67,29 +78,51 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN window.addEventListener( 'pointermove', (move) => { + if (move.pointerId !== pointerId) return const next = Math.max(MIN_WIDTH, Math.round(startWidth + (move.clientX - startX))) dragWidthRef.current = next setDragWidth(next) }, { signal } ) - const finish = () => { + const finish = (commit: boolean) => { const finalWidth = dragWidthRef.current setDragging(false) setDragWidth(null) dragWidthRef.current = null + if (handle.hasPointerCapture(pointerId)) handle.releasePointerCapture(pointerId) controller.abort() - if (finalWidth !== null) updateAttributes({ width: String(finalWidth) }) + if (commit && finalWidth !== null && editor.isEditable && !editor.isDestroyed) { + updateAttributes({ width: String(finalWidth), height: null }) + } } - window.addEventListener('pointerup', finish, { signal }) - window.addEventListener('pointercancel', finish, { signal }) + window.addEventListener( + 'pointerup', + (up) => { + if (up.pointerId === pointerId) finish(true) + }, + { signal } + ) + window.addEventListener( + 'pointercancel', + (cancel) => { + if (cancel.pointerId === pointerId) finish(false) + }, + { signal } + ) + window.addEventListener('blur', () => finish(false), { signal }) } const committedWidth = attrs.width - ? BARE_PIXEL_WIDTH.test(attrs.width) + ? BARE_PIXEL_SIZE.test(attrs.width) ? `${attrs.width}px` : attrs.width : undefined + const committedHeight = attrs.height + ? BARE_PIXEL_SIZE.test(attrs.height) + ? `${attrs.height}px` + : attrs.height + : undefined // Stored intrinsic dimensions reserve the box on the very first render. Memoized on the src (not the // live drag width) so a resize drag never re-scans the file list. Falls back to what we measured on // load this session for a first-ever view the metadata hasn't caught up on. @@ -104,15 +137,18 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN const displayWidth = dragWidth !== null ? `${dragWidth}px` - : (committedWidth ?? (intrinsicDimensions ? `${intrinsicDimensions.width}px` : undefined)) + : (committedWidth ?? + (!committedHeight && intrinsicDimensions ? `${intrinsicDimensions.width}px` : undefined)) // width + aspect-ratio (with `max-w-full`/`h-auto` from the class list) reserves a responsive box the // image can't reflow into, per the CLS-avoidance pattern for known-ratio responsive images. React drops // the undefined keys, so an unmeasured image simply gets no reservation (its prior behavior). const imageStyle: CSSProperties = { width: displayWidth, - aspectRatio: intrinsicDimensions - ? `${intrinsicDimensions.width} / ${intrinsicDimensions.height}` - : undefined, + height: dragWidth === null ? committedHeight : undefined, + aspectRatio: + intrinsicDimensions && !committedHeight + ? `${intrinsicDimensions.width} / ${intrinsicDimensions.height}` + : undefined, } // Sanitize the linked-image target before rendering the anchor — a parsed markdown href is @@ -182,11 +218,25 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN image )} {editable && (selected || dragging) && ( - + )} + {editable && selected && !dragging && ( + updateAttributes({ alt, href: href || null })} + onResetSize={() => updateAttributes({ width: null, height: null })} + onReturnFocus={() => editor.commands.focus()} /> )} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome.ts index 83a10d3f63c..e037451cc9b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome.ts @@ -6,4 +6,4 @@ /** The floating toolbar: bordered card, popover layer, with the enter animation. */ export const BUBBLE_MENU_CLASS = - 'fade-in-0 z-[var(--z-popover)] flex animate-in items-center gap-0.5 rounded-lg border border-[var(--border)] bg-[var(--bg)] p-1 shadow-xs duration-150 ease-out motion-reduce:animate-none' + 'scrollbar-none fade-in-0 z-[var(--z-popover)] flex max-w-[calc(100vw_-_1rem)] animate-in items-center gap-0.5 overflow-x-auto rounded-lg border border-[var(--border)] bg-[var(--bg)] p-1 shadow-xs duration-150 ease-out motion-reduce:animate-none' diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx index 1ba7457cc4b..899610ce185 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx @@ -16,6 +16,7 @@ import { TextQuote, Unlink, } from '@sim/emcn/icons' +import { getMarkRange } from '@tiptap/core' import { PluginKey, type SelectionBookmark, @@ -58,6 +59,16 @@ function revealBubbleMenu(editor: Editor, key: PluginKey): void { editor.commands.setMeta(key, 'updatePosition') } +/** Captures selected text, or the complete link mark when the caret sits inside one. */ +function linkSelectionBookmark(editor: Editor): SelectionBookmark | null { + const { doc, selection, schema } = editor.state + if (!selection.empty) return selection.getBookmark() + const linkType = schema.marks.link + if (!linkType) return null + const range = getMarkRange(selection.$from, linkType) + return range ? TextSelection.create(doc, range.from, range.to).getBookmark() : null +} + interface EditorBubbleMenuProps { editor: Editor /** The editor's scrollable viewport, so the toolbar repositions with the selection as the pane scrolls. */ @@ -145,8 +156,8 @@ export function EditorBubbleMenu({ /** * Linear-style reveal: the toolbar stays hidden while the pointer is down (the drag gate in - * `shouldShow`) and surfaces on release. `mouseup`/`blur` listen on `window` so a release outside - * the editor — or off-screen, where no `mouseup` fires — still clears the drag flag; otherwise it + * `shouldShow`) and surfaces on release. `pointerup`/`pointercancel`/`blur` listen on `window` so a + * release outside the editor — or a cancelled touch gesture — still clears the drag flag; otherwise it * could wedge `true` and suppress the toolbar for later keyboard selections. */ useEffect(() => { @@ -163,19 +174,23 @@ export function EditorBubbleMenu({ const onWindowBlur = () => { isPointerDownRef.current = false } - dom.addEventListener('mousedown', onPointerDown) - window.addEventListener('mouseup', onPointerUp) + dom.addEventListener('pointerdown', onPointerDown) + window.addEventListener('pointerup', onPointerUp) + window.addEventListener('pointercancel', onWindowBlur) window.addEventListener('blur', onWindowBlur) return () => { - dom.removeEventListener('mousedown', onPointerDown) - window.removeEventListener('mouseup', onPointerUp) + dom.removeEventListener('pointerdown', onPointerDown) + window.removeEventListener('pointerup', onPointerUp) + window.removeEventListener('pointercancel', onWindowBlur) window.removeEventListener('blur', onWindowBlur) } }, [editor, bubbleMenuKey]) const openLinkEditor = () => { if (!editor.isEditable || editor.isActive('codeBlock') || editor.isActive('code')) return - linkRangeRef.current = editor.state.selection.getBookmark() + const bookmark = linkSelectionBookmark(editor) + if (!bookmark) return + linkRangeRef.current = bookmark setLinkValue(editor.getAttributes('link').href ?? '') } @@ -192,10 +207,11 @@ export function EditorBubbleMenu({ ) return if (event.key?.toLowerCase() !== 'k') return - const { from, to } = editor.state.selection - if (from === to || editor.isActive('codeBlock') || editor.isActive('code')) return + if (editor.isActive('codeBlock') || editor.isActive('code')) return + const bookmark = linkSelectionBookmark(editor) + if (!bookmark) return event.preventDefault() - linkRangeRef.current = editor.state.selection.getBookmark() + linkRangeRef.current = bookmark setLinkValue(editor.getAttributes('link').href ?? '') } dom.addEventListener('keydown', openLinkOnShortcut) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx index e57463a6708..330f548858d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx @@ -269,6 +269,26 @@ describe('real editor BubbleMenu keyboard integration', () => { expect(key(remove, 'Tab').defaultPrevented).toBe(false) }) + it('edits the complete existing link from a collapsed caret with Cmd/Ctrl+K', async () => { + select('format') + act(() => editor.commands.setLink({ href: 'https://example.com/original' })) + select('format', true) + + expect(key(editor.view.dom, 'k', { ctrlKey: true }).defaultPrevented).toBe(true) + await frame() + const input = linkGroup().querySelector('input[aria-label="Link URL"]') + expect(input).not.toBeNull() + if (!input) return + + changeUrl(input, 'https://example.com/replacement') + key(input, 'Enter') + await frame() + + const link = editor.view.dom.querySelector('a') + expect(link?.textContent).toBe('format') + expect(link?.getAttribute('href')).toBe('https://example.com/replacement') + }) + it('maps the captured link target through a prefix edit and an appended transaction', async () => { const input = await openLinkEditor() changeUrl(input, 'https://example.com/mapped') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx index 3ebcc2c6312..441ecee8a2b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx @@ -63,7 +63,7 @@ export function LinkUrlInput({ } }} placeholder='Paste or type a link…' - className='h-[28px] w-[220px] bg-transparent px-2 text-[var(--text-body)] text-small outline-hidden placeholder:text-[var(--text-subtle)]' + className='h-10 w-[220px] bg-transparent px-2 text-[var(--text-body)] text-small outline-hidden placeholder:text-[var(--text-subtle)] sm:h-[28px]' /> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx index e6bf381b8e8..340a43c1e50 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx @@ -44,4 +44,18 @@ describe('ToolbarButton', () => { expect(button?.className).toContain('size-[28px]') expect(button?.querySelector('svg')?.className.baseVal).toContain('size-[12px]') }) + + it('preserves the editor selection for mouse, pen, and touch activation', () => { + const host = renderButton() + const button = host.querySelector('button[aria-label="Bold"]') + expect(button).not.toBeNull() + if (!button) return + + for (const pointerType of ['mouse', 'pen', 'touch']) { + const event = new Event('pointerdown', { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'pointerType', { value: pointerType }) + act(() => button.dispatchEvent(event)) + expect(event.defaultPrevented).toBe(true) + } + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx index 3436913633f..4bdcb5b19d8 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx @@ -33,10 +33,10 @@ export function ToolbarButton({ aria-label={label} aria-pressed={isActive} disabled={disabled} - onMouseDown={(event) => event.preventDefault()} + onPointerDown={(event) => event.preventDefault()} onClick={onClick} className={cn( - 'size-[28px] focus-visible:bg-[var(--surface-hover)]', + 'size-10 focus-visible:bg-[var(--surface-hover)] sm:size-[28px]', !isActive && 'hover-hover:bg-[var(--surface-hover)]' )} > @@ -52,5 +52,5 @@ export function ToolbarButton({ /** Thin vertical separator between groups of {@link ToolbarButton}s. */ export function ToolbarDivider() { - return
+ return
} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 55384a26ff1..7ae83d24c24 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -1,8 +1,9 @@ 'use client' -import { memo, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' -import { Chip, cn, toast } from '@sim/emcn' +import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' +import { Chip, ChipConfirmModal, cn, toast } from '@sim/emcn' import { FILE_DOC_SEED, type JoinFileDocError } from '@sim/realtime-protocol/file-doc' +import { getErrorMessage } from '@sim/utils/errors' import { PASTE_LIMITS, PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import type { Extensions, JSONContent, Range } from '@tiptap/core' import { isChangeOrigin } from '@tiptap/extension-collaboration' @@ -14,6 +15,7 @@ import { buildFileSelectionLabel, truncateSelectionText, } from '@/lib/copilot/chat/selection-context' +import { saveBlob } from '@/lib/uploads/client/download' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractEmbeddedFileRef, extractImgSrcs } from '@/lib/uploads/utils/embedded-image-ref' import { FindBar } from '@/app/workspace/[workspaceId]/components' @@ -30,6 +32,7 @@ import { beginAgentStream, endAgentStream, } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown' +import type { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' import { isCollabReady } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness' import { useFileDocCollaboration } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration' import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' @@ -102,6 +105,81 @@ function warnRichMarkdownPasteLimit(reason?: 'paste' | 'formatting') { }) } +interface CollaborationFailureBannerProps { + failure: JoinFileDocError + provider: FileDocProvider | null + onDownloadDraft: () => void +} + +/** Keeps a stale local draft recoverable when live editing must stop rather than silently retry. */ +function CollaborationFailureBanner({ + failure, + provider, + onDownloadDraft, +}: CollaborationFailureBannerProps) { + const [confirmDiscardOpen, setConfirmDiscardOpen] = useState(false) + const [isDiscarding, setIsDiscarding] = useState(false) + const requiresDraftRecovery = + failure.code === 'DOCUMENT_REPLACED' || + failure.code === 'PENDING_UPDATE_LIMIT' || + failure.code === 'INVALID_UPDATE' + const accessLost = failure.code === 'ACCESS_REVOKED' || failure.code === 'ACCESS_DENIED' + const discardPendingChangesAndReload = async () => { + setIsDiscarding(true) + try { + await provider?.discardPendingChanges() + window.location.reload() + } catch (error) { + toast.error(getErrorMessage(error, 'Could not discard the stale local recovery copy.')) + setIsDiscarding(false) + } + } + + const message = accessLost + ? 'You no longer have edit access to this document.' + : failure.code === 'DOCUMENT_REPLACED' + ? 'The live document changed while this tab was disconnected. Your local draft is preserved.' + : failure.code === 'PENDING_UPDATE_LIMIT' + ? 'Local edits exceeded this browser’s safe recovery limit. Download your draft before reloading.' + : failure.code === 'INVALID_UPDATE' + ? 'A local edit could not be synchronized safely. Download your draft before reloading.' + : failure.code === 'SCHEMA_VERSION_MISMATCH' + ? 'This app version cannot edit the live document. Reload to update.' + : 'Live editing could not connect. Reload to try again.' + + return ( +
+

{message}

+ Download local draft + {requiresDraftRecovery ? ( + <> + setConfirmDiscardOpen(true)}> + Discard draft + + void discardPendingChangesAndReload(), + pending: isDiscarding, + pendingLabel: 'Discarding...', + }} + /> + + ) : !accessLost ? ( + window.location.reload()}>Reload + ) : null} +
+ ) +} + /** * The editor's reading column — the centered, padded surface both the live editor and the read-only * {@link ReadOnlyPlaceholder} render into, so the two are geometrically identical and the placeholder → @@ -325,6 +403,7 @@ function RichMarkdownSurface({ onDeriveTitleFromHeading={onDeriveTitleFromHeading} enableFind={enableFind} onEditSource={onEditSource} + onDownloadDraft={downloadDraft} /> ) @@ -361,6 +440,7 @@ interface LoadedRichMarkdownEditorProps { /** See {@link RichMarkdownEditorProps.enableFind}. */ enableFind: boolean onEditSource?: () => void + onDownloadDraft: () => void } type CollaborationStatus = 'connecting' | 'ready' | 'reconnecting' | 'fatal' @@ -397,6 +477,7 @@ export function LoadedRichMarkdownEditor({ onDeriveTitleFromHeading, enableFind, onEditSource, + onDownloadDraft, }: LoadedRichMarkdownEditorProps) { /** Whether this editor mounted mid-stream — if so it starts empty and syncs streamed chunks until settle. */ const [streamingAtMount] = useState(isStreaming) @@ -446,6 +527,7 @@ export function LoadedRichMarkdownEditor({ const isEditable = canEdit && !isStreaming && (settled?.verdict ?? false) && collabReady const collaboration = useFileDocCollaboration({ + workspaceId, fileId: file.id, userId, userName, @@ -921,7 +1003,9 @@ export function LoadedRichMarkdownEditor({ * fire an observer and the editor would stay editable on a document the provider has abandoned. */ const onJoinError = (error: JoinFileDocError) => { - if (error.retryable === false) seedFromLoaded() + if (error.retryable === false) { + seedFromLoaded() + } report() } @@ -1311,9 +1395,21 @@ export function LoadedRichMarkdownEditor({ useSelectionCopyBridge(containerRef, buildSelectionContext, workspaceId) + const downloadLiveDraft = () => { + if (!editor) { + onDownloadDraft() + return + } + const body = postProcessSerializedMarkdown(editor.getMarkdown()) + const markdown = applyFrontmatter(saveFrontmatterResolverRef.current(), body) + saveBlob(new Blob([markdown], { type: 'text/markdown;charset=utf-8' }), file.name) + } + /** Use the stored-content placeholder only while the live document is bootstrapping. */ const showPlaceholder = collaborationEnabled && collabStatus === 'connecting' const showReconnecting = collaborationEnabled && collabStatus === 'reconnecting' + const collabFailure = collaboration?.provider?.joinError ?? null + const showCollabFailure = collaborationEnabled && collabStatus === 'fatal' ? collabFailure : null /** * Find is off while the placeholder is up. The text on screen then belongs to the placeholder's own @@ -1322,6 +1418,28 @@ export function LoadedRichMarkdownEditor({ * native find reads the rendered placeholder correctly; it becomes ours once the seed lands. */ const find = useMarkdownFind({ editor, enabled: enableFind && !showPlaceholder }) + const replaceControls = useMemo( + () => + isEditable + ? { + value: find.replacement, + onChange: find.setReplacement, + onReplace: find.replaceCurrent, + onReplaceAll: find.replaceAll, + canReplace: find.count > 0, + canReplaceAll: find.count > 0 && !find.truncated, + } + : undefined, + [ + find.count, + find.replaceAll, + find.replaceCurrent, + find.replacement, + find.setReplacement, + find.truncated, + isEditable, + ] + ) return ( // The find bar is a sibling of the scroller, not a child: pinned inside `containerRef` it would @@ -1347,6 +1465,13 @@ export function LoadedRichMarkdownEditor({ Reconnecting…
)} + {showCollabFailure && ( + + )} {find.isOpen && ( )}
{ isRoundTripSafe('[![build](https://img.shields.io/badge/x-green)](https://ci.example.com)') ).toBe(true) expect(isRoundTripSafe('[![alt](https://e.com/i.png "t")](https://e.com "h")')).toBe(true) + expect( + isRoundTripSafe( + '[](https://e.com)' + ) + ).toBe(true) }) it('passes inline code without an interior backtick', () => { @@ -146,6 +151,15 @@ describe('isRoundTripSafe', () => { expect(isRoundTripSafe('')).toBe(true) }) + it('keeps HTML images with unsupported attributes in source mode', () => { + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('a')).toBe( + true + ) + }) + it.each([ '| |\n| --- |\n| body |', '| header |\n| --- |\n| |', diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts index bab3759891f..a7fb777e2f2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts @@ -42,6 +42,23 @@ function stripCode(content: string): string { } const fidelityLexer = new Marked({ gfm: true }) +const SUPPORTED_IMAGE_ATTRIBUTES = new Set(['src', 'alt', 'title', 'width', 'height']) + +/** + * The image node deliberately models only the attributes it can render and serialize. An HTML image + * carrying anything else must stay in source mode; comparing only its `src` would declare a stable but + * lossy conversion safe after the unsupported attribute had already disappeared. + */ +function hasUnsupportedHtmlImageAttribute(content: string): boolean { + for (const image of content.matchAll(/]*)>/gi)) { + const attributes = image[1] + const pattern = /(?:^|\s)([^\s=/>]+)(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>`]+))?/g + for (const attribute of attributes.matchAll(pattern)) { + if (!SUPPORTED_IMAGE_ATTRIBUTES.has(attribute[1].toLowerCase())) return true + } + } + return false +} function imageSources(token: Token): string[] { if (token.type === 'image') return [token.href] @@ -60,11 +77,13 @@ function imageSources(token: Token): string[] { function inspectMarkdownFidelity(content: string) { const targets = new Map() let hasTaskReference = false + let hasTableHtmlImage = false const add = (kind: 'image' | 'linkedImage', ...destinations: string[]) => { const target = JSON.stringify([kind, ...destinations.map(decodeHtmlEntities)]) targets.set(target, (targets.get(target) ?? 0) + 1) } fidelityLexer.walkTokens(fidelityLexer.lexer(splitFrontmatter(content).body), (token) => { + if (token.type === 'table' && / { @@ -83,7 +102,7 @@ function inspectMarkdownFidelity(content: string) { } } }) - return { targets, hasTaskReference } + return { targets, hasTaskReference, hasTableHtmlImage } } /** @@ -141,9 +160,10 @@ export function isRoundTripSafe(content: string): boolean { const stripped = stripCode(content) if (STABLE_LOSS_PATTERNS.some((pattern) => pattern.test(stripped))) return false if (hasOrphanReferenceDefinition(stripped)) return false + if (hasUnsupportedHtmlImageAttribute(stripped)) return false try { const source = inspectMarkdownFidelity(content) - if (source.hasTaskReference) return false + if (source.hasTaskReference || source.hasTableHtmlImage) return false const once = serializeMarkdownDocument(content) const serialized = inspectMarkdownFidelity(once) for (const [target, count] of source.targets) { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts index 961fd1d86df..bcc50b243eb 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts @@ -301,6 +301,52 @@ describe('editor markdown round-trip', () => { expect(roundTrip('![a](https://e.com/i.png)')).toContain('![a](https://e.com/i.png)') }) + it('round-trips every sized linked-image attribute without dropping dimensions', () => { + const source = + '[](https://e.com "Details")' + const out = roundTrip(source) + + expect(out).toContain('alt=""') + expect(out).toContain('width="320" height="180"') + expect(out).toContain('](https://e.com "Details")') + expect(roundTrip(out)).toBe(out) + }) + + it('uses empty alt text when a linked HTML image has no alt attribute', () => { + const source = '[](https://e.com)' + const out = roundTrip(source) + + expect(out).toContain('alt=""') + expect(out).not.toContain('alt="<img') + expect(roundTrip(out)).toBe(out) + }) + + it('round-trips linked images with escaped alt text and angle-bracket destinations', () => { + const source = '[![a\\]b]()]( "Details")' + const out = roundTrip(source) + + expect(out).toContain('a\\]b') + expect(out).toContain('') + expect(out).toContain('') + expect(roundTrip(out)).toBe(out) + }) + + it('parses a paragraph of adjacent links and linked images without recursive suffix scans', () => { + const links = Array.from( + { length: 80 }, + (_, index) => `[Link ${index}](https://e.com/${index})` + ) + const images = Array.from( + { length: 40 }, + (_, index) => `[![Image ${index}](https://e.com/${index}.png)](https://e.com/${index})` + ) + const out = roundTrip([...links, ...images].join(' ')) + + for (const link of links) expect(out).toContain(link) + for (const image of images) expect(out).toContain(image) + expect(roundTrip(out)).toBe(out) + }) + it('preserves a sized base64 image and escapes quotes in attributes', () => { const dataUrl = '' expect(roundTrip(dataUrl)).toContain('data:image/png;base64,iVBORw0KGgo=') diff --git a/apps/sim/lib/collab-doc/converter.test.ts b/apps/sim/lib/collab-doc/converter.test.ts index f31796d2a82..20fd26e6436 100644 --- a/apps/sim/lib/collab-doc/converter.test.ts +++ b/apps/sim/lib/collab-doc/converter.test.ts @@ -40,6 +40,7 @@ const SAMPLES = [ 'A footnote reference[^1].\n\n[^1]: the footnote body.', 'Before.\n\n
untouched raw html
\n\nAfter.', '- [ ] todo\n- [x] done', + '[](https://e.com)', ] beforeAll(() => { diff --git a/apps/sim/lib/core/outbox/service.test.ts b/apps/sim/lib/core/outbox/service.test.ts index 9817363630c..64c128226e0 100644 --- a/apps/sim/lib/core/outbox/service.test.ts +++ b/apps/sim/lib/core/outbox/service.test.ts @@ -255,15 +255,28 @@ describe('processOutboxEvents — empty / no handler', () => { }) }) - it('dead-letters events with no registered handler', async () => { + it('retries events with no registered handler during rolling deployments', async () => { queueTableRows(outboxEvent, [makePendingRow({ eventType: 'unknown.event' })]) holdLease() const result = await processOutboxEvents({}) + expect(result.retried).toBe(1) + const retry = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) + expect(retry).toBeDefined() + expect(retry?.attempts).toBe(1) + }) + + it('dead-letters a missing handler after the configured retry budget', async () => { + queueTableRows(outboxEvent, [ + makePendingRow({ eventType: 'unknown.event', attempts: 2, maxAttempts: 3 }), + ]) + holdLease() + + const result = await processOutboxEvents({}) + expect(result.deadLettered).toBe(1) const terminal = updateSets().find((set) => set.status === 'dead_letter') - expect(terminal).toBeDefined() expect(terminal?.lastError).toMatch(/No handler registered/) }) }) diff --git a/apps/sim/lib/core/outbox/service.ts b/apps/sim/lib/core/outbox/service.ts index a4ede767836..152caecfa5a 100644 --- a/apps/sim/lib/core/outbox/service.ts +++ b/apps/sim/lib/core/outbox/service.ts @@ -568,17 +568,15 @@ async function runHandler( const handler = handlers[event.eventType] if (!handler) { - logger.error('No handler registered for outbox event type', { + const reason = `No handler registered for event type '${event.eventType}'` + logger.warn('No handler registered for outbox event type; scheduling a bounded retry', { eventId: event.id, eventType: event.eventType, }) - await updateIfLeaseHeld(event, { - status: 'dead_letter', - lastError: `No handler registered for event type '${event.eventType}'`, - processedAt: new Date(), - lockedAt: null, + return scheduleDeferred(event, { + outcome: 'deferred', + reason, }) - return 'dead_letter' } try { diff --git a/apps/sim/lib/realtime/notify.test.ts b/apps/sim/lib/realtime/notify.test.ts index 98196492257..0955f0766bc 100644 --- a/apps/sim/lib/realtime/notify.test.ts +++ b/apps/sim/lib/realtime/notify.test.ts @@ -6,18 +6,21 @@ import { afterEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/core/utils/urls', () => ({ getSocketServerUrl: () => 'http://realtime' })) vi.mock('@/lib/core/config/env', () => ({ env: { INTERNAL_API_SECRET: 'secret' } })) -import { mergeEditIntoLiveFileDoc } from './notify' +import { applyEditToLiveFileDoc, invalidateLiveFileDoc } from '@/lib/realtime/notify' -describe('mergeEditIntoLiveFileDoc', () => { +describe('applyEditToLiveFileDoc', () => { afterEach(() => { vi.unstubAllGlobals() }) it('POSTs the edit to the realtime apply-edit endpoint with the api key', async () => { - const fetchMock = vi.fn().mockResolvedValue({ ok: true }) + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({ applied: true, status: 'applied' }), + }) vi.stubGlobal('fetch', fetchMock) - await mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) + await applyEditToLiveFileDoc('file-1', '# hello', { version: 42 }) expect(fetchMock).toHaveBeenCalledWith( 'http://realtime/api/file-doc/apply-edit', @@ -30,83 +33,58 @@ describe('mergeEditIntoLiveFileDoc', () => { ) }) - it('never throws when the realtime call fails (best-effort)', async () => { + it('throws when the realtime call fails so the outbox can retry', async () => { vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('socket pod down'))) - await expect( - mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) - ).resolves.toBeUndefined() + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).rejects.toThrow( + 'socket pod down' + ) }) - it('never throws on a non-2xx response', async () => { + it('surfaces retryable delivery failures to durable outbox callers', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, status: 503 })) - await expect( - mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) - ).resolves.toBeUndefined() + + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).rejects.toThrow( + 'status 503' + ) }) - it('a later durable merge waits for an in-flight earlier one, then applies last', async () => { - let resolveFirst: (value: { ok: boolean }) => void = () => {} - const fetchMock = vi - .fn() - .mockImplementationOnce(() => new Promise((resolve) => (resolveFirst = resolve))) - .mockResolvedValue({ ok: true }) - vi.stubGlobal('fetch', fetchMock) + it('returns the relay reconciliation status to durable outbox callers', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({ applied: false, status: 'no-live-room' }), + }) + ) - const first = mergeEditIntoLiveFileDoc('file-durable', 'earlier', { version: 99 }) // in flight - await Promise.resolve() - const durable = mergeEditIntoLiveFileDoc('file-durable', 'final content', { version: 100 }) - await Promise.resolve() - await Promise.resolve() + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).resolves.toEqual({ + applied: false, + status: 'no-live-room', + }) + }) +}) - // The later write waits for the in-flight earlier one → its fetch has not fired yet, so it cannot be - // reordered before a straggler and cannot be clobbered by one. - expect(fetchMock).toHaveBeenCalledTimes(1) +describe('invalidateLiveFileDoc', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) - resolveFirst({ ok: true }) - await first - await durable + it('POSTs a durability-sensitive invalidation and surfaces delivery failures', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true }) + vi.stubGlobal('fetch', fetchMock) - // Only after the earlier merge completed does the later (final) merge apply — always last. - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(fetchMock.mock.calls[1][1].body).toBe( - JSON.stringify({ fileId: 'file-durable', markdown: 'final content', version: 100 }) - ) - }) + await invalidateLiveFileDoc('file-1', 42) - it('serializes concurrent durable writes to a file strictly in order', async () => { - const applied: number[] = [] - const resolvers: Array<() => void> = [] - vi.stubGlobal( - 'fetch', - vi.fn((_url: string, init: { body: string }) => { - applied.push(JSON.parse(init.body).version) - return new Promise<{ ok: boolean }>((resolve) => - resolvers.push(() => resolve({ ok: true })) - ) + expect(fetchMock).toHaveBeenCalledWith( + 'http://realtime/api/file-doc/invalidate', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ 'x-api-key': 'secret' }), + body: JSON.stringify({ fileId: 'file-1', version: 42 }), }) ) - const flush = async () => { - for (let i = 0; i < 6; i++) await Promise.resolve() - } - - const s = mergeEditIntoLiveFileDoc('file-order', 's', { version: 0 }) // in flight - await flush() - // Two later durable writes arrive while the first merge is in flight — both must chain, not both - // resume-and-fire concurrently. - const a = mergeEditIntoLiveFileDoc('file-order', 'a', { version: 1 }) - const b = mergeEditIntoLiveFileDoc('file-order', 'b', { version: 2 }) - await flush() - expect(applied).toEqual([0]) // A and B queued behind the in-flight first merge - - resolvers[0]() // finish first → A applies next (not B) - await flush() - expect(applied).toEqual([0, 1]) - - resolvers[1]() // finish A → B applies after A - await flush() - expect(applied).toEqual([0, 1, 2]) - - resolvers[2]() - await Promise.all([s, a, b]) + + fetchMock.mockResolvedValueOnce({ ok: false, status: 503 }) + await expect(invalidateLiveFileDoc('file-1', 42)).rejects.toThrow('status 503') }) }) diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index 2374a8d5580..df3e13db400 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -169,79 +169,73 @@ export async function notifyFolderResourceChanged( * How a durable live-doc merge is positioned on the file's monotonic version line. Omit `version` to * apply the merge without ordering it (legacy). */ -interface LiveFileDocMergeOrder { +export interface LiveFileDocMergeOrder { /** A durable write's `contentUpdatedAt` (epoch ms): applied only if newer than the version the doc * already incorporates, AND recorded as the synced version (the persist If-Match guard). */ version?: number } +export type LiveFileDocMergeStatus = 'applied' | 'no-live-room' | 'merge-unavailable' | 'stale' + +interface LiveFileDocMergeResponse { + applied: boolean + status: LiveFileDocMergeStatus +} + /** - * Best-effort: ask the realtime relay to merge a durable copilot/file write into a file's LIVE - * collaborative document, so open editors reconcile to it as a CRDT merge rather than the file changing - * underneath them, and a late joiner is seeded from it. No-op when no doc is (or was recently) live (the - * relay reports `applied: false`). The file itself is written durably by the caller regardless — this - * only drives the live view. Never throws. - * - * (Streaming copilot output is NOT merged here: the open editor applies the stream client-side as minimal - * CRDT diffs — see `applyStreamedMarkdownToLiveDoc` — which renders smoothly and broadcasts to peers. This - * merge is the stream-end durable reconcile, and by then it is usually a noop diff.) - * - * The former clobber gap — an open editor's autosave dropping this edit — is closed: a collaborative - * editor no longer client-autosaves (the relay persists the shared doc to markdown server-side), and the - * relay applies this merge THROUGH the shared Redis stream, so it reaches the live doc on whichever task - * holds it and can't go stale relative to this direct write. - * - * The caller awaits this so the fetch dispatches before the route handler returns. Bounded to - * {@link APPLY_EDIT_TIMEOUT_MS}, so it adds latency only when the socket pod is unreachable. - * - * `order.version` positions the merge so a stale write never regresses the doc: it applies only if newer - * than the version the doc already incorporates, and is recorded as the synced version. Ordering is - * enforced at two scales: within this process, merges for a file run on a single serialized chain (each - * chained after the current tail) so writes never apply concurrently; across processes the relay orders - * by that monotonic version under a cluster-wide lock. + * Applies one durable file version to the live collaboration document and surfaces delivery + * failures to callers that own a retry policy, such as the transactional outbox. */ -export async function mergeEditIntoLiveFileDoc( +export async function applyEditToLiveFileDoc( fileId: string, markdown: string, - order: LiveFileDocMergeOrder = {} -): Promise { - const tail = liveDocMergeChain.get(fileId) ?? Promise.resolve() - const run = tail.then(() => applyLiveFileDocMerge(fileId, markdown, order)) - liveDocMergeChain.set(fileId, run) - try { - await run - } finally { - if (liveDocMergeChain.get(fileId) === run) liveDocMergeChain.delete(fileId) + order: LiveFileDocMergeOrder = {}, + signal?: AbortSignal +): Promise { + const timeoutSignal = AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS) + const response = await fetch(`${getSocketServerUrl()}/api/file-doc/apply-edit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ fileId, markdown, version: order.version }), + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) + if (!response.ok) { + throw new Error(`Live document reconciliation failed with status ${response.status}`) } -} -/** Per file, the tail of the serialized merge chain (each merge applies after it); never rejects - * because {@link applyLiveFileDocMerge} never throws. Absent when the file's chain is idle. */ -const liveDocMergeChain = new Map>() + const result = (await response.json()) as unknown + if (typeof result !== 'object' || result === null) { + throw new Error('Live document reconciliation returned an invalid response') + } + const candidate = result as Partial + const validStatus = + candidate.status === 'applied' || + candidate.status === 'no-live-room' || + candidate.status === 'merge-unavailable' || + candidate.status === 'stale' + if (typeof candidate.applied !== 'boolean' || !validStatus) { + throw new Error('Live document reconciliation returned an invalid response') + } + return { applied: candidate.applied, status: candidate.status as LiveFileDocMergeStatus } +} -/** POST the merge to the relay. Never throws (a live-doc merge is best-effort). */ -async function applyLiveFileDocMerge( +/** + * Invalidates one live document after a durable replacement that cannot be merged into the rich + * editor. Unlike list notifications this is durability-sensitive and throws so the outbox retries. + */ +export async function invalidateLiveFileDoc( fileId: string, - markdown: string, - order: LiveFileDocMergeOrder + version: number, + signal?: AbortSignal ): Promise { - try { - const response = await fetch(`${getSocketServerUrl()}/api/file-doc/apply-edit`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, - // `version` (durable `contentUpdatedAt`) records the synced version the live doc now incorporates - // (the persist If-Match guard). JSON.stringify drops it when undefined (an unordered legacy merge). - body: JSON.stringify({ - fileId, - markdown, - version: order.version, - }), - signal: AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS), - }) - if (!response.ok) { - logger.warn('file-doc apply-edit failed', { fileId, status: response.status }) - } - } catch (error) { - logger.warn('file-doc apply-edit error', { fileId, error: getErrorMessage(error) }) + const timeoutSignal = AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS) + const response = await fetch(`${getSocketServerUrl()}/api/file-doc/invalidate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ fileId, version }), + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) + if (!response.ok) { + throw new Error(`Live document invalidation failed with status ${response.status}`) } } diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts new file mode 100644 index 00000000000..3c2bbc43ccb --- /dev/null +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts @@ -0,0 +1,197 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockApplyEditToLiveFileDoc, mockDownloadFile, mockInvalidateLiveFileDoc } = vi.hoisted( + () => ({ + mockApplyEditToLiveFileDoc: vi.fn(), + mockDownloadFile: vi.fn(), + mockInvalidateLiveFileDoc: vi.fn(), + }) +) + +vi.mock('@/lib/realtime/notify', () => ({ + applyEditToLiveFileDoc: mockApplyEditToLiveFileDoc, + invalidateLiveFileDoc: mockInvalidateLiveFileDoc, +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFile: mockDownloadFile, +})) + +import type { OutboxEventContext } from '@/lib/core/outbox/service' +import { + WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, + workspaceFileLiveDocOutboxHandlers, +} from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' + +const VERSION = new Date('2026-09-04T12:00:00.000Z') +const PAYLOAD = { + workspaceId: 'workspace-1', + fileId: 'file-1', + version: VERSION.getTime(), +} + +function context(): OutboxEventContext { + return { + eventId: 'event-1', + eventType: WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, + attempts: 0, + maxAttempts: 10, + signal: new AbortController().signal, + checkpointPayload: vi.fn(), + } +} + +function handler() { + const registered = workspaceFileLiveDocOutboxHandlers[WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT] + if (!registered) throw new Error('Workspace file live-document handler is not registered') + return registered +} + +describe('workspace file live-document outbox', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockDownloadFile.mockResolvedValue(Buffer.from('# Durable content')) + mockApplyEditToLiveFileDoc.mockResolvedValue({ applied: true, status: 'applied' }) + }) + + it('loads the committed version and reconciles it into the live document', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace/workspace-1/file.md', context: 'workspace' }) + ) + expect(mockApplyEditToLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + '# Durable content', + { version: VERSION.getTime() }, + expect.any(AbortSignal) + ) + }) + + it('completes a stale event without reading or regressing newer content', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: new Date(VERSION.getTime() + 1), + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).not.toHaveBeenCalled() + }) + + it('defers transient merge-lock contention for an outbox retry', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + mockApplyEditToLiveFileDoc.mockResolvedValueOnce({ + applied: false, + status: 'merge-unavailable', + }) + + await expect(handler()(PAYLOAD, context())).resolves.toEqual( + expect.objectContaining({ outcome: 'deferred' }) + ) + }) + + it('rejects malformed payloads before touching durable state', async () => { + await expect(handler()({ ...PAYLOAD, version: 0 }, context())).rejects.toThrow( + 'invalid version' + ) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('does not materialize files beyond the collaborative editor boundary', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 6 * 1024 * 1024, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + VERSION.getTime(), + expect.any(AbortSignal) + ) + }) + + it('invalidates a live markdown generation when the durable file changes type', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.bin', + name: 'file.bin', + type: 'application/octet-stream', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + VERSION.getTime(), + expect.any(AbortSignal) + ) + }) + + it('still invalidates after a later binary write supersedes the type-changing event', async () => { + const latestVersion = VERSION.getTime() + 1 + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.bin', + name: 'file.bin', + type: 'application/octet-stream', + sizeBytes: 100, + contentUpdatedAt: new Date(latestVersion), + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + latestVersion, + expect.any(AbortSignal) + ) + }) +}) diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts new file mode 100644 index 00000000000..b7dfe968f71 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts @@ -0,0 +1,116 @@ +import { db } from '@sim/db' +import { workspaceFiles } from '@sim/db/schema' +import { PASTE_LIMITS } from '@sim/utils/paste' +import { and, eq, isNull } from 'drizzle-orm' +import { + deferOutboxHandler, + enqueueOutboxEvent, + type OutboxHandler, + type OutboxHandlerRegistry, + processOutboxEventById, +} from '@/lib/core/outbox/service' +import { applyEditToLiveFileDoc, invalidateLiveFileDoc } from '@/lib/realtime/notify' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { isMarkdownFile } from '@/lib/uploads/utils/file-utils' + +export const WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT = 'workspace-file.live-doc.reconcile' + +interface WorkspaceFileLiveDocPayload { + workspaceId: string + fileId: string + version: number +} + +function parsePayload(payload: unknown): WorkspaceFileLiveDocPayload { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + throw new Error('Workspace file live-document outbox payload must be an object') + } + const candidate = payload as Partial + if (typeof candidate.workspaceId !== 'string' || candidate.workspaceId.length === 0) { + throw new Error('Workspace file live-document outbox payload is missing workspaceId') + } + if (typeof candidate.fileId !== 'string' || candidate.fileId.length === 0) { + throw new Error('Workspace file live-document outbox payload is missing fileId') + } + if ( + typeof candidate.version !== 'number' || + !Number.isSafeInteger(candidate.version) || + candidate.version <= 0 + ) { + throw new Error('Workspace file live-document outbox payload has an invalid version') + } + return candidate as WorkspaceFileLiveDocPayload +} + +const reconcileWorkspaceFileLiveDoc: OutboxHandler = async (rawPayload, context) => { + const payload = parsePayload(rawPayload) + context.signal.throwIfAborted() + const [file] = await db + .select({ + key: workspaceFiles.key, + name: workspaceFiles.originalName, + type: workspaceFiles.contentType, + sizeBytes: workspaceFiles.sizeBytes, + contentUpdatedAt: workspaceFiles.contentUpdatedAt, + }) + .from(workspaceFiles) + .where( + and( + eq(workspaceFiles.id, payload.fileId), + eq(workspaceFiles.workspaceId, payload.workspaceId), + eq(workspaceFiles.context, 'workspace'), + isNull(workspaceFiles.deletedAt) + ) + ) + .limit(1) + + if (!file) return + const currentVersion = file.contentUpdatedAt.getTime() + if (currentVersion < payload.version) { + throw new Error('Workspace file live-document reconciliation is ahead of durable content') + } + if ( + !isMarkdownFile(file) || + file.sizeBytes === null || + file.sizeBytes > PASTE_LIMITS.RICH_MARKDOWN_BYTES + ) { + /** Later binary writes do not enqueue reconciliation, so retire the latest unsupported version. */ + await invalidateLiveFileDoc(payload.fileId, currentVersion, context.signal) + return + } + if (currentVersion > payload.version) return + + const content = await downloadFile({ + key: file.key, + context: 'workspace', + maxBytes: PASTE_LIMITS.RICH_MARKDOWN_BYTES, + signal: context.signal, + }) + context.signal.throwIfAborted() + const result = await applyEditToLiveFileDoc( + payload.fileId, + content.toString('utf-8'), + { version: payload.version }, + context.signal + ) + if (result.status === 'merge-unavailable') { + return deferOutboxHandler('Live document merge slot is temporarily unavailable') + } +} + +export const workspaceFileLiveDocOutboxHandlers = { + [WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT]: reconcileWorkspaceFileLiveDoc, +} satisfies OutboxHandlerRegistry + +/** Enqueues live-document reconciliation in the same transaction as the durable file version. */ +export function enqueueWorkspaceFileLiveDocReconciliation( + executor: Pick, + payload: WorkspaceFileLiveDocPayload +): Promise { + return enqueueOutboxEvent(executor, WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, payload) +} + +/** Attempts a newly committed reconciliation immediately; the outbox worker owns retries. */ +export function processWorkspaceFileLiveDocReconciliationNow(eventId: string) { + return processOutboxEventById(eventId, workspaceFileLiveDocOutboxHandlers) +} diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 20a99b69163..3d79ea322c6 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -55,8 +55,13 @@ import { acquireFolderMutationLock } from '@/lib/folders/locks' import { parseFolderPath } from '@/lib/folders/paths' import { loadActiveFolderPathIndex, resolveFolderPathFromIndex } from '@/lib/folders/queries' import type { FolderIdScope } from '@/lib/folders/scope' -import { mergeEditIntoLiveFileDoc, notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { getServePathPrefix } from '@/lib/uploads' +import type { WorkspaceFileFolderRecord } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager' +import { + enqueueWorkspaceFileLiveDocReconciliation, + processWorkspaceFileLiveDocReconciliationNow, +} from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' import { EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, initializeWorkspaceFileSecretProvenanceInTx, @@ -87,7 +92,6 @@ import { import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import { isUuid } from '@/executor/constants' import type { UserFile } from '@/executor/types' -import type { WorkspaceFileFolderRecord } from './workspace-file-folder-manager' import { assertWorkspaceFileFolderTarget, buildWorkspaceFileFolderPathMap, @@ -1816,6 +1820,7 @@ export async function updateWorkspaceFileContent( oldKey: string sizeDiff: number updatedUsage: number | undefined + liveDocEventId: string | undefined } try { finalized = await db.transaction(async (tx) => { @@ -1926,11 +1931,23 @@ export async function updateWorkspaceFileContent( ) } + const liveDocEventId = + options?.syncLiveDoc !== false && + (isMarkdownFile({ type: currentFile.contentType, name: currentFile.originalName }) || + isMarkdownFile({ type: updatedFile.contentType, name: updatedFile.originalName })) + ? await enqueueWorkspaceFileLiveDocReconciliation(tx, { + workspaceId, + fileId, + version: updatedFile.contentUpdatedAt.getTime(), + }) + : undefined + return { file: updatedFile, oldKey: currentFile.key, sizeDiff, updatedUsage, + liveDocEventId, } }) } catch (finalizationError) { @@ -1949,22 +1966,25 @@ export async function updateWorkspaceFileContent( await cleanupWorkspaceStorageObject(finalized.oldKey, 'version replacement') } - // Stream this write into any open collaborative editor as a CRDT merge, so a copilot/tool edit - // shows up live instead of the file silently changing underneath the reader. Gated to markdown (the - // only format the collaborative editor renders) and best-effort (a no-op when nobody has the file - // open; never throws). This is the single chokepoint every external writer shares — the relay's own - // persist and empty-shell creates pass `syncLiveDoc: false` to stay out of it. - if ( - options?.syncLiveDoc !== false && - isMarkdownFile({ type: nextContentType, name: finalized.file.originalName }) - ) { - // Pass the new CONTENT version this write produced, so the relay records that its live doc now - // incorporates this durable version — the collab persist's optimistic-concurrency guard then won't - // treat this (already-merged) write as an out-of-band conflict. Must be the SAME field the CAS - // guards on (`contentUpdatedAt`), not `updatedAt`, or the relay's token wouldn't match the CAS. - await mergeEditIntoLiveFileDoc(fileId, content.toString('utf-8'), { - version: finalized.file.contentUpdatedAt.getTime(), - }) + if (finalized.liveDocEventId) { + try { + const result = await processWorkspaceFileLiveDocReconciliationNow(finalized.liveDocEventId) + if (result !== 'completed') { + logger.warn('Live document reconciliation deferred to outbox retry', { + workspaceId, + fileId, + eventId: finalized.liveDocEventId, + result, + }) + } + } catch (error) { + logger.warn('Live document reconciliation deferred after inline processing error', { + workspaceId, + fileId, + eventId: finalized.liveDocEventId, + error: getErrorMessage(error), + }) + } } const pathPrefix = getServePathPrefix() diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts index e084217a435..eb44f367899 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts @@ -4,6 +4,7 @@ import { workspaceFiles } from '@sim/db/schema' import { dbChainMockFns, resetDbChainMock } from '@sim/testing' import { describeError } from '@sim/utils/errors' +import { PASTE_LIMITS } from '@sim/utils/paste' import { eq } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -11,6 +12,7 @@ const { mockDecrementStorageUsageForBillingContextInTx, mockDeleteFile, mockEnqueueWorkspaceFileStorageCleanup, + mockEnqueueWorkspaceFileLiveDocReconciliation, mockGetWorkspaceWithOwner, mockHasCloudStorage, mockHeadObject, @@ -20,9 +22,9 @@ const { mockLoadActiveFolderPathIndex, mockInitializeWorkspaceFileSecretProvenanceInTx, mockMaybeNotifyStorageLimitForBillingContext, - mockMergeEditIntoLiveFileDoc, mockNotifyWorkspaceFilesChanged, mockProcessWorkspaceFileStorageCleanupNow, + mockProcessWorkspaceFileLiveDocReconciliationNow, mockResolveStorageBillingContext, mockResolveFolderPathFromIndex, mockResolveWorkspaceFileFolderTarget, @@ -32,6 +34,7 @@ const { mockDecrementStorageUsageForBillingContextInTx: vi.fn(), mockDeleteFile: vi.fn(), mockEnqueueWorkspaceFileStorageCleanup: vi.fn(), + mockEnqueueWorkspaceFileLiveDocReconciliation: vi.fn(), mockGetWorkspaceWithOwner: vi.fn(), mockHasCloudStorage: vi.fn(), mockHeadObject: vi.fn(), @@ -41,9 +44,9 @@ const { mockLoadActiveFolderPathIndex: vi.fn(), mockInitializeWorkspaceFileSecretProvenanceInTx: vi.fn(), mockMaybeNotifyStorageLimitForBillingContext: vi.fn(), - mockMergeEditIntoLiveFileDoc: vi.fn(), mockNotifyWorkspaceFilesChanged: vi.fn(), mockProcessWorkspaceFileStorageCleanupNow: vi.fn(), + mockProcessWorkspaceFileLiveDocReconciliationNow: vi.fn(), mockResolveStorageBillingContext: vi.fn(), mockResolveFolderPathFromIndex: vi.fn(), mockResolveWorkspaceFileFolderTarget: vi.fn(), @@ -59,10 +62,14 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () })) vi.mock('@/lib/realtime/notify', () => ({ - mergeEditIntoLiveFileDoc: mockMergeEditIntoLiveFileDoc, notifyWorkspaceFilesChanged: mockNotifyWorkspaceFilesChanged, })) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox', () => ({ + enqueueWorkspaceFileLiveDocReconciliation: mockEnqueueWorkspaceFileLiveDocReconciliation, + processWorkspaceFileLiveDocReconciliationNow: mockProcessWorkspaceFileLiveDocReconciliationNow, +})) + vi.mock('@/lib/billing/storage', () => ({ decrementStorageUsageForBillingContextInTx: mockDecrementStorageUsageForBillingContextInTx, incrementStorageUsageForBillingContextInTx: mockIncrementStorageUsageForBillingContextInTx, @@ -166,9 +173,10 @@ describe('workspace file metadata and storage accounting', () => { mockMaybeNotifyStorageLimitForBillingContext.mockResolvedValue(undefined) mockDeleteFile.mockResolvedValue(undefined) mockEnqueueWorkspaceFileStorageCleanup.mockResolvedValue('cleanup-event-1') - mockMergeEditIntoLiveFileDoc.mockResolvedValue(undefined) + mockEnqueueWorkspaceFileLiveDocReconciliation.mockResolvedValue('live-doc-event-1') mockNotifyWorkspaceFilesChanged.mockResolvedValue(undefined) mockProcessWorkspaceFileStorageCleanupNow.mockResolvedValue('completed') + mockProcessWorkspaceFileLiveDocReconciliationNow.mockResolvedValue('completed') mockReplaceWorkspaceFileSecretProvenanceInTx.mockResolvedValue(undefined) }) @@ -740,7 +748,7 @@ describe('workspace file metadata and storage accounting', () => { const MD_ROW = { ...FILE_ROW, originalName: 'note.md', contentType: 'text/markdown' } - it('streams a markdown overwrite into any open collaborative editor (the shared merge chokepoint)', async () => { + it('transactionally enqueues a markdown overwrite for live-document reconciliation', async () => { // Distinct updatedAt vs contentUpdatedAt so the assertion proves the merge carries the CONTENT // version (the persist If-Match token), not `updatedAt` — reverting that wiring would fail here. const updatedFile = { @@ -761,9 +769,14 @@ describe('workspace file metadata and storage accounting', () => { Buffer.from('# new content', 'utf-8') ) - expect(mockMergeEditIntoLiveFileDoc).toHaveBeenCalledWith(MD_ROW.id, '# new content', { + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(expect.anything(), { + workspaceId: MD_ROW.workspaceId, + fileId: MD_ROW.id, version: updatedFile.contentUpdatedAt.getTime(), }) + expect(mockProcessWorkspaceFileLiveDocReconciliationNow).toHaveBeenCalledWith( + 'live-doc-event-1' + ) }) it('does NOT merge when syncLiveDoc is false (the relay persist / empty-shell opt-out)', async () => { @@ -781,7 +794,7 @@ describe('workspace file metadata and storage accounting', () => { { syncLiveDoc: false } ) - expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).not.toHaveBeenCalled() }) it('does NOT merge a non-markdown write (the collaborative editor only renders markdown)', async () => { @@ -798,7 +811,57 @@ describe('workspace file metadata and storage accounting', () => { 'application/octet-stream' ) - expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).not.toHaveBeenCalled() + }) + + it('enqueues an oversized markdown write so an older live generation is invalidated', async () => { + const size = PASTE_LIMITS.RICH_MARKDOWN_BYTES + 1 + const updatedFile = { ...MD_ROW, size, sizeBytes: size } + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: MD_ROW.key }) + + await updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.alloc(size) + ) + + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(expect.anything(), { + workspaceId: MD_ROW.workspaceId, + fileId: MD_ROW.id, + version: updatedFile.contentUpdatedAt.getTime(), + }) + }) + + it('enqueues a markdown-to-binary replacement so an older live generation is invalidated', async () => { + const markdownByType = { ...MD_ROW, originalName: 'note.txt' } + const updatedFile = { + ...markdownByType, + contentType: 'application/octet-stream', + size: 12, + sizeBytes: 12, + } + dbChainMockFns.limit + .mockResolvedValueOnce([markdownByType]) + .mockResolvedValueOnce([markdownByType]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: markdownByType.key }) + + await updateWorkspaceFileContent( + markdownByType.workspaceId, + markdownByType.id, + markdownByType.userId, + Buffer.alloc(12), + 'application/octet-stream' + ) + + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(expect.anything(), { + workspaceId: markdownByType.workspaceId, + fileId: markdownByType.id, + version: updatedFile.contentUpdatedAt.getTime(), + }) }) it('writes when the expectedUpdatedAt optimistic-concurrency guard matches', async () => { diff --git a/packages/realtime-protocol/src/file-doc.test.ts b/packages/realtime-protocol/src/file-doc.test.ts index 48fd7db5b8a..ca1d8a495d1 100644 --- a/packages/realtime-protocol/src/file-doc.test.ts +++ b/packages/realtime-protocol/src/file-doc.test.ts @@ -9,5 +9,7 @@ describe('FILE_DOC_TIMEOUTS ordering invariants', () => { // The relay's `/seed` fetch must finish before the client's readiness deadline lapses into its // read-only fallback, or a late-but-successful seed can never reach the client. expect(FILE_DOC_TIMEOUTS.seedRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.readinessDeadlineMs) + expect(FILE_DOC_TIMEOUTS.seedRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.joinAckMs) + expect(FILE_DOC_TIMEOUTS.joinAckMs).toBeLessThan(FILE_DOC_TIMEOUTS.readinessDeadlineMs) }) }) diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index dde704bee73..705272dcb73 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -25,6 +25,13 @@ export const FILE_DOC_EVENTS = { LEAVE: 'leave-file-doc', /** Both directions: a framed Yjs message (binary), tagged by {@link FILE_DOC_MESSAGE_TYPE}. */ MESSAGE: 'file-doc-message', + /** + * Client → server: one idempotent batch of user-authored Yjs updates. Unlike the handshake and + * awareness channel, this event is acknowledged only after the shared stream accepts the batch. + */ + UPDATE: 'file-doc-update', + /** Server → client: the durable file was replaced outside this live document's generation. */ + INVALIDATED: 'file-doc-invalidated', /** * Server → client: the roster of collaborators currently in the document * ({@link FileDocPresence}), for the avatar stack. Identity is server-authenticated (from @@ -34,6 +41,12 @@ export const FILE_DOC_EVENTS = { PRESENCE: 'file-doc-presence', } as const +/** Schema assumed for peers from before schema negotiation was added. */ +export const FILE_DOC_LEGACY_SCHEMA_VERSION = 1 + +/** Current collaborative-document schema understood by this client and relay. */ +export const FILE_DOC_SCHEMA_VERSION = 1 + /** * The tag carried in the first varUint of a {@link FILE_DOC_EVENTS.MESSAGE} * payload — the standard Yjs websocket framing distinguishing a document-sync @@ -67,8 +80,8 @@ export const FILE_DOC_MESSAGE_TYPE = { * 1. **Never overwrite content with an unseeded doc.** The markdown-mirror autosave MUST be gated on * the document being both synced AND seeded — otherwise an empty/still-syncing doc could be saved * over the real file (the one true data-loss path). - * 2. **One provider per socket.** Destroy the previous provider before creating the next (document - * switch), so a stale provider's binary-frame listener can't apply another document's updates. + * 2. **One active file per shared socket.** Multiple providers may show the same file, but opening a + * different file must make older providers terminal before its unscoped binary frames can arrive. * 3. **Treat a fatal (`retryable: false`) join error as terminal.** Latch it and fall back to a * read-only view of the file's stored content — do not keep rejoining. The server auto-reclaims a * same-user client-id collision silently (the reconnecting socket succeeds), so `CLIENT_ID_IN_USE` @@ -121,10 +134,17 @@ export const FILE_DOC_TIMEOUTS = { seedRequestMs: 8_000, mergeRequestMs: 3_000, applyEditMs: 6_000, + joinAckMs: 10_000, + updateAckMs: 6_000, readinessDeadlineMs: 12_000, persistRequestMs: 8_000, } as const +export const FILE_DOC_LIMITS = { + /** Leaves framing and acknowledgement headroom under Socket.IO's 8 MiB event ceiling. */ + updateBytes: 6 * 1024 * 1024, +} as const + /** Client → server join request. `fileId` is the `workspace_files.id`. */ export interface JoinFileDocPayload { fileId: string @@ -134,6 +154,8 @@ export interface JoinFileDocPayload { * client — an authenticated peer cannot forge or clear another's presence. */ clientId: number + /** Optional during rolling deploys; absent peers use the original version-1 schema. */ + schemaVersion?: number } /** Server → client acceptance of a {@link FILE_DOC_EVENTS.JOIN}. */ @@ -141,6 +163,8 @@ export interface JoinFileDocSuccess { fileId: string /** The provider whose join was accepted. Optional while older relays are still deployed. */ clientId?: number + /** Whether this relay durably acknowledges client updates. Absent on older relays. */ + acknowledgedUpdates?: true /** * The identity of the document this room holds ({@link FILE_DOC_SEED.docIdKey}), so a client can tell * "the room I left" from "a document built in its place" BEFORE it syncs. Absent for a room whose doc @@ -148,6 +172,8 @@ export interface JoinFileDocSuccess { * exactly the case where there is nothing to compare and the client proceeds. */ docId?: string + /** Optional while older relays are still deployed. */ + schemaVersion?: number } /** Server → client rejection of a {@link FILE_DOC_EVENTS.JOIN}. */ @@ -165,6 +191,35 @@ export interface LeaveFileDocPayload { fileId: string } +/** Server → client invalidation after a durable replacement that cannot merge into the rich editor. */ +export interface FileDocInvalidated { + fileId: string + message: string +} + +/** A bounded, retry-safe batch of user-authored changes. */ +export interface FileDocUpdatePayload { + fileId: string + docId: string + updateId: string + update: Uint8Array +} + +/** Relay acknowledgement for a {@link FileDocUpdatePayload}. */ +export type FileDocUpdateAck = + | { status: 'accepted'; updateId: string } + | { + status: 'rejected' + updateId?: string + code: + | 'ACCESS_REVOKED' + | 'DOCUMENT_REPLACED' + | 'INVALID_UPDATE' + | 'NOT_JOINED' + | 'TEMPORARY_FAILURE' + retryable: boolean + } + /** One collaborator session in a {@link FileDocPresence} roster — server-authenticated identity. * Keyed per socket (session), not per user: the client excludes its OWN `socketId` and then * dedupes the rest per user for the avatar stack, so a second tab of the same account still From 166fe55c871d806f0cb791cfcd29b823b09fc608 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 5 Sep 2026 10:55:43 -0700 Subject: [PATCH 2/9] fix(files): harden recovery and editor edge cases --- .../src/handlers/file-doc-store.test.ts | 98 +++++++- apps/realtime/src/handlers/file-doc-store.ts | 52 +++-- apps/realtime/src/handlers/file-doc.test.ts | 30 +++ apps/realtime/src/handlers/file-doc.ts | 3 +- .../collaboration/file-doc-provider.test.ts | 213 +++++++++++++++++- .../collaboration/file-doc-provider.ts | 58 ++--- .../pending-update-journal.test.ts | 17 ++ .../collaboration/pending-update-journal.ts | 15 +- .../find/find-extension.test.ts | 14 ++ .../find/find-matches.test.ts | 25 +- .../rich-markdown-editor/find/find-matches.ts | 26 ++- .../menus/bubble-menu.tsx | 12 +- .../menus/editor-toolbar-integration.test.tsx | 103 +++++++++ .../round-trip-safety.test.ts | 28 +++ .../rich-markdown-editor/round-trip-safety.ts | 18 +- .../workspace-file-storage-accounting.test.ts | 20 +- 16 files changed, 626 insertions(+), 106 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index b36588e550f..bda4a725ef9 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -30,7 +30,7 @@ interface Backing { /** Largest stream range response requested, proving replay is paginated. */ maxRangeCount: number /** Optional deterministic compaction hook invoked before each range page is read. */ - onRange?: (call: number, key: string) => void + onRange?: (call: number, key: string, start: string) => void rangeCalls: number /** Largest multiplexed XREAD request and COUNT observed. */ maxReadStreams: number @@ -69,7 +69,7 @@ function makeClient(): any { }, xRange: async (key: string, start: string, end: string, options?: { COUNT?: number }) => { b().rangeCalls++ - b().onRange?.(b().rangeCalls, key) + b().onRange?.(b().rangeCalls, key, start) const startId = start.startsWith('(') ? start.slice(1) : start const entries = (b().streams.get(key) ?? []).filter( (entry) => @@ -138,6 +138,9 @@ function makeClient(): any { }, eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { const [key] = opts.keys + if (script.includes("redis.call('exists', KEYS[1])") && !b().streams.has(key)) { + return script.includes('zscore') ? -1 : false + } if (script.includes('return ARGV[1]')) { const generation = b().kv.get(opts.keys[1]) if (generation !== undefined) return generation @@ -291,6 +294,15 @@ function updateFor(text: string): Uint8Array { } let stores: FileDocStore[] = [] + +/** An existing stream from a relay predating generation markers; modern seeds use seedIfEmpty. */ +function seedLegacyStream(update = updateFor('')): void { + const backing = state.backing! + backing.streams.set(`filedoc:stream:${NAME}`, [ + { id: `${++backing.seq}-0`, message: { u: Buffer.from(update).toString('base64') } }, + ]) +} + async function newStore(): Promise { const store = new FileDocStore(REDIS_URL) await store.init() @@ -415,7 +427,7 @@ describe('FileDocStore', () => { const token = await a.shouldSeed(NAME) expect(token).toBeTruthy() // A seeds and releases its lock. - a.publish(NAME, updateFor('hello')) + await a.seedIfEmpty(NAME, updateFor('hello')) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) await a.releaseSeedLock(NAME, token as string) // A different task must NOT seed again — the lock is free but the stream is non-empty. @@ -426,7 +438,7 @@ describe('FileDocStore', () => { it('fences stale publishers after invalidation and lets the next authoritative seed start fresh', async () => { const store = await newStore() const original = updateFor('old generation') - await store.publishAndWait(NAME, original) + await store.seedIfEmpty(NAME, original) await store.invalidateDocument(NAME, 10) await expect(store.getStreamState(NAME)).resolves.toBeNull() @@ -447,7 +459,7 @@ describe('FileDocStore', () => { it('getStreamState reconstructs the shared document from the stream', async () => { const a = await newStore() - a.publish(NAME, updateFor('shared content')) + await a.seedIfEmpty(NAME, updateFor('shared content')) let state: Uint8Array | null = null await vi.waitFor(async () => { state = await a.getStreamState(NAME) @@ -512,13 +524,31 @@ describe('FileDocStore', () => { await expect(store.getStreamState(NAME)).resolves.toBeNull() }) + it('rejects appends and duplicate acknowledgements when only the stream is lost', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('base'), 10) + const generation = await store.getDocumentGeneration(NAME) + const delta = updateFor('edit') + await store.publishClientUpdateAndWait(NAME, 'accepted-update', delta, generation) + state.backing!.streams.delete(`filedoc:stream:${NAME}`) + + await expect(store.publishAndWait(NAME, delta, generation)).rejects.toThrow('replaced') + await expect( + store.publishClientUpdateAndWait(NAME, 'new-update', delta, generation) + ).rejects.toThrow('replaced') + await expect( + store.publishClientUpdateAndWait(NAME, 'accepted-update', delta, generation) + ).rejects.toThrow('replaced') + expect(state.backing!.streams.has(`filedoc:stream:${NAME}`)).toBe(false) + }) + it('adopts the identity of a pre-upgrade stream before acknowledging its edits', async () => { const store = await newStore() const seed = new Y.Doc() seed.getMap('config').set('initialContentLoaded', true) seed.getMap('config').set('docId', 'legacy-document') seed.getText('body').insert(0, 'legacy') - await store.publishAndWait(NAME, Y.encodeStateAsUpdate(seed)) + seedLegacyStream(Y.encodeStateAsUpdate(seed)) const attached = new Y.Doc() await store.attachRoom(NAME, attached) expect(await store.getDocumentGeneration(NAME)).toBe('legacy-document') @@ -631,9 +661,44 @@ describe('FileDocStore', () => { recovered.destroy() }) + it('reads the replacement snapshot when peer deltas cross the old replay tail', async () => { + const streamKey = `filedoc:stream:${NAME}` + const source = new Y.Doc() + const entries: Array<{ id: string; message: Record }> = [] + source.on('update', (update: Uint8Array) => { + entries.push({ + id: `${entries.length + 1}-0`, + message: { u: Buffer.from(update).toString('base64') }, + }) + }) + for (let i = 1; i <= 8; i++) + source.getText('body').insert(source.getText('body').length, String(i)) + const snapshot = Y.encodeStateAsUpdate(source) + state.backing!.streams.set(streamKey, entries.slice()) + for (let i = 9; i <= 11; i++) + source.getText('body').insert(source.getText('body').length, String(i)) + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(4-0') return + state.backing!.streams.set(streamKey, [ + ...entries.slice(7), + { id: '12-0', message: { u: Buffer.from(snapshot).toString('base64'), s: '1' } }, + ]) + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe(source.getText('body').toString()) + } finally { + source.destroy() + recovered.destroy() + } + }) + it('attachRoom catches a fresh task up to the current shared state', async () => { const a = await newStore() - a.publish(NAME, updateFor('already here')) + await a.seedIfEmpty(NAME, updateFor('already here')) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) // A second task opens the same file: its doc must load the existing content, not start empty. @@ -645,6 +710,7 @@ describe('FileDocStore', () => { }) it('converges a peer task via the tailer after attach', async () => { + seedLegacyStream() const a = await newStore() const b = await newStore() const bDoc = new Y.Doc() @@ -707,6 +773,7 @@ describe('FileDocStore', () => { }) it('tags an agent-streamed frame so a peer tailer applies it as REDIS_AGENT_ORIGIN (never persisted)', async () => { + seedLegacyStream() const streamKey = `filedoc:stream:${NAME}` const a = await newStore() const b = await newStore() @@ -732,6 +799,7 @@ describe('FileDocStore', () => { }) it('latches realEdited synchronously so a concurrent compaction can never mislabel a real edit', async () => { + seedLegacyStream() // The data-loss race: a real edit sits in room.doc synchronously, but if realEdited were set only // AFTER appendUpdate's awaits, a concurrent agent-triggered compaction could snapshot that content and // stamp it an agent (no-persist) frame — losing the edit. The latch must be set in the same tick. @@ -787,6 +855,7 @@ describe('FileDocStore', () => { }) it('retries a transient append failure so the edit is not lost from the shared log', async () => { + seedLegacyStream() const a = await newStore() state.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed a.publish(NAME, updateFor('resilient')) @@ -802,34 +871,38 @@ describe('FileDocStore', () => { }) it('deduplicates acknowledged client retries by update id', async () => { + seedLegacyStream() const store = await newStore() const update = updateFor('retry-safe') await store.publishClientUpdateAndWait(NAME, 'update-1', update) await store.publishClientUpdateAndWait(NAME, 'update-1', update) - expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(1) + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) }) it('does not drop different payloads that reuse an acknowledged update id', async () => { + seedLegacyStream() const store = await newStore() await store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('first')) await store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('second')) - expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) }) it('uses unambiguous acknowledged-update deduplication keys', async () => { + seedLegacyStream() const store = await newStore() await store.publishClientUpdateAndWait(NAME, 'a', new Uint8Array([0, 98])) await store.publishClientUpdateAndWait(NAME, 'a\0', new Uint8Array([98])) - expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) }) it('bounds acknowledged-update deduplication independently of stream traffic', async () => { + seedLegacyStream() const store = await newStore() const update = updateFor('bounded') @@ -923,6 +996,7 @@ describe('FileDocStore', () => { }) it('preserves exactly the delta bytes observed after a compaction barrier', async () => { + seedLegacyStream() const store = await newStore() const doc = new Y.Doc() await store.attachRoom(NAME, doc) @@ -984,7 +1058,7 @@ describe('FileDocStore', () => { it('streamHasContent fences a seed apply against an already-seeded stream', async () => { const a = await newStore() expect(await a.streamHasContent(NAME)).toBe(false) - a.publish(NAME, updateFor('seeded')) + await a.seedIfEmpty(NAME, updateFor('seeded')) await vi.waitFor(async () => expect(await a.streamHasContent(NAME)).toBe(true)) }) @@ -1065,7 +1139,7 @@ describe('FileDocStore', () => { author.getText('body').insert(4, 'peer') const a = await newStore() - a.publish(NAME, updates[0]) // 'base' + seedLegacyStream(updates[0]) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) // Task B attaches; while its synchronous catch-up runs, task A publishes the second edit. The tailer diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 8e66bd465d7..2870e14712e 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -80,18 +80,18 @@ const ADOPT_GENERATION_SCRIPT = * Returns the new stream id, or `false` while invalidated. */ const APPEND_UPDATE_SCRIPT = - "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] then return false end; if ARGV[3] ~= '' then return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1') else return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]) end" + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; if ARGV[3] ~= '' then return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1') else return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]) end" /** A compacted snapshot replaces the seed entry, so it must carry that seed's generation forward. */ const APPEND_SNAPSHOT_SCRIPT = - "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] then return false end; return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1', ARGV[5], ARGV[4])" + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1', ARGV[5], ARGV[4])" /** * Atomically deduplicate and append an acknowledged client update. Socket acknowledgements can be * lost, so a retry with the same id must not inflate the stream or its compaction counters. */ const APPEND_CLIENT_UPDATE_SCRIPT = - "local generation = redis.call('get', KEYS[3]) or ''; if generation ~= ARGV[6] then return -1 end; if redis.call('zscore', KEYS[2], ARGV[1]) then redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 0 end; local id = redis.call('xadd', KEYS[1], '*', ARGV[2], ARGV[3]); local score = string.match(id, '^(%d+)'); redis.call('zadd', KEYS[2], score, ARGV[1]); local excess = redis.call('zcard', KEYS[2]) - tonumber(ARGV[4]); if excess > 0 then redis.call('zpopmin', KEYS[2], excess) end; if redis.call('ttl', KEYS[2]) < 0 then redis.call('expire', KEYS[2], ARGV[5]) end; redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 1" + "local generation = redis.call('get', KEYS[3]) or ''; if generation ~= ARGV[6] or redis.call('exists', KEYS[1]) == 0 then return -1 end; if redis.call('zscore', KEYS[2], ARGV[1]) then redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 0 end; local id = redis.call('xadd', KEYS[1], '*', ARGV[2], ARGV[3]); local score = string.match(id, '^(%d+)'); redis.call('zadd', KEYS[2], score, ARGV[1]); local excess = redis.call('zcard', KEYS[2]) - tonumber(ARGV[4]); if excess > 0 then redis.call('zpopmin', KEYS[2], excess) end; if redis.call('ttl', KEYS[2]) < 0 then redis.call('expire', KEYS[2], ARGV[5]) end; redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 1" /** * Monotonic set of the synced-version token: overwrite ONLY when the new value is greater than the @@ -802,34 +802,44 @@ export class FileDocStore { ): Promise { if (!this.write) return 0 const key = streamKey(name) + let firstId = (await this.write.xRange(key, '-', '+', { COUNT: 1 }))[0]?.id + if (!firstId) return 0 const tail = await this.write.xRevRange(key, '+', '-', { COUNT: 1 }) - if (tail.length === 0) return 0 - const endId = tail[0].id + if (tail.length === 0) throw new FileDocInvalidatedError() + let endId = tail[0].id let cursor = afterId.includes('-') ? afterId : `${afterId}-0` let entriesRead = 0 let encodedBytes = 0 - while (isAfterStreamId(endId, cursor)) { - const page = await this.write.xRange(key, `(${cursor}`, '+', { - COUNT: REPLAY_PAGE_COUNT, - }) - if (page.length === 0) { - if (isAfterStreamId(endId, cursor)) { + while (true) { + while (isAfterStreamId(endId, cursor)) { + const page = await this.write.xRange(key, `(${cursor}`, '+', { + COUNT: REPLAY_PAGE_COUNT, + }) + if (page.length === 0) { throw new Error(`File document replay lost its completion barrier for ${name}`) } - break - } - for (const entry of page) { - entriesRead += 1 - encodedBytes += entry.message[UPDATE_FIELD]?.length ?? 0 - if (entriesRead > REPLAY_MAX_ENTRIES || encodedBytes > REPLAY_MAX_ENCODED_BYTES) { - throw new Error(`File document replay exceeded its safety limit for ${name}`) + for (const entry of page) { + entriesRead += 1 + encodedBytes += entry.message[UPDATE_FIELD]?.length ?? 0 + if (entriesRead > REPLAY_MAX_ENTRIES || encodedBytes > REPLAY_MAX_ENCODED_BYTES) { + throw new Error(`File document replay exceeded its safety limit for ${name}`) + } + cursor = entry.id + if (!visit(entry)) return entriesRead } - cursor = entry.id - if (!visit(entry)) return entriesRead } + + /** A moving head means compaction may have removed unread dependencies. Replay its snapshot. */ + const currentFirstId = (await this.write.xRange(key, '-', '+', { COUNT: 1 }))[0]?.id + if (!currentFirstId) throw new FileDocInvalidatedError() + if (currentFirstId === firstId) return entriesRead + firstId = currentFirstId + cursor = '0-0' + const currentTail = await this.write.xRevRange(key, '+', '-', { COUNT: 1 }) + if (currentTail.length === 0) throw new FileDocInvalidatedError() + endId = currentTail[0].id } - return entriesRead } /** Release the seed lock (compare-and-delete) once the seed has been published or a seed attempt failed. */ diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index d3dc7193615..9e0223e0713 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -817,6 +817,36 @@ describe('setupWorkspaceFileDocHandlers', () => { clientDoc.destroy() }) + it('does not discard a rebuilt room after a delayed check of its predecessor', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io, left } = createIo() + const original = setup('socket-original', io) + await original.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const checks: Array<(current: boolean) => void> = [] + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockImplementation(() => new Promise((resolve) => checks.push(resolve))) + try { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# New', 'doc-new')) + const first = setup('socket-first-new', io) + const second = setup('socket-second-new', io) + const firstJoin = first.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + const secondJoin = second.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 3 }) + await vi.waitFor(() => expect(checks).toHaveLength(2)) + checks[0](false) + await firstJoin + checks[1](false) + await secondJoin + + expect(joinSuccessFileId(first.socket)).toBe('file-1') + expect(joinSuccessFileId(second.socket)).toBe('file-1') + expect(left).toContainEqual({ socketId: 'socket-original', room: ROOM_NAME }) + expect(left).not.toContainEqual({ socketId: 'socket-first-new', room: ROOM_NAME }) + } finally { + guard.mockRestore() + } + }) + it('seeds once across concurrent joiners, and every one of them waits for that seed', async () => { // Keep the first seed fetch IN FLIGHT so the doc is still unseeded when the second socket joins: // that forces the dedup onto the in-flight seed rather than `isDocSeeded`. Both joins must WAIT diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 08d58b55451..5a4faae2bae 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -1430,7 +1430,8 @@ export function setupWorkspaceFileDocHandlers( if ( existing && isDocSeeded(existing.doc) && - !(await store.isDocumentGenerationCurrent(name, docIdOf(existing.doc))) + !(await store.isDocumentGenerationCurrent(name, docIdOf(existing.doc))) && + fileDocRooms.get(name) === existing ) { discardInvalidatedRoom(name, io) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index 8640f6f88ca..f37844d6564 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -39,9 +39,23 @@ const UPDATE_BATCH_TEST_WINDOW_MS = 100 function createSocket(connected = true) { const listeners = new Map void>>() const emit = vi.fn() + const timeout = vi.fn((delay: number) => ({ + emit( + event: string, + payload: unknown, + acknowledge: (error: Error | null, ack?: FileDocUpdateAck) => void + ) { + const timer = setTimeout(() => acknowledge(new Error('operation has timed out')), delay) + emit(event, payload, (error: Error | null, ack?: FileDocUpdateAck) => { + clearTimeout(timer) + acknowledge(error, ack) + }) + }, + })) const socket = { connected, emit, + timeout, on(event: string, cb: (...args: unknown[]) => void) { let set = listeners.get(event) if (!set) { @@ -59,15 +73,15 @@ function createSocket(connected = true) { if (event === 'disconnect') socket.connected = false for (const cb of listeners.get(event) ?? []) cb(...args) } - return { socket: socket as unknown as Socket, emit, fire } + return { socket: socket as unknown as Socket, emit, fire, timeout } } function createProvider(connected = true) { - const { socket, emit, fire } = createSocket(connected) + const { socket, emit, fire, timeout } = createSocket(connected) const doc = new Y.Doc() const awareness = new awarenessProtocol.Awareness(doc) const provider = new FileDocProvider(socket, 'file-1', doc, awareness) - return { provider, doc, awareness, emit, fire } + return { provider, doc, awareness, emit, fire, timeout } } function acceptJoin( @@ -356,6 +370,48 @@ describe('FileDocProvider', () => { expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) }) + it('uses ordinary Yjs synchronization without stale unload protection on a no-ACK relay', () => { + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const { provider, doc, awareness, emit, fire } = createProvider(true) + const serverDoc = new Y.Doc() + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + doc.getText('default').insert(0, 'before join') + expect(unloadIsPrevented()).toBe(true) + acceptJoin(fire, doc.clientID, undefined, false) + expect(unloadIsPrevented()).toBe(false) + + fire('disconnect') + doc.getText('default').insert(11, ' and offline') + expect(unloadIsPrevented()).toBe(false) + fire('connect') + acceptJoin(fire, doc.clientID, undefined, false) + emit.mockClear() + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + + for (const message of emittedMessages(emit)) { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) + } + expect(serverDoc.getText('default').toString()).toBe('before join and offline') + expect(unloadIsPrevented()).toBe(false) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + vi.unstubAllGlobals() + } + }) + it('recovers an unacknowledged edit after a tab restart and clears it only after acceptance', async () => { journalStorage.clear() const scope = { workspaceId: 'workspace-1', userId: 'user-1' } @@ -410,9 +466,9 @@ describe('FileDocProvider', () => { ([event]) => event === FILE_DOC_EVENTS.UPDATE ) const payload = updateCall?.[1] as { updateId: string } - const acknowledge = updateCall?.[2] as (ack: FileDocUpdateAck) => void + const acknowledge = updateCall?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void Y.applyUpdate(serverDoc, (updateCall?.[1] as { update: Uint8Array }).update) - acknowledge({ status: 'accepted', updateId: payload.updateId }) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) await vi.waitFor(async () => { @@ -501,8 +557,8 @@ describe('FileDocProvider', () => { await vi.advanceTimersByTimeAsync(0) const firstUpdate = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) const firstPayload = firstUpdate?.[1] as { updateId: string } - const acknowledge = firstUpdate?.[2] as (ack: FileDocUpdateAck) => void - acknowledge({ status: 'accepted', updateId: firstPayload.updateId }) + const acknowledge = firstUpdate?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + acknowledge(null, { status: 'accepted', updateId: firstPayload.updateId }) await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) expect(save).toHaveBeenCalledTimes(2) @@ -517,7 +573,7 @@ describe('FileDocProvider', () => { it('retries an unacknowledged update with the same idempotency key', async () => { vi.useFakeTimers() try { - const { provider, doc, emit, fire } = createProvider(true) + const { provider, doc, emit, fire, timeout } = createProvider(true) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') acceptJoin(fire, doc.clientID, 'doc-1') emit.mockClear() @@ -526,6 +582,7 @@ describe('FileDocProvider', () => { await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) expect(first).toBeDefined() + expect(timeout).toHaveBeenCalledWith(FILE_DOC_TIMEOUTS.updateAckMs) await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 2_000) const updates = emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE) @@ -550,9 +607,9 @@ describe('FileDocProvider', () => { await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) const payload = first?.[1] as { updateId: string } - const acknowledge = first?.[2] as (ack: FileDocUpdateAck) => void + const acknowledge = first?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void - acknowledge({ + acknowledge(null, { status: 'rejected', updateId: payload.updateId, code: 'NOT_JOINED', @@ -570,6 +627,138 @@ describe('FileDocProvider', () => { } }) + it('ignores expired acknowledgements after the provider is destroyed', async () => { + vi.useFakeTimers() + const { provider, doc, awareness, emit, fire, timeout } = createProvider(true) + try { + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + doc.getText('default').insert(0, 'pending') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(timeout).toHaveBeenCalledWith(FILE_DOC_TIMEOUTS.updateAckMs) + + provider.destroy() + emit.mockClear() + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 6_000) + expect(emit).not.toHaveBeenCalled() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + vi.useRealTimers() + } + }) + + it('preserves pending acknowledged edits across a downgrade without calling legacy sync an acceptance', async () => { + vi.useFakeTimers() + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const clear = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'clear') + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + const serverDoc = new Y.Doc() + Y.applyUpdate(serverDoc, Y.encodeStateAsUpdate(doc)) + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + doc.getText('default').insert(0, 'pending acknowledged edit') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const firstPayload = first?.[1] as { updateId: string } + expect(first).toBeDefined() + + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.advanceTimersByTimeAsync(0) + emit.mockClear() + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + for (const message of emittedMessages(emit)) { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) + } + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 6_000) + + expect(serverDoc.getText('default').toString()).toBe('pending acknowledged edit') + expect(provider.synced).toBe(true) + expect(unloadIsPrevented()).toBe(true) + expect(await journal.load('doc-1')).not.toBeNull() + expect(clear).not.toHaveBeenCalled() + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + const retry = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(retry?.[1]).toMatchObject({ updateId: firstPayload.updateId }) + const acknowledge = retry?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + acknowledge(null, { status: 'accepted', updateId: firstPayload.updateId }) + await vi.advanceTimersByTimeAsync(0) + expect(unloadIsPrevented()).toBe(false) + expect(await journal.load('doc-1')).toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + clear.mockRestore() + vi.unstubAllGlobals() + vi.useRealTimers() + } + }) + + it('protects a recovered pending journal even when the new relay has no acknowledged channel', async () => { + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const recoveredDoc = new Y.Doc() + recoveredDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + recoveredDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + recoveredDoc.getText('default').insert(0, 'recover me') + const update = Y.encodeStateAsUpdate(recoveredDoc) + await journal.save('doc-1', update, update) + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.waitFor(() => expect(doc.getText('default').toString()).toBe('recover me')) + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + expect(event.defaultPrevented).toBe(true) + expect(await journal.load('doc-1')).not.toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + recoveredDoc.destroy() + vi.unstubAllGlobals() + } + }) + it('journals an edit made while disconnected before page teardown', async () => { journalStorage.clear() const scope = { workspaceId: 'workspace-1', userId: 'user-1' } @@ -919,9 +1108,9 @@ describe('FileDocProvider', () => { const update = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) expect(update).toBeDefined() const payload = update?.[1] as { updateId: string } - const acknowledge = update?.[2] as (ack: FileDocUpdateAck) => void + const acknowledge = update?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void expect(unloadIsPrevented()).toBe(true) - acknowledge({ status: 'accepted', updateId: payload.updateId }) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) expect(unloadIsPrevented()).toBe(false) doc.getText('default').insert(0, 'another ') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index f1cce26a7bc..af4ddca55bf 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -151,7 +151,7 @@ export class FileDocProvider extends ObservableV2 { /** Deadline for reaching readiness (synced + seeded); fires the fallback if it is never reached. */ private readinessTimer: ReturnType | null = null private joinAccepted = false - private acknowledgedUpdates = false + private updateMode: 'negotiating' | 'legacy' | 'acknowledged' = 'negotiating' private joinPending = false private joinRetryAttempt = 0 private joinRetryTimer: ReturnType | null = null @@ -165,7 +165,6 @@ export class FileDocProvider extends ObservableV2 { private pendingUpdateBatch: Uint8Array[] = [] private inFlightUpdate: PendingClientUpdate | null = null private updateBatchTimer: ReturnType | null = null - private updateAckTimer: ReturnType | null = null private updateRetryTimer: ReturnType | null = null private updateRetryAttempt = 0 private updateFlushInProgress = false @@ -233,7 +232,7 @@ export class FileDocProvider extends ObservableV2 { /** Clear the readiness deadline once the editor is usable (synced AND seeded). */ private handleConfigChange = () => { if (this.synced && this.isSeeded()) this.clearReadinessTimer() - if (this.acknowledgedUpdates && this.docId() && this.pendingUpdateBatch.length > 0) { + if (this.updateMode === 'acknowledged' && this.docId() && this.pendingUpdateBatch.length > 0) { this.scheduleUpdateFlush(0) } } @@ -291,10 +290,8 @@ export class FileDocProvider extends ObservableV2 { private clearUpdateTimers() { if (this.updateBatchTimer !== null) clearTimeout(this.updateBatchTimer) - if (this.updateAckTimer !== null) clearTimeout(this.updateAckTimer) if (this.updateRetryTimer !== null) clearTimeout(this.updateRetryTimer) this.updateBatchTimer = null - this.updateAckTimer = null this.updateRetryTimer = null } @@ -357,9 +354,7 @@ export class FileDocProvider extends ObservableV2 { this.clearJoinRetryTimer() this.clearJoinAckTimer() this.clearSyncRetryTimer() - if (this.updateAckTimer !== null) clearTimeout(this.updateAckTimer) if (this.updateRetryTimer !== null) clearTimeout(this.updateRetryTimer) - this.updateAckTimer = null this.updateRetryTimer = null this.joinAccepted = false this.joinHydrating = false @@ -401,7 +396,9 @@ export class FileDocProvider extends ObservableV2 { this.joinPending = false this.joinRetryAttempt = 0 this.clearJoinRetryTimer() - this.acknowledgedUpdates = data.acknowledgedUpdates === true && data.docId !== undefined + if (data.acknowledgedUpdates === true && data.docId !== undefined) { + this.updateMode = 'acknowledged' + } this.joinHydrating = true const generation = this.connectionGeneration if (!this.journal) { @@ -482,10 +479,17 @@ export class FileDocProvider extends ObservableV2 { return } - if (recovered !== null && this.acknowledgedUpdates && !this.recoveryQueued) { + const updateMode = + data.acknowledgedUpdates === true && data.docId !== undefined ? 'acknowledged' : 'legacy' + /** Pre-negotiation deltas stay in Y.Doc for legacy sync; existing recovery is never acknowledged here. */ + if (updateMode === 'legacy' && this.updateMode === 'negotiating') this.pendingUpdateBatch = [] + this.updateMode = updateMode + + if (recovered !== null && !this.recoveryQueued) { this.queuePendingUpdate(recovered.pendingUpdate) this.recoveryQueued = true } + this.updateBeforeUnloadProtection() this.joinHydrating = false this.joinAccepted = true @@ -495,7 +499,7 @@ export class FileDocProvider extends ObservableV2 { const bufferedMessages = this.bufferedMessages this.clearBufferedMessages() for (const message of bufferedMessages) this.applyMessage(message) - if (this.acknowledgedUpdates) { + if (this.updateMode === 'acknowledged') { if (this.inFlightUpdate) this.sendInFlightUpdate() else if (this.pendingUpdateBatch.length > 0) this.scheduleUpdateFlush(0) } @@ -659,7 +663,7 @@ export class FileDocProvider extends ObservableV2 { const syncType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this) if (encoding.length(encoder) > 1) { const response = encoding.toUint8Array(encoder) - if (this.acknowledgedUpdates && syncType === syncProtocol.messageYjsSyncStep1) { + if (this.updateMode === 'acknowledged' && syncType === syncProtocol.messageYjsSyncStep1) { const responseDecoder = decoding.createDecoder(response) decoding.readVarUint(responseDecoder) decoding.readVarUint(responseDecoder) @@ -705,12 +709,12 @@ export class FileDocProvider extends ObservableV2 { } if (!this.joinAccepted) { - this.queuePendingUpdate(update) - if (this.acknowledgedUpdates) this.scheduleUpdateFlush(UPDATE_BATCH_MS) + if (this.updateMode !== 'legacy') this.queuePendingUpdate(update) + if (this.updateMode === 'acknowledged') this.scheduleUpdateFlush(UPDATE_BATCH_MS) return } - if (!this.acknowledgedUpdates) { + if (this.updateMode !== 'acknowledged') { if (!this.socket.connected) return const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) @@ -739,7 +743,7 @@ export class FileDocProvider extends ObservableV2 { private async flushPendingUpdates(): Promise { if ( - !this.acknowledgedUpdates || + this.updateMode !== 'acknowledged' || this.pendingUpdateBatch.length === 0 || this.disposed || this.fatal @@ -792,7 +796,7 @@ export class FileDocProvider extends ObservableV2 { if ( !pending || !docId || - !this.acknowledgedUpdates || + this.updateMode !== 'acknowledged' || this.disposed || this.fatal || !this.socket.connected || @@ -800,28 +804,28 @@ export class FileDocProvider extends ObservableV2 { ) return - if (this.updateAckTimer !== null) clearTimeout(this.updateAckTimer) - this.updateAckTimer = setTimeout(() => { - this.updateAckTimer = null - this.scheduleUpdateRetry() - }, FILE_DOC_TIMEOUTS.updateAckMs) - + const generation = this.connectionGeneration const payload: FileDocUpdatePayload = { fileId: this.fileId, docId, updateId: pending.updateId, update: pending.update, } - this.socket.emit(FILE_DOC_EVENTS.UPDATE, payload, (ack: FileDocUpdateAck) => { - this.handleUpdateAck(ack) - }) + this.socket + .timeout(FILE_DOC_TIMEOUTS.updateAckMs) + .emit(FILE_DOC_EVENTS.UPDATE, payload, (error: Error | null, ack?: FileDocUpdateAck) => { + if (this.disposed || this.fatal || this.inFlightUpdate !== pending) return + if (error) { + if (generation === this.connectionGeneration) this.scheduleUpdateRetry() + return + } + if (ack) this.handleUpdateAck(ack) + }) } private handleUpdateAck(ack: FileDocUpdateAck) { const pending = this.inFlightUpdate if (!pending || ack.updateId !== pending.updateId || this.disposed || this.fatal) return - if (this.updateAckTimer !== null) clearTimeout(this.updateAckTimer) - this.updateAckTimer = null if (ack.status === 'accepted') { const docId = this.docId() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts index 618bc443c67..7cdaac391b7 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts @@ -9,6 +9,7 @@ import * as Y from 'yjs' const storage = vi.hoisted(() => new Map()) vi.mock('idb-keyval', () => ({ + get: vi.fn(async (key: string) => storage.get(key)), update: vi.fn((key: string, updater: (value: unknown) => unknown) => { storage.set(key, updater(storage.get(key))) }), @@ -33,6 +34,11 @@ function updateWith(text: string): Uint8Array { describe('PendingFileDocUpdateJournal', () => { beforeEach(() => { storage.clear() + vi.mocked(updateValue) + .mockReset() + .mockImplementation(async (key, updater) => { + storage.set(String(key), updater(storage.get(String(key)))) + }) }) it('stores a full recovery snapshot separately from the pending wire update', async () => { @@ -60,6 +66,17 @@ describe('PendingFileDocUpdateJournal', () => { expect(result).toMatchObject({ status: 'limit-exceeded' }) }) + it('loads an existing draft without requiring a writable transaction', async () => { + const subject = journal() + const pendingUpdate = updateWith('recoverable draft') + await subject.save('doc-1', pendingUpdate, pendingUpdate) + const writes = vi.mocked(updateValue).mock.calls.length + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Read-only storage')) + + await expect(subject.load('doc-1')).resolves.toMatchObject({ docId: 'doc-1', pendingUpdate }) + expect(updateValue).toHaveBeenCalledTimes(writes) + }) + it('distinguishes unavailable browser storage from a configured size limit', async () => { vi.mocked(updateValue).mockRejectedValueOnce(new Error('Storage denied')) const pendingUpdate = updateWith('pending') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts index df2b92cf214..b19a454edef 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts @@ -2,7 +2,7 @@ import { createLogger } from '@sim/logger' import { FILE_DOC_LIMITS } from '@sim/realtime-protocol/file-doc' -import { update as updateValue } from 'idb-keyval' +import { get, update as updateValue } from 'idb-keyval' import * as Y from 'yjs' const logger = createLogger('PendingFileDocUpdateJournal') @@ -98,15 +98,10 @@ export class PendingFileDocUpdateJournal { async load(preferredDocId?: string): Promise { try { await this.mutationQueue - let recovered: PendingDocumentRecovery | null = null - await updateValue(this.key, (value) => { - const documents = liveDocuments(value, Date.now()) - recovered = preferredDocId - ? (documents.find((document) => document.docId === preferredDocId) ?? null) - : (documents[0] ?? null) - return record(documents) - }) - return recovered + const documents = liveDocuments(await get(this.key), Date.now()) + return preferredDocId + ? (documents.find((document) => document.docId === preferredDocId) ?? null) + : (documents[0] ?? null) } catch (error) { logger.warn('Failed to load pending file edits', { error }) return null diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts index 3bba8143644..32afc078615 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts @@ -164,6 +164,20 @@ describe('RichMarkdownFind', () => { expect(instance.getMarkdown()).toBe('gamma and beta') }) + it.each([ + ['he**llo**', 'world'], + ['**he**llo', '**world**'], + ])('follows native ProseMirror replacement formatting for %s', (source, expected) => { + const instance = mountEditor(source) + setFindQuery(instance, 'hello') + const { from, to } = getFindTally(instance.state).matches[0] + const nativeResult = instance.state.tr.setStoredMarks(null).insertText('world', from, to).doc + + expect(replaceActiveFindMatch(instance, 'world')).toBe(true) + expect(instance.state.doc.eq(nativeResult)).toBe(true) + expect(instance.getMarkdown()).toBe(expected) + }) + it('preserves each matched range formatting during Replace All', () => { const instance = mountEditor('**alpha** and alpha and *alpha*') instance.commands.setTextSelection(instance.state.doc.content.size - 1) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts index b62f392cea7..ed0cebdf333 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts @@ -3,8 +3,11 @@ */ import { Editor } from '@tiptap/core' import { afterEach, describe, expect, it } from 'vitest' -import { createMarkdownContentExtensions } from '../extensions' -import { FIND_MATCH_LIMIT, findMatches } from './find-matches' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + FIND_MATCH_LIMIT, + findMatches, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' let editor: Editor | null = null afterEach(() => { @@ -59,6 +62,24 @@ describe('findMatches', () => { expect(matchedText('ab\n\ncd', 'abcd')).toEqual([]) }) + it.each(['\uFFFF', 'a\uFFFFb'])('never matches an inline atom using %j', (query) => { + const doc = docFor('a
b') + expect(() => doc.check()).not.toThrow() + expect(findMatches(doc, query)).toEqual({ matches: [], truncated: false }) + }) + + it('does not count atom placeholders toward the match limit', () => { + const doc = docFor('a
b\uFFFF') + expect(() => doc.check()).not.toThrow() + const { matches, truncated } = findMatches(doc, '\uFFFF', 1) + expect(matches.map(({ from, to }) => doc.textBetween(from, to))).toEqual(['\uFFFF']) + expect(truncated).toBe(false) + }) + + it('keeps real non-character text searchable across a formatting boundary', () => { + expect(matchedText('a**\uFFFF**b', 'a\uFFFFb')).toEqual(['a\uFFFFb']) + }) + it('never matches across an inline atom', () => { // The image between them occupies a position; joining `a` to `b` would be a phantom match. expect(matchedText('a![alt](https://x.com/i.png)b', 'ab')).toEqual([]) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts index 5133557d52f..933313c8ec2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts @@ -25,8 +25,8 @@ export const EMPTY_FIND_RESULT: FindResult = { matches: [], truncated: false } /** * Stands in for one position of a non-text inline node (an image, a mention chip) so a match can * never span one — searching `ab` must not join the `a` before an image to the `b` after it. U+FFFF - * is a permanent Unicode non-character, so no query can contain it and match the placeholder itself, - * and it is not whitespace, so the shared scan's whitespace folding leaves it alone. + * is not whitespace, so the shared scan's whitespace folding leaves it alone. Segment checks exclude + * atoms even when a query contains this character, without excluding genuine U+FFFF text. */ const ATOM_PLACEHOLDER = '￿' @@ -34,6 +34,7 @@ const ATOM_PLACEHOLDER = ' interface TextSegment { textStart: number docStart: number + isText: boolean } /** @@ -74,7 +75,7 @@ export function findMatches( if (soleText === null) { const built: TextSegment[] = [] node.forEach((child, offset) => { - built.push({ textStart: text.length, docStart: pos + 1 + offset }) + built.push({ textStart: text.length, docStart: pos + 1 + offset, isText: child.isText }) text += child.isText && child.text ? child.text : ATOM_PLACEHOLDER.repeat(child.nodeSize) }) segments = built @@ -83,6 +84,21 @@ export function findMatches( let segmentIndex = 0 forEachSearchOccurrence(text, query, (start, end) => { if (truncated) return + if (segments) { + while ( + segmentIndex + 1 < segments.length && + segments[segmentIndex + 1].textStart <= start + ) { + segmentIndex++ + } + for ( + let index = segmentIndex; + index < segments.length && segments[index].textStart < end; + index++ + ) { + if (!segments[index].isText) return + } + } if (matches.length >= limit) { truncated = true return @@ -91,10 +107,6 @@ export function findMatches( matches.push({ from: pos + 1 + start, to: pos + 1 + end }) return } - // Segments are ordered and occurrences arrive left to right, so the cursor only moves forward. - while (segmentIndex + 1 < segments.length && segments[segmentIndex + 1].textStart <= start) { - segmentIndex++ - } const segment = segments[segmentIndex] const from = segment.docStart + (start - segment.textStart) matches.push({ from, to: from + (end - start) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx index 899610ce185..8f6fd1c251d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx @@ -16,7 +16,6 @@ import { TextQuote, Unlink, } from '@sim/emcn/icons' -import { getMarkRange } from '@tiptap/core' import { PluginKey, type SelectionBookmark, @@ -59,14 +58,11 @@ function revealBubbleMenu(editor: Editor, key: PluginKey): void { editor.commands.setMeta(key, 'updatePosition') } -/** Captures selected text, or the complete link mark when the caret sits inside one. */ +/** Selects a caret's complete link so the bookmark, URL, and active controls share one target. */ function linkSelectionBookmark(editor: Editor): SelectionBookmark | null { - const { doc, selection, schema } = editor.state - if (!selection.empty) return selection.getBookmark() - const linkType = schema.marks.link - if (!linkType) return null - const range = getMarkRange(selection.$from, linkType) - return range ? TextSelection.create(doc, range.from, range.to).getBookmark() : null + if (editor.state.selection.empty) editor.commands.extendMarkRange('link') + const { selection } = editor.state + return selection.empty ? null : selection.getBookmark() } interface EditorBubbleMenuProps { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx index 330f548858d..b798501d8e2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx @@ -109,6 +109,16 @@ async function openLinkEditor(): Promise { return input } +async function openLinkAtCaret(offset: number): Promise { + select('format', true) + act(() => editor.commands.setTextSelection(editor.state.selection.from + offset)) + expect(key(editor.view.dom, 'k', { ctrlKey: true }).defaultPrevented).toBe(true) + await frame() + const input = linkGroup().querySelector('input[aria-label="Link URL"]') + if (!input) throw new Error('Missing link URL field') + return input +} + function changeUrl(input: HTMLInputElement, value: string): void { act(() => { Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(input, value) @@ -289,6 +299,99 @@ describe('real editor BubbleMenu keyboard integration', () => { expect(link?.getAttribute('href')).toBe('https://example.com/replacement') }) + it.each([0, 2, 6])( + 'prefills the complete link at caret offset %i without changing it on apply', + async (offset) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + const before = editor.getJSON() + const input = await openLinkAtCaret(offset) + + expect(input.value).toBe('https://example.com/original') + expect(button(linkGroup(), 'Remove link').disabled).toBe(false) + key(input, 'Enter') + await frame() + expect(editor.getJSON()).toEqual(before) + } + ) + + it('uses the same adjacent link for the captured range, URL, and update', async () => { + act(() => + editor.commands.setContent( + editorNormalForm( + '[before](https://example.com/first)[format](https://example.com/second) after' + ) + ) + ) + const input = await openLinkAtCaret(0) + expect(input.value).toBe('https://example.com/second') + changeUrl(input, 'https://example.com/replacement') + key(input, 'Enter') + await frame() + + const links = editor.view.dom.querySelectorAll('a') + expect([...links].map((link) => [link.textContent, link.getAttribute('href')])).toEqual([ + ['before', 'https://example.com/first'], + ['format', 'https://example.com/replacement'], + ]) + }) + + it.each([false, true])( + 'keeps a caret-opened link draft through a peer edit with read-only interval %s', + async (readOnly) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + const input = await openLinkAtCaret(2) + const group = linkGroup() + const apply = button(group, 'Apply link') + changeUrl(input, 'https://example.com/replacement') + if (readOnly) { + const before = editor.getJSON() + act(() => editor.setEditable(false)) + act(() => apply.click()) + expect(editor.getJSON()).toEqual(before) + } + act(() => editor.view.dispatch(editor.state.tr.insertText('remote ', 1))) + + if (readOnly) act(() => editor.setEditable(true)) + await frame() + expect((readOnly ? group : linkGroup()).querySelector('input')).toBe(input) + expect(input.value).toBe('https://example.com/replacement') + act(() => apply.click()) + await frame() + expect(editor.getText()).toBe('remote before format after') + expect(editor.view.dom.querySelector('a')?.textContent).toBe('format') + expect(editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/replacement' + ) + } + ) + + it('cancels a caret-opened URL draft without changing the link or losing its selection', async () => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + const input = await openLinkAtCaret(2) + const selection = editor.state.selection.toJSON() + const before = editor.getJSON() + changeUrl(input, 'https://example.com/cancelled') + key(input, 'Escape') + await frame() + + expect(viewport.contains(input)).toBe(false) + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.toJSON()).toEqual(selection) + expect(editor.getJSON()).toEqual(before) + }) + it('maps the captured link target through a prefix edit and an appended transaction', async () => { const input = await openLinkEditor() changeUrl(input, 'https://example.com/mapped') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts index dc00da977b6..0ba8ef68884 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts @@ -160,6 +160,34 @@ describe('isRoundTripSafe', () => { ) }) + it.each([ + '', + '[](/link)', + '[](/link)', + "[](/link)", + ])('checks attributes after quoted angle brackets without losing source: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(false) + expect(normalizeMarkdownContent(source)).toBe(source) + }) + + it.each([ + 'first', + '[first](/link)', + '[first](/link)', + '[](/link)', + ])('keeps duplicate image attributes in source mode: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(false) + expect(normalizeMarkdownContent(source)).toBe(source) + }) + + it('allows supported image attributes containing quoted angle brackets', () => { + expect(isRoundTripSafe('')).toBe(true) + expect(isRoundTripSafe('[a>b](/link)')).toBe(true) + expect( + isRoundTripSafe('[](/link)') + ).toBe(true) + }) + it.each([ '| |\n| --- |\n| body |', '| header |\n| --- |\n| |', diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts index a7fb777e2f2..0504c2d1b9f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts @@ -1,6 +1,6 @@ import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import { decodeHtmlEntities } from '@tiptap/core' -import { Marked, type Token } from 'marked' +import { Lexer, Marked, type Token, Tokenizer } from 'marked' import { extractImgSrcs } from '@/lib/uploads/utils/embedded-image-ref' import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { serializeMarkdownDocument } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' @@ -48,13 +48,23 @@ const SUPPORTED_IMAGE_ATTRIBUTES = new Set(['src', 'alt', 'title', 'width', 'hei * The image node deliberately models only the attributes it can render and serialize. An HTML image * carrying anything else must stay in source mode; comparing only its `src` would declare a stable but * lossy conversion safe after the unsupported attribute had already disappeared. + * Duplicate attributes also stay in source mode rather than choosing between conflicting values. */ function hasUnsupportedHtmlImageAttribute(content: string): boolean { - for (const image of content.matchAll(/]*)>/gi)) { - const attributes = image[1] + const tokenizer = new Tokenizer() + new Lexer({ gfm: true, tokenizer }) + const imagePattern = /])/gi + for (let image = imagePattern.exec(content); image; image = imagePattern.exec(content)) { + const tag = tokenizer.tag(content.slice(image.index)) + if (!tag) continue + imagePattern.lastIndex = image.index + tag.raw.length + const attributes = tag.raw.slice(4, -1) + const seen = new Set() const pattern = /(?:^|\s)([^\s=/>]+)(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>`]+))?/g for (const attribute of attributes.matchAll(pattern)) { - if (!SUPPORTED_IMAGE_ATTRIBUTES.has(attribute[1].toLowerCase())) return true + const name = attribute[1].toLowerCase() + if (!SUPPORTED_IMAGE_ATTRIBUTES.has(name) || seen.has(name)) return true + seen.add(name) } } return false diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts index eb44f367899..d58fb0200f6 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ import { workspaceFiles } from '@sim/db/schema' -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing' import { describeError } from '@sim/utils/errors' import { PASTE_LIMITS } from '@sim/utils/paste' import { eq } from 'drizzle-orm' @@ -749,6 +749,22 @@ describe('workspace file metadata and storage accounting', () => { const MD_ROW = { ...FILE_ROW, originalName: 'note.md', contentType: 'text/markdown' } it('transactionally enqueues a markdown overwrite for live-document reconciliation', async () => { + const transaction = { ...dbChainMock.db } + let committed = false + dbChainMockFns.transaction.mockImplementationOnce(async (callback) => { + const result = await callback(transaction) + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith( + transaction, + expect.objectContaining({ fileId: MD_ROW.id }) + ) + expect(mockProcessWorkspaceFileLiveDocReconciliationNow).not.toHaveBeenCalled() + committed = true + return result + }) + mockProcessWorkspaceFileLiveDocReconciliationNow.mockImplementationOnce(async () => { + expect(committed).toBe(true) + return 'completed' + }) // Distinct updatedAt vs contentUpdatedAt so the assertion proves the merge carries the CONTENT // version (the persist If-Match token), not `updatedAt` — reverting that wiring would fail here. const updatedFile = { @@ -769,7 +785,7 @@ describe('workspace file metadata and storage accounting', () => { Buffer.from('# new content', 'utf-8') ) - expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(expect.anything(), { + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(transaction, { workspaceId: MD_ROW.workspaceId, fileId: MD_ROW.id, version: updatedFile.contentUpdatedAt.getTime(), From 9410da449718589773b71b929a724acb7fe7b30f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 5 Sep 2026 11:21:27 -0700 Subject: [PATCH 3/9] fix(files): simplify recovery UI and restore canceled link carets --- .../collaboration/file-doc-provider.test.ts | 21 ++-- .../collaboration/file-doc-provider.ts | 35 +----- .../pending-update-journal.test.ts | 4 +- .../collaboration/pending-update-journal.ts | 15 +-- .../editor-lifecycle.test.tsx | 47 ++++---- .../menus/bubble-menu.tsx | 67 +++++++---- .../menus/editor-toolbar-integration.test.tsx | 72 +++++++++++- .../menus/use-editor-toolbar.ts | 3 + .../rich-markdown-editor.tsx | 107 ++---------------- 9 files changed, 169 insertions(+), 202 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index f37844d6564..740886194cb 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -786,7 +786,7 @@ describe('FileDocProvider', () => { provider.destroy() }) - it('does not recreate a discarded recovery record during page teardown or destroy', async () => { + it('preserves pending recovery through page teardown and destroy', async () => { journalStorage.clear() const scope = { workspaceId: 'workspace-1', userId: 'user-1' } const { socket, fire } = createSocket(true) @@ -801,15 +801,21 @@ describe('FileDocProvider', () => { ) acceptJoin(fire, doc.clientID, 'doc-1') fire('disconnect') - doc.getText('default').insert(0, 'discard me') + doc.getText('default').insert(0, 'preserve me') const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) await vi.waitFor(async () => expect(await journal.load('doc-1')).not.toBeNull()) - await provider.discardPendingChanges() ;(provider as unknown as { handlePageHide: () => void }).handlePageHide() provider.destroy() - await expect(journal.load('doc-1')).resolves.toBeNull() + const stored = await journal.load('doc-1') + expect(stored).not.toBeNull() + const recovered = new Y.Doc() + Y.applyUpdate(recovered, stored!.recoverySnapshot!) + Y.applyUpdate(recovered, stored!.pendingUpdate) + expect(recovered.getText('default').toString()).toBe('preserve me') + recovered.destroy() + doc.destroy() }) it('hydrates the complete local draft before reporting a replaced document', async () => { @@ -890,8 +896,7 @@ describe('FileDocProvider', () => { updatedAt: Date.now(), }) const scope = { workspaceId: 'workspace-1', userId: 'user-1' } - const discard = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'discard').mockResolvedValue() - const { socket, fire } = createSocket(true) + const { socket, emit, fire } = createSocket(true) const doc = new Y.Doc() const provider = new FileDocProvider( socket, @@ -904,12 +909,10 @@ describe('FileDocProvider', () => { await vi.waitFor(() => expect(provider.joinError).toMatchObject({ code: 'INVALID_UPDATE' })) expect(doc.getText('default').toString()).toBe('') - await provider.discardPendingChanges() - expect(discard).toHaveBeenCalledWith('doc-1') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) provider.destroy() doc.destroy() load.mockRestore() - discard.mockRestore() }) it('ignores an obsolete schema rejection when recovery finishes after reconnecting', async () => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index af4ddca55bf..bb5b3f4718a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -170,8 +170,6 @@ export class FileDocProvider extends ObservableV2 { private updateFlushInProgress = false private recoveryApplied = false private recoveryQueued = false - private recoveryDocId: string | null = null - private pendingChangesDiscarded = false private beforeUnloadProtected = false private readonly journal: PendingFileDocUpdateJournal | null private readonly journalLoad: ReturnType @@ -434,7 +432,6 @@ export class FileDocProvider extends ObservableV2 { } if (recovered !== null && !this.recoveryApplied) { - this.recoveryDocId = recovered.docId const validationDoc = new Y.Doc() try { if (recovered.recoverySnapshot) { @@ -442,10 +439,7 @@ export class FileDocProvider extends ObservableV2 { } Y.applyUpdate(validationDoc, recovered.pendingUpdate) } catch { - this.failFatally( - 'The local recovery copy is damaged. Download the current draft before discarding it.', - 'INVALID_UPDATE' - ) + this.failFatally('The local recovery copy is damaged.', 'INVALID_UPDATE') return } finally { validationDoc.destroy() @@ -456,10 +450,7 @@ export class FileDocProvider extends ObservableV2 { } Y.applyUpdate(this.doc, recovered.pendingUpdate, RECOVERY_ORIGIN) } catch { - this.failFatally( - 'The local recovery copy could not be restored. Download the current draft before discarding it.', - 'INVALID_UPDATE' - ) + this.failFatally('The local recovery copy could not be restored.', 'INVALID_UPDATE') return } this.recoveryApplied = true @@ -760,16 +751,13 @@ export class FileDocProvider extends ObservableV2 { ? Y.mergeUpdates([this.inFlightUpdate.update, update]) : update const saved = await this.journal?.save(docId, journalUpdate, Y.encodeStateAsUpdate(this.doc)) - if (this.disposed || this.fatal || this.pendingChangesDiscarded) { - if (!this.pendingChangesDiscarded) this.queuePendingUpdate(update) + if (this.disposed || this.fatal) { + this.queuePendingUpdate(update) return } if (saved?.status === 'limit-exceeded') { this.queuePendingUpdate(update) - this.failFatally( - 'Local edits exceeded the safe recovery limit; download your draft before reloading', - 'PENDING_UPDATE_LIMIT' - ) + this.failFatally('Local edits exceeded the safe recovery limit.', 'PENDING_UPDATE_LIMIT') return } const durableUpdate = saved?.pendingUpdate ?? update @@ -881,7 +869,6 @@ export class FileDocProvider extends ObservableV2 { } private persistPendingSnapshot(): Promise | undefined { - if (this.pendingChangesDiscarded) return const update = this.pendingJournalUpdate() const docId = this.docId() if (!update || !docId || !this.journal) return @@ -901,7 +888,6 @@ export class FileDocProvider extends ObservableV2 { if (typeof window === 'undefined') return const shouldProtect = !this.disposed && - !this.pendingChangesDiscarded && (this.pendingUpdateBatch.length > 0 || this.inFlightUpdate !== null || this.updateFlushInProgress) @@ -911,17 +897,6 @@ export class FileDocProvider extends ObservableV2 { else window.removeEventListener('beforeunload', this.handleBeforeUnload) } - /** Remove the stale recovery record before deliberately loading a replacement document. */ - discardPendingChanges(): Promise { - this.pendingChangesDiscarded = true - this.clearUpdateTimers() - this.pendingUpdateBatch = [] - this.inFlightUpdate = null - this.updateBeforeUnloadProtection() - const docId = this.recoveryDocId ?? this.docId() - return docId ? (this.journal?.discard(docId) ?? Promise.resolve()) : Promise.resolve() - } - private clearBufferedMessages(): void { this.bufferedMessages = [] this.bufferedMessageBytes = 0 diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts index 7cdaac391b7..0f32bd41c79 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts @@ -176,14 +176,14 @@ describe('PendingFileDocUpdateJournal', () => { await expect(subject.load()).resolves.toMatchObject({ docId: 'doc-4' }) }) - it('discards only the selected document identity', async () => { + it('clears only the acknowledged document identity', async () => { const subject = journal() const oldUpdate = updateWith('old') const currentUpdate = updateWith('current') await subject.save('old-doc', oldUpdate, oldUpdate) await subject.save('current-doc', currentUpdate, currentUpdate) - await subject.discard('old-doc') + await subject.clear('old-doc', oldUpdate) await expect(subject.load('old-doc')).resolves.toBeNull() await expect(subject.load('current-doc')).resolves.toMatchObject({ docId: 'current-doc' }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts index b19a454edef..b2f129674b3 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts @@ -178,19 +178,7 @@ export class PendingFileDocUpdateJournal { ) } - /** Deliberately abandon one recovery identity after the user has preserved its local draft. */ - discard(docId: string): Promise { - return this.enqueue( - () => - updateValue(this.key, (value) => - record(liveDocuments(value, Date.now()).filter((document) => document.docId !== docId)) - ), - undefined, - true - ) - } - - private enqueue(operation: () => Promise, fallback: T, rethrow = false): Promise { + private enqueue(operation: () => Promise, fallback: T): Promise { const result = this.mutationQueue.then(operation, operation) this.mutationQueue = result.then( () => undefined, @@ -198,7 +186,6 @@ export class PendingFileDocUpdateJournal { ) return result.catch((error) => { logger.warn('Failed to persist pending file edits', { error }) - if (rethrow) throw error return fallback }) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx index 70bcd21f71c..744ecfa9178 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx @@ -17,10 +17,9 @@ import { } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/paste-admission' import { LoadedRichMarkdownEditor } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor' -const { collaborationRef, uploadFile, saveBlob } = vi.hoisted(() => ({ +const { collaborationRef, uploadFile } = vi.hoisted(() => ({ collaborationRef: { current: null as unknown }, uploadFile: vi.fn(), - saveBlob: vi.fn(), })) vi.mock('next/navigation', () => ({ @@ -32,7 +31,6 @@ vi.mock('@/hooks/queries/workspace-files', () => ({ useUploadWorkspaceFile: () => ({ mutateAsync: uploadFile }), })) vi.mock('@/hooks/use-add-to-chat', () => ({ useAddToChat: () => vi.fn() })) -vi.mock('@/lib/uploads/client/download', () => ({ saveBlob })) vi.mock('@/hooks/use-file-content-source', () => ({ useFileContentSource: () => ({ resolveImageSrc: (src: string) => src }), })) @@ -103,7 +101,6 @@ const onChange = vi.fn() const onEditSource = vi.fn() const onClientAutosaveChange = vi.fn() const onSaveShortcut = vi.fn() -const onDownloadDraft = vi.fn() const onSuspendedRender = vi.fn() const pendingRender = new Promise(() => {}) @@ -122,7 +119,6 @@ function SuspendAfterEditor({ active }: SuspendAfterEditorProps) { class FakeFileDocProvider { synced = false joinError: JoinFileDocError | null = null - discardPendingChanges = vi.fn(() => Promise.resolve()) private readonly listeners = new Map void>>() on(event: string, listener: (value: unknown) => void) { @@ -182,7 +178,6 @@ async function render( onEditSource={onEditSource} onClientAutosaveChange={onClientAutosaveChange} onSaveShortcut={options.onSaveShortcut ?? onSaveShortcut} - onDownloadDraft={onDownloadDraft} /> @@ -261,9 +256,13 @@ describe('loaded rich editor lifecycle', () => { expect(editor.isEditable).toBe(true) expect(editor.view.dom.getAttribute('aria-readonly')).toBe('false') expect(container.textContent).not.toContain('Reconnecting…') + expect(container.querySelector('[role="status"]')).toBeNull() + expect(container.querySelector('[role="alert"]')).toBeNull() + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() }) - it('keeps a revoked unacknowledged draft visible, read-only, and downloadable', async () => { + it('keeps revoked pending edits visible and read-only without draft-management prompts', async () => { const provider = new FakeFileDocProvider() const doc = new Y.Doc() doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) @@ -294,7 +293,13 @@ describe('loaded rich editor lifecycle', () => { expect(editor.view.dom.closest('.hidden')).toBeNull() expect(container.textContent).not.toContain('stale opening snapshot') expect(container.textContent).not.toContain('Reconnecting…') - expect(container.textContent).toContain('Download local draft') + expect(container.querySelector('[role="status"]')?.textContent).toBe( + 'You no longer have edit access to this document.' + ) + expect(container.querySelector('button')).toBeNull() + expect(container.querySelector('[role="alert"], [role="dialog"]')).toBeNull() + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() }) it('shows stored content read-only when collaboration fails before the first sync', async () => { @@ -326,7 +331,7 @@ describe('loaded rich editor lifecycle', () => { }) it.each(['DOCUMENT_REPLACED', 'PENDING_UPDATE_LIMIT', 'INVALID_UPDATE'])( - 'keeps a local draft downloadable before offering a destructive reload for %s', + 'preserves pending edits with only a passive status for %s', async (code) => { const provider = new FakeFileDocProvider() const doc = new Y.Doc() @@ -350,21 +355,15 @@ describe('loaded rich editor lifecycle', () => { }) ) - const buttons = [...container.querySelectorAll('button')] - const download = buttons.find((button) => button.textContent === 'Download local draft') - expect(download).toBeDefined() - expect(buttons.some((button) => button.textContent === 'Discard draft')).toBe(true) - await act(async () => download?.click()) - expect(onDownloadDraft).not.toHaveBeenCalled() - expect(saveBlob).toHaveBeenCalledOnce() - const downloaded = saveBlob.mock.calls[0][0] as Blob - const downloadedText = await new Promise((resolve, reject) => { - const reader = new FileReader() - reader.onload = () => resolve(String(reader.result)) - reader.onerror = () => reject(reader.error) - reader.readAsText(downloaded) - }) - expect(downloadedText).toContain('preserved local change') + expect(container.querySelector('[role="status"]')?.textContent).toBe( + 'Live editing is unavailable.' + ) + expect(container.querySelector('button')).toBeNull() + expect(container.querySelector('[role="alert"], [role="dialog"]')).toBeNull() + expect(container.textContent).not.toContain('Reconnecting…') + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() + expect(getEditor().isEditable).toBe(false) expect(getEditor().getText()).toContain('preserved local change') } ) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx index 8f6fd1c251d..02bec540177 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx @@ -58,11 +58,17 @@ function revealBubbleMenu(editor: Editor, key: PluginKey): void { editor.commands.setMeta(key, 'updatePosition') } -/** Selects a caret's complete link so the bookmark, URL, and active controls share one target. */ -function linkSelectionBookmark(editor: Editor): SelectionBookmark | null { +interface LinkSelection { + target: SelectionBookmark + original: SelectionBookmark +} + +/** Keep the editing target separate from the selection restored when the user cancels. */ +function captureLinkSelection(editor: Editor): LinkSelection | null { + const original = editor.state.selection.getBookmark() if (editor.state.selection.empty) editor.commands.extendMarkRange('link') const { selection } = editor.state - return selection.empty ? null : selection.getBookmark() + return selection.empty ? null : { target: selection.getBookmark(), original } } interface EditorBubbleMenuProps { @@ -86,7 +92,7 @@ export function EditorBubbleMenu({ }: EditorBubbleMenuProps) { const [linkValue, setLinkValue] = useState(null) const linkInputRef = useRef(null) - const linkRangeRef = useRef(null) + const linkSelectionRef = useRef(null) const isEditingLink = linkValue !== null const [bubbleMenuKey] = useState(() => new PluginKey('markdownBubbleMenu')) @@ -129,18 +135,25 @@ export function EditorBubbleMenu({ transaction: Transaction appendedTransactions?: Transaction[] }) => { - let bookmark = linkRangeRef.current - if (!bookmark) return - for (const change of [transaction, ...appendedTransactions]) - bookmark = bookmark.map(change.mapping) - const selection = bookmark.resolve(editor.state.doc) - linkRangeRef.current = - selection instanceof TextSelection && !selection.empty ? bookmark : null - if (!linkRangeRef.current) setLinkValue(null) + let captured = linkSelectionRef.current + if (!captured) return + for (const change of [transaction, ...appendedTransactions]) { + captured = { + target: captured.target.map(change.mapping), + original: captured.original.map(change.mapping), + } + } + const selection = captured.target.resolve(editor.state.doc) + linkSelectionRef.current = + selection instanceof TextSelection && !selection.empty ? captured : null + if (!linkSelectionRef.current) setLinkValue(null) } const exitOnCollapse = () => { const { from, to } = editor.state.selection - if (from === to) setLinkValue(null) + if (from === to) { + linkSelectionRef.current = null + setLinkValue(null) + } } editor.on('selectionUpdate', exitOnCollapse) editor.on('transaction', mapLinkRange) @@ -184,9 +197,9 @@ export function EditorBubbleMenu({ const openLinkEditor = () => { if (!editor.isEditable || editor.isActive('codeBlock') || editor.isActive('code')) return - const bookmark = linkSelectionBookmark(editor) - if (!bookmark) return - linkRangeRef.current = bookmark + const captured = captureLinkSelection(editor) + if (!captured) return + linkSelectionRef.current = captured setLinkValue(editor.getAttributes('link').href ?? '') } @@ -204,10 +217,10 @@ export function EditorBubbleMenu({ return if (event.key?.toLowerCase() !== 'k') return if (editor.isActive('codeBlock') || editor.isActive('code')) return - const bookmark = linkSelectionBookmark(editor) - if (!bookmark) return + const captured = captureLinkSelection(editor) + if (!captured) return event.preventDefault() - linkRangeRef.current = bookmark + linkSelectionRef.current = captured setLinkValue(editor.getAttributes('link').href ?? '') } dom.addEventListener('keydown', openLinkOnShortcut) @@ -218,19 +231,28 @@ export function EditorBubbleMenu({ const commitCapturedLink = (href: string) => { if (editor.isDestroyed || !editor.isEditable) return - const selection = linkRangeRef.current?.resolve(editor.state.doc) + const selection = linkSelectionRef.current?.target.resolve(editor.state.doc) if (selection instanceof TextSelection && !selection.empty) { applyLink( editor.chain().focus().setTextSelection({ from: selection.from, to: selection.to }), href ) } - linkRangeRef.current = null + linkSelectionRef.current = null setLinkValue(null) } const commitLink = () => commitCapturedLink(linkValue ?? '') const removeLink = () => commitCapturedLink('') + const cancelLink = () => { + const captured = linkSelectionRef.current + linkSelectionRef.current = null + setLinkValue(null) + if (!captured || editor.isDestroyed) return + editor.view.dispatch(editor.state.tr.setSelection(captured.original.resolve(editor.state.doc))) + editor.commands.focus() + } + const { resolveAnchor, appendTo } = useBubbleMenuFloating(editor, scrollContainerRef) const canFocus = useCallback( () => hasFormattableSelection(editor, editor.state.selection.from, editor.state.selection.to), @@ -241,6 +263,7 @@ export function EditorBubbleMenu({ pluginKey: bubbleMenuKey, roving: !isEditingLink, canFocus, + onEscape: isEditingLink ? cancelLink : undefined, }) const shouldShow = useCallback( @@ -279,7 +302,7 @@ export function EditorBubbleMenu({ value={linkValue ?? ''} onChange={setLinkValue} onCommit={commitLink} - onCancel={() => setLinkValue(null)} + onCancel={cancelLink} /> {active.link && ( diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx index b798501d8e2..817cfbf314a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx @@ -373,14 +373,15 @@ describe('real editor BubbleMenu keyboard integration', () => { } ) - it('cancels a caret-opened URL draft without changing the link or losing its selection', async () => { + it.each([0, 2, 6])('restores the original caret at link offset %i on cancel', async (offset) => { act(() => editor.commands.setContent( editorNormalForm('before [format](https://example.com/original) after') ) ) - const input = await openLinkAtCaret(2) - const selection = editor.state.selection.toJSON() + select('format', true) + const caret = editor.state.selection.from + offset + const input = await openLinkAtCaret(offset) const before = editor.getJSON() changeUrl(input, 'https://example.com/cancelled') key(input, 'Escape') @@ -388,8 +389,71 @@ describe('real editor BubbleMenu keyboard integration', () => { expect(viewport.contains(input)).toBe(false) expect(document.activeElement).toBe(editor.view.dom) - expect(editor.state.selection.toJSON()).toEqual(selection) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret) + expect(editor.getJSON()).toEqual(before) + act(() => editor.commands.insertContent('X')) + expect(editor.getText()).toBe( + `before ${'format'.slice(0, offset)}X${'format'.slice(offset)} after` + ) + }) + + it('maps the original caret through peer and appended edits before canceling', async () => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + select('format', true) + const caret = editor.state.selection.from + 2 + const input = await openLinkAtCaret(2) + editor.registerPlugin( + new Plugin({ + appendTransaction: (transactions, _oldState, newState) => + transactions.some((transaction) => transaction.getMeta('toolbar-prefix')) + ? newState.tr.insertText('appended ', 1) + : null, + }) + ) + act(() => editor.setEditable(false)) + act(() => + editor.view.dispatch(editor.state.tr.insertText('peer ', 1).setMeta('toolbar-prefix', true)) + ) + act(() => editor.setEditable(true)) + const before = editor.getJSON() + key(input, 'Escape') + await frame() + + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret + 'appended peer '.length) expect(editor.getJSON()).toEqual(before) + act(() => editor.commands.insertContent('X')) + expect(editor.getText()).toBe('appended peer before foXrmat after') + }) + + it.each(['Apply link', 'Remove link'])('restores the caret on Escape from %s', async (label) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + select('format', true) + const caret = editor.state.selection.from + 2 + const input = await openLinkAtCaret(2) + changeUrl(input, 'https://example.com/cancelled') + const action = button(linkGroup(), label) + act(() => action.focus()) + key(action, 'Escape') + await frame() + + expect(viewport.contains(input)).toBe(false) + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret) + expect(editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/original' + ) }) it('maps the captured link target through a prefix edit and an appended transaction', async () => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts index da99cbb72b3..f86e2220fde 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts @@ -15,6 +15,7 @@ interface EditorToolbarOptions { canFocus: () => boolean /** URL editing uses ordinary form tab order so its native arrow keys do not trap action buttons. */ roving?: boolean + onEscape?: () => void } function controls(toolbar: HTMLElement): HTMLElement[] { @@ -33,6 +34,7 @@ export function useEditorToolbar({ pluginKey, canFocus, roving = true, + onEscape, }: EditorToolbarOptions) { const ref = useRef(null) @@ -113,6 +115,7 @@ export function useEditorToolbar({ ) return if (event.key === 'Escape') { + if (!event.defaultPrevented) onEscape?.() event.preventDefault() editor.commands.focus() editor.commands.setMeta(pluginKey, 'hide') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 7ae83d24c24..ce1eb7bdc09 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -1,9 +1,8 @@ 'use client' import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' -import { Chip, ChipConfirmModal, cn, toast } from '@sim/emcn' +import { Chip, cn, toast } from '@sim/emcn' import { FILE_DOC_SEED, type JoinFileDocError } from '@sim/realtime-protocol/file-doc' -import { getErrorMessage } from '@sim/utils/errors' import { PASTE_LIMITS, PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import type { Extensions, JSONContent, Range } from '@tiptap/core' import { isChangeOrigin } from '@tiptap/extension-collaboration' @@ -15,7 +14,6 @@ import { buildFileSelectionLabel, truncateSelectionText, } from '@/lib/copilot/chat/selection-context' -import { saveBlob } from '@/lib/uploads/client/download' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractEmbeddedFileRef, extractImgSrcs } from '@/lib/uploads/utils/embedded-image-ref' import { FindBar } from '@/app/workspace/[workspaceId]/components' @@ -32,7 +30,6 @@ import { beginAgentStream, endAgentStream, } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown' -import type { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' import { isCollabReady } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness' import { useFileDocCollaboration } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration' import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' @@ -105,81 +102,6 @@ function warnRichMarkdownPasteLimit(reason?: 'paste' | 'formatting') { }) } -interface CollaborationFailureBannerProps { - failure: JoinFileDocError - provider: FileDocProvider | null - onDownloadDraft: () => void -} - -/** Keeps a stale local draft recoverable when live editing must stop rather than silently retry. */ -function CollaborationFailureBanner({ - failure, - provider, - onDownloadDraft, -}: CollaborationFailureBannerProps) { - const [confirmDiscardOpen, setConfirmDiscardOpen] = useState(false) - const [isDiscarding, setIsDiscarding] = useState(false) - const requiresDraftRecovery = - failure.code === 'DOCUMENT_REPLACED' || - failure.code === 'PENDING_UPDATE_LIMIT' || - failure.code === 'INVALID_UPDATE' - const accessLost = failure.code === 'ACCESS_REVOKED' || failure.code === 'ACCESS_DENIED' - const discardPendingChangesAndReload = async () => { - setIsDiscarding(true) - try { - await provider?.discardPendingChanges() - window.location.reload() - } catch (error) { - toast.error(getErrorMessage(error, 'Could not discard the stale local recovery copy.')) - setIsDiscarding(false) - } - } - - const message = accessLost - ? 'You no longer have edit access to this document.' - : failure.code === 'DOCUMENT_REPLACED' - ? 'The live document changed while this tab was disconnected. Your local draft is preserved.' - : failure.code === 'PENDING_UPDATE_LIMIT' - ? 'Local edits exceeded this browser’s safe recovery limit. Download your draft before reloading.' - : failure.code === 'INVALID_UPDATE' - ? 'A local edit could not be synchronized safely. Download your draft before reloading.' - : failure.code === 'SCHEMA_VERSION_MISMATCH' - ? 'This app version cannot edit the live document. Reload to update.' - : 'Live editing could not connect. Reload to try again.' - - return ( -
-

{message}

- Download local draft - {requiresDraftRecovery ? ( - <> - setConfirmDiscardOpen(true)}> - Discard draft - - void discardPendingChangesAndReload(), - pending: isDiscarding, - pendingLabel: 'Discarding...', - }} - /> - - ) : !accessLost ? ( - window.location.reload()}>Reload - ) : null} -
- ) -} - /** * The editor's reading column — the centered, padded surface both the live editor and the read-only * {@link ReadOnlyPlaceholder} render into, so the two are geometrically identical and the placeholder → @@ -403,7 +325,6 @@ function RichMarkdownSurface({ onDeriveTitleFromHeading={onDeriveTitleFromHeading} enableFind={enableFind} onEditSource={onEditSource} - onDownloadDraft={downloadDraft} /> ) @@ -440,7 +361,6 @@ interface LoadedRichMarkdownEditorProps { /** See {@link RichMarkdownEditorProps.enableFind}. */ enableFind: boolean onEditSource?: () => void - onDownloadDraft: () => void } type CollaborationStatus = 'connecting' | 'ready' | 'reconnecting' | 'fatal' @@ -477,7 +397,6 @@ export function LoadedRichMarkdownEditor({ onDeriveTitleFromHeading, enableFind, onEditSource, - onDownloadDraft, }: LoadedRichMarkdownEditorProps) { /** Whether this editor mounted mid-stream — if so it starts empty and syncs streamed chunks until settle. */ const [streamingAtMount] = useState(isStreaming) @@ -1395,16 +1314,6 @@ export function LoadedRichMarkdownEditor({ useSelectionCopyBridge(containerRef, buildSelectionContext, workspaceId) - const downloadLiveDraft = () => { - if (!editor) { - onDownloadDraft() - return - } - const body = postProcessSerializedMarkdown(editor.getMarkdown()) - const markdown = applyFrontmatter(saveFrontmatterResolverRef.current(), body) - saveBlob(new Blob([markdown], { type: 'text/markdown;charset=utf-8' }), file.name) - } - /** Use the stored-content placeholder only while the live document is bootstrapping. */ const showPlaceholder = collaborationEnabled && collabStatus === 'connecting' const showReconnecting = collaborationEnabled && collabStatus === 'reconnecting' @@ -1466,11 +1375,15 @@ export function LoadedRichMarkdownEditor({
)} {showCollabFailure && ( - +
+ {showCollabFailure.code === 'ACCESS_REVOKED' || showCollabFailure.code === 'ACCESS_DENIED' + ? 'You no longer have edit access to this document.' + : 'Live editing is unavailable.'} +
)} {find.isOpen && ( Date: Sat, 5 Sep 2026 11:47:51 -0700 Subject: [PATCH 4/9] fix(files): simplify replay and isolate invalid recovery --- .../src/handlers/file-doc-store.test.ts | 69 ++++ apps/realtime/src/handlers/file-doc-store.ts | 31 +- apps/realtime/src/handlers/file-doc.ts | 25 +- .../collaboration/file-doc-provider.test.ts | 294 ++++++++++-------- .../collaboration/file-doc-provider.ts | 22 +- .../pending-update-journal.test.ts | 113 ++++++- .../collaboration/pending-update-journal.ts | 75 ++++- .../find/find-extension.ts | 1 - .../rich-markdown-editor/image-inspector.tsx | 1 - .../image-resize.test.tsx | 36 ++- .../rich-markdown-editor/image-schema.ts | 9 +- .../rich-markdown-editor/image.tsx | 10 +- .../menus/bubble-menu.tsx | 6 +- .../rich-markdown-editor/round-trip-safety.ts | 6 +- packages/realtime-protocol/src/file-doc.ts | 2 - 15 files changed, 477 insertions(+), 223 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index bda4a725ef9..ab7e04f7819 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -661,6 +661,75 @@ describe('FileDocStore', () => { recovered.destroy() }) + it.each(['headless', 'attached'] as const)( + 'does not recount a replacement snapshot near the byte budget during %s replay', + async (mode) => { + const streamKey = `filedoc:stream:${NAME}` + const source = new Y.Doc() + source.getText('body').insert(0, 'x'.repeat(10 * 1024 * 1024)) + const initial = Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64') + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 8 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: index === 0 ? initial : noop }, + })) + ) + source.getText('body').insert(source.getText('body').length, ' joined') + const compacted = Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64') + state.backing!.seq = 8 + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(4-0') return + state.backing!.streams.set(streamKey, [{ id: '9-0', message: { u: compacted, s: '1' } }]) + state.backing!.seq = 9 + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + if (mode === 'attached') await store.attachRoom(NAME, recovered) + else Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe(source.getText('body').toString()) + } finally { + store.detachRoom(NAME) + recovered.destroy() + source.destroy() + } + } + ) + + it('does not recount retained entries when compaction meets the exact entry budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + const entries = Array.from({ length: 1_999 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + state.backing!.streams.set(streamKey, entries) + state.backing!.seq = 1_999 + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(1996-0') return + state.backing!.streams.set(streamKey, [ + ...entries.slice(1_996), + { + id: '2000-0', + message: { u: Buffer.from(updateFor('complete')).toString('base64'), s: '1' }, + }, + ]) + state.backing!.seq = 2_000 + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('complete') + } finally { + recovered.destroy() + } + }) + it('reads the replacement snapshot when peer deltas cross the old replay tail', async () => { const streamKey = `filedoc:stream:${NAME}` const source = new Y.Doc() diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 2870e14712e..bd872b100b3 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -73,12 +73,7 @@ const INVALIDATE_DOCUMENT_SCRIPT = const ADOPT_GENERATION_SCRIPT = "local generation = redis.call('get', KEYS[2]); if generation then return generation end; if redis.call('xlen', KEYS[1]) == 0 then return false end; redis.call('set', KEYS[2], ARGV[1], 'EX', ARGV[2]); return ARGV[1]" -/** - * Append an ordinary update only while the live-document generation is valid. A durable replacement - * that cannot be represented by the rich editor first sets the invalidation tombstone and then removes - * the stream; keeping the guard and XADD in one script prevents an old room from recreating that stream. - * Returns the new stream id, or `false` while invalidated. - */ +/** Atomically fence XADD so stale rooms cannot recreate a replaced or expired stream. Returns false when fenced. */ const APPEND_UPDATE_SCRIPT = "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; if ARGV[3] ~= '' then return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1') else return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]) end" @@ -403,13 +398,8 @@ export class FileDocStore { } /** - * PULL the shared state into a registered room: read the stream and apply every entry the doc has - * not integrated yet (origin {@link REDIS_ORIGIN}), advancing `lastId` so the tailer resumes exactly - * after it. This is the ONLY way a room loads shared state, so a caller that must not depend on the - * tailer's asynchronous push — the join, which may not serve a client a half-assembled document — - * can converge on demand. Idempotent and safe to call repeatedly; no-op when disabled or the room is - * not registered (a fast open→close detached it). Throws without applying partial state when replay - * cannot complete, so a join never serves a prefix of the shared document. + * Completes shared replay before applying entries, so joins never receive a partial document. + * Repeated calls skip integrated entries; detached rooms and disabled stores are ignored. */ async catchUp(name: string): Promise { if (!this.enabled) return @@ -543,10 +533,8 @@ export class FileDocStore { } /** - * Append a user update exactly once within the stream's bounded deduplication window and return only - * after Redis has accepted it. The client keeps the batch in IndexedDB until this promise succeeds - * and its socket acknowledgement arrives, so a relay restart or lost acknowledgement is safe to - * retry throughout that window. + * Waits for Redis acceptance before the relay acknowledges the client. Retries are deduplicated + * within the bounded window; clients retain their journal until the acknowledgement arrives. */ async publishClientUpdateAndWait( name: string, @@ -656,10 +644,8 @@ export class FileDocStore { } /** - * Invalidate the current live-document generation after a durable replacement that the rich editor - * cannot represent. The invalid generation marker is written before deleting the stream, so stale - * room publishers cannot recreate it; the next authoritative seed atomically replaces the marker - * with its generation. The same TTL as the stream bounds abandoned markers. + * Fences an unsupported durable replacement before deleting its stream. The next authoritative + * seed replaces the tombstone; abandoned tombstones expire with the stream TTL. */ async invalidateDocument(name: string, version: number): Promise { if (!this.enabled) { @@ -830,12 +816,11 @@ export class FileDocStore { } } - /** A moving head means compaction may have removed unread dependencies. Replay its snapshot. */ + /** Compaction appends its snapshot before trimming; extend the barrier without rereading it. */ const currentFirstId = (await this.write.xRange(key, '-', '+', { COUNT: 1 }))[0]?.id if (!currentFirstId) throw new FileDocInvalidatedError() if (currentFirstId === firstId) return entriesRead firstId = currentFirstId - cursor = '0-0' const currentTail = await this.write.xRevRange(key, '+', '-', { COUNT: 1 }) if (currentTail.length === 0) throw new FileDocInvalidatedError() endId = currentTail[0].id diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 5a4faae2bae..80d7e53d22a 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -268,9 +268,8 @@ const AGENT_SYNC_ORIGIN = Symbol('file-doc-agent-sync') const MAX_LEGACY_FRAME_BYTES = FILE_DOC_LIMITS.updateBytes + 64 /** - * Preflight the update-bearing inner Yjs message before `readSyncMessage` can mutate the room. Legacy - * clients use the unacknowledged sync channel, so the relay itself must ensure any applied update also - * fits the durable Redis stream; checking only the outer frame leaves a small framing-sized gap. + * Checks the inner update before readSyncMessage mutates the room: the legacy outer-frame limit + * includes framing headroom, which must not allow an update too large for the shared stream. */ function hasOversizedLegacyUpdate(bytes: Uint8Array): boolean { const decoder = decoding.createDecoder(bytes) @@ -441,15 +440,11 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr await store.setSyncedVersion(name, result.version, generation) return } - // status === 'conflict': the durable file advanced out-of-band since our If-Match token. We do NOT - // re-persist against the current stream: an external write commits durable BEFORE its chokepoint merge - // (`applyEditToLiveFileDoc`) reaches the stream, so a re-persist landing in that window would CAS-pass - // with a stream that still lacks the external content and clobber the committed write. Instead leave the - // durable content authoritative — the chokepoint merges the change into the stream and, ONLY once it is - // actually there, advances the synced version (via the merge's own `recordVersion`); a later flush - // (a subsequent debounced persist, or the final flush) then projects the converged stream with a token - // that matches. The session's edits stay in the stream meanwhile. Deliberately do NOT advance the synced - // version here: before the stream reflects the durable content, that would let the next flush clobber it. + /** + * External writes commit before merging into the stream. Retrying or advancing the synced + * version here could overwrite content not yet merged; leave the durable file authoritative + * until the merge advances the version, then let a later flush persist the converged state. + */ logger.warn( `Persist conflict for file ${room.fileId}; durable content advanced out-of-band, left authoritative` ) @@ -579,10 +574,8 @@ export async function flushAllFileDocRooms(): Promise { } /** - * Bring a room's document to its AUTHORITATIVE state — reflecting the file's shared stream and - * carrying its seed — so the join can attach a client to a document that is already whole. Rejects - * when hydration or seeding cannot complete; serving an unseeded or partial room would make a client - * appear editable before the authoritative document exists. + * Waits for shared hydration and authoritative seeding before joining; failures must not expose + * an editable partial document. */ async function ensureRoomReady( name: string, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index 740886194cb..a85c85c7ddc 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -370,7 +370,7 @@ describe('FileDocProvider', () => { expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) }) - it('uses ordinary Yjs synchronization without stale unload protection on a no-ACK relay', () => { + it('protects only unsent changes when a legacy relay provides no document identity', () => { const browserWindow = new EventTarget() vi.stubGlobal('window', browserWindow) const { provider, doc, awareness, emit, fire } = createProvider(true) @@ -386,10 +386,12 @@ describe('FileDocProvider', () => { expect(unloadIsPrevented()).toBe(true) acceptJoin(fire, doc.clientID, undefined, false) expect(unloadIsPrevented()).toBe(false) + doc.getText('default').insert(11, ' online') + expect(unloadIsPrevented()).toBe(false) fire('disconnect') - doc.getText('default').insert(11, ' and offline') - expect(unloadIsPrevented()).toBe(false) + doc.getText('default').insert(18, ' and offline') + expect(unloadIsPrevented()).toBe(true) fire('connect') acceptJoin(fire, doc.clientID, undefined, false) emit.mockClear() @@ -400,8 +402,8 @@ describe('FileDocProvider', () => { decoding.readVarUint(decoder) syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) } - expect(serverDoc.getText('default').toString()).toBe('before join and offline') - expect(unloadIsPrevented()).toBe(false) + expect(serverDoc.getText('default').toString()).toBe('before join online and offline') + expect(unloadIsPrevented()).toBe(true) expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) } finally { provider.destroy() @@ -412,86 +414,96 @@ describe('FileDocProvider', () => { } }) - it('recovers an unacknowledged edit after a tab restart and clears it only after acceptance', async () => { - journalStorage.clear() - const scope = { workspaceId: 'workspace-1', userId: 'user-1' } - const serverDoc = new Y.Doc() - const serverConfig = serverDoc.getMap(FILE_DOC_SEED.configMap) - serverConfig.set(FILE_DOC_SEED.docIdKey, 'doc-1') - serverConfig.set(FILE_DOC_SEED.flag, true) - serverDoc.getText('default').insert(0, 'base') - - const firstSocket = createSocket(true) - const firstDoc = new Y.Doc() - Y.applyUpdate(firstDoc, Y.encodeStateAsUpdate(serverDoc)) - const firstProvider = new FileDocProvider( - firstSocket.socket, - 'file-1', - firstDoc, - new awarenessProtocol.Awareness(firstDoc), - scope - ) - acceptJoin(firstSocket.fire, firstDoc.clientID, 'doc-1') - firstSocket.emit.mockClear() - firstDoc.getText('default').insert(4, ' local') - await vi.waitFor(() => { + it.each(['acknowledged', 'legacy-offline', 'legacy-rejoining'] as const)( + 'recovers an unacknowledged %s edit after restart and clears it only after acceptance', + async (mode) => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const serverDoc = new Y.Doc() + const serverConfig = serverDoc.getMap(FILE_DOC_SEED.configMap) + serverConfig.set(FILE_DOC_SEED.docIdKey, 'doc-1') + serverConfig.set(FILE_DOC_SEED.flag, true) + serverDoc.getText('default').insert(0, 'base') + + const firstSocket = createSocket(true) + const firstDoc = new Y.Doc() + Y.applyUpdate(firstDoc, Y.encodeStateAsUpdate(serverDoc)) + const firstProvider = new FileDocProvider( + firstSocket.socket, + 'file-1', + firstDoc, + new awarenessProtocol.Awareness(firstDoc), + scope + ) + acceptJoin(firstSocket.fire, firstDoc.clientID, 'doc-1', mode === 'acknowledged') + await vi.waitFor(() => expect(emittedMessages(firstSocket.emit).length).toBeGreaterThan(0)) + if (mode !== 'acknowledged') firstSocket.fire('disconnect') + if (mode === 'legacy-rejoining') firstSocket.fire('connect') + firstSocket.emit.mockClear() + firstDoc.getText('default').insert(4, ' local') + const firstJournal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + expect(await firstJournal.load('doc-1')).not.toBeNull() + }) expect(firstSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe( - true + mode === 'acknowledged' ) - }) - firstProvider.destroy() + firstProvider.destroy() - const secondSocket = createSocket(true) - const secondDoc = new Y.Doc() - const secondProvider = new FileDocProvider( - secondSocket.socket, - 'file-1', - secondDoc, - new awarenessProtocol.Awareness(secondDoc), - scope - ) - acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') - const syncEncoder = encoding.createEncoder() - encoding.writeVarUint(syncEncoder, FILE_DOC_MESSAGE_TYPE.SYNC) - syncProtocol.writeSyncStep2(syncEncoder, serverDoc) - secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(syncEncoder)) - - await vi.waitFor(() => { - expect(secondDoc.getText('default').toString()).toBe('base local') - expect(secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe( - true + const secondSocket = createSocket(true) + const secondDoc = new Y.Doc() + const secondProvider = new FileDocProvider( + secondSocket.socket, + 'file-1', + secondDoc, + new awarenessProtocol.Awareness(secondDoc), + scope ) - }) - const updateCall = secondSocket.emit.mock.calls.find( - ([event]) => event === FILE_DOC_EVENTS.UPDATE - ) - const payload = updateCall?.[1] as { updateId: string } - const acknowledge = updateCall?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void - Y.applyUpdate(serverDoc, (updateCall?.[1] as { update: Uint8Array }).update) - acknowledge(null, { status: 'accepted', updateId: payload.updateId }) - - const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) - await vi.waitFor(async () => { - await expect(journal.load()).resolves.toBeNull() - }) - - vi.useFakeTimers() - try { - secondSocket.fire('disconnect') - secondSocket.fire('connect') - secondSocket.emit.mockClear() acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') - secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) - await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) - expect(secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe( - false + const syncEncoder = encoding.createEncoder() + encoding.writeVarUint(syncEncoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(syncEncoder, serverDoc) + secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(syncEncoder)) + + await vi.waitFor(() => { + expect(secondDoc.getText('default').toString()).toBe('base local') + expect( + secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE) + ).toBe(true) + }) + const updateCall = secondSocket.emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.UPDATE ) - } finally { - vi.useRealTimers() + const payload = updateCall?.[1] as { updateId: string } + const acknowledge = updateCall?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + Y.applyUpdate(serverDoc, (updateCall?.[1] as { update: Uint8Array }).update) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + await expect(journal.load()).resolves.toBeNull() + }) + + vi.useFakeTimers() + try { + secondSocket.fire('disconnect') + secondSocket.fire('connect') + secondSocket.emit.mockClear() + acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') + secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect( + secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE) + ).toBe(false) + } finally { + vi.useRealTimers() + } + secondProvider.destroy() + firstDoc.destroy() + secondDoc.destroy() + serverDoc.destroy() } - secondProvider.destroy() - serverDoc.destroy() - }) + ) it('batches local document edits into the acknowledged update channel', async () => { const { doc, emit, fire } = createProvider(true) @@ -888,31 +900,47 @@ describe('FileDocProvider', () => { oldDoc.destroy() }) - it('fails terminally without partially applying a malformed local recovery record', async () => { - const load = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'load').mockResolvedValue({ - docId: 'doc-1', - recoverySnapshot: null, - pendingUpdate: new Uint8Array([255]), - updatedAt: Date.now(), - }) + it('syncs after malformed recovery without replaying it on subsequent mounts', async () => { + journalStorage.clear() const scope = { workspaceId: 'workspace-1', userId: 'user-1' } - const { socket, emit, fire } = createSocket(true) - const doc = new Y.Doc() - const provider = new FileDocProvider( - socket, - 'file-1', - doc, - new awarenessProtocol.Awareness(doc), - scope + const oldDoc = new Y.Doc() + oldDoc.getText('default').insert(0, 'quarantined snapshot') + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'doc-1', + new Uint8Array([255]), + Y.encodeStateAsUpdate(oldDoc) ) - acceptJoin(fire, doc.clientID, 'doc-1') + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + serverDoc.getText('default').insert(0, 'server content') + const frame = encoding.createEncoder() + encoding.writeVarUint(frame, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(frame, serverDoc) + for (let mount = 0; mount < 2; mount++) { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(frame)) - await vi.waitFor(() => expect(provider.joinError).toMatchObject({ code: 'INVALID_UPDATE' })) - expect(doc.getText('default').toString()).toBe('') - expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) - provider.destroy() - doc.destroy() - load.mockRestore() + expect(provider.joinError).toBeNull() + expect(provider.synced).toBe(true) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(doc.getText('default').toString()).toBe('server content') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + doc.destroy() + } + oldDoc.destroy() + serverDoc.destroy() }) it('ignores an obsolete schema rejection when recovery finishes after reconnecting', async () => { @@ -1041,39 +1069,45 @@ describe('FileDocProvider', () => { remote.destroy() }) - it('stops editing while the complete local recovery snapshot cannot be persisted', async () => { - vi.useFakeTimers() - const save = vi - .spyOn(PendingFileDocUpdateJournal.prototype, 'save') - .mockImplementation(async (_docId, pendingUpdate) => ({ - pendingUpdate, - status: 'limit-exceeded', - })) - try { - const { socket, emit, fire } = createSocket(true) - const doc = new Y.Doc() - doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') - const provider = new FileDocProvider( - socket, - 'file-1', - doc, - new awarenessProtocol.Awareness(doc), - { workspaceId: 'workspace-1', userId: 'user-1' } - ) - acceptJoin(fire, doc.clientID, 'doc-1') - emit.mockClear() + it.each(['acknowledged', 'legacy-offline'] as const)( + 'stops editing when the complete %s recovery snapshot cannot be persisted', + async (mode) => { + vi.useFakeTimers() + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => ({ + pendingUpdate, + status: 'limit-exceeded', + })) + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + { workspaceId: 'workspace-1', userId: 'user-1' } + ) + acceptJoin(fire, doc.clientID, 'doc-1', mode === 'acknowledged') + await vi.advanceTimersByTimeAsync(0) + if (mode === 'legacy-offline') fire('disconnect') + emit.mockClear() - doc.getText('default').insert(0, 'must remain downloadable') - await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + doc.getText('default').insert(0, 'must remain visible') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) - expect(provider.joinError).toMatchObject({ code: 'PENDING_UPDATE_LIMIT' }) - expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) - provider.destroy() - } finally { - save.mockRestore() - vi.useRealTimers() + expect(provider.joinError).toMatchObject({ code: 'PENDING_UPDATE_LIMIT' }) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + doc.destroy() + } finally { + save.mockRestore() + vi.useRealTimers() + } } - }) + ) it.each(['saved', 'unavailable'] as const)( 'warns before unloading pending edits and continues acknowledged saves when storage is %s', diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index bb5b3f4718a..b078c237647 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -381,7 +381,6 @@ export class FileDocProvider extends ObservableV2 { * rebuilt only when the room AND the shared stream are both gone (a tab that slept through it), which * is precisely when a stale tab reconnects. There is no way to un-merge afterwards, so the sync never * happens: take the fatal path, which leaves the editor read-only on the content it already shows. - * A reload binds a fresh document and recovers. */ private handleJoinSuccess = (data: JoinFileDocSuccess) => { if ( @@ -432,18 +431,6 @@ export class FileDocProvider extends ObservableV2 { } if (recovered !== null && !this.recoveryApplied) { - const validationDoc = new Y.Doc() - try { - if (recovered.recoverySnapshot) { - Y.applyUpdate(validationDoc, recovered.recoverySnapshot) - } - Y.applyUpdate(validationDoc, recovered.pendingUpdate) - } catch { - this.failFatally('The local recovery copy is damaged.', 'INVALID_UPDATE') - return - } finally { - validationDoc.destroy() - } try { if (recovered.recoverySnapshot) { Y.applyUpdate(this.doc, recovered.recoverySnapshot, RECOVERY_ORIGIN) @@ -699,14 +686,13 @@ export class FileDocProvider extends ObservableV2 { return } - if (!this.joinAccepted) { - if (this.updateMode !== 'legacy') this.queuePendingUpdate(update) - if (this.updateMode === 'acknowledged') this.scheduleUpdateFlush(UPDATE_BATCH_MS) + if (!this.joinAccepted || !this.socket.connected) { + this.queuePendingUpdate(update) + if (this.updateMode !== 'negotiating') this.scheduleUpdateFlush(UPDATE_BATCH_MS) return } if (this.updateMode !== 'acknowledged') { - if (!this.socket.connected) return const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeUpdate(encoder, update) @@ -734,7 +720,7 @@ export class FileDocProvider extends ObservableV2 { private async flushPendingUpdates(): Promise { if ( - this.updateMode !== 'acknowledged' || + this.updateMode === 'negotiating' || this.pendingUpdateBatch.length === 0 || this.disposed || this.fatal diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts index 0f32bd41c79..5d4a910ccb1 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ import { FILE_DOC_LIMITS } from '@sim/realtime-protocol/file-doc' -import { update as updateValue } from 'idb-keyval' +import { get, update as updateValue } from 'idb-keyval' import { beforeEach, describe, expect, it, vi } from 'vitest' import * as Y from 'yjs' @@ -77,6 +77,117 @@ describe('PendingFileDocUpdateJournal', () => { expect(updateValue).toHaveBeenCalledTimes(writes) }) + it.each(['pendingUpdate', 'recoverySnapshot'] as const)( + 'isolates malformed %s bytes without replaying or deleting them', + async (field) => { + const subject = journal() + const valid = updateWith('preserved snapshot') + const invalid = new Uint8Array([255]) + const pending = field === 'pendingUpdate' ? invalid : valid + const snapshot = field === 'recoverySnapshot' ? invalid : valid + await subject.save('doc-1', pending, snapshot) + + await expect(subject.load('doc-1')).resolves.toBeNull() + await expect(journal().load()).resolves.toBeNull() + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: [ + expect.objectContaining({ + docId: 'doc-1', + pendingUpdate: pending, + recoverySnapshot: snapshot, + quarantined: true, + }), + ], + }), + ]) + + const newUpdate = updateWith('new edits') + await expect(subject.save('doc-1', newUpdate, newUpdate)).resolves.toMatchObject({ + status: 'saved', + pendingUpdate: newUpdate, + }) + await expect(subject.load('doc-1')).resolves.toMatchObject({ pendingUpdate: newUpdate }) + await subject.clear('doc-1', newUpdate) + await expect(subject.load()).resolves.toBeNull() + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: [expect.objectContaining({ pendingUpdate: pending, quarantined: true })], + }), + ]) + } + ) + + it('ignores malformed recovery even if browser storage cannot be updated', async () => { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('doc-1', invalid, invalid) + const before = structuredClone([...storage.values()]) + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Storage denied')) + + await expect(subject.load()).resolves.toBeNull() + expect([...storage.values()]).toEqual(before) + }) + + it('does not quarantine a record that another tab replaced after the read', async () => { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('doc-1', invalid, invalid) + const stale = structuredClone([...storage.values()][0]) + storage.clear() + const valid = updateWith('concurrent valid edits') + await subject.save('doc-1', valid, valid) + vi.mocked(get).mockResolvedValueOnce(stale) + + await expect(subject.load()).resolves.toBeNull() + await expect(subject.load()).resolves.toMatchObject({ pendingUpdate: valid }) + }) + + it('prioritizes valid recovery within the existing record cap', async () => { + const subject = journal() + const valid = updateWith('valid') + await subject.save('first', valid, valid) + const invalid = new Uint8Array([255]) + await subject.save('invalid', invalid, invalid) + await expect(subject.load('invalid')).resolves.toBeNull() + await subject.save('second', valid, valid) + await subject.save('third', valid, valid) + + for (const docId of ['first', 'second', 'third']) { + await expect(subject.load(docId)).resolves.toMatchObject({ docId }) + } + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: expect.arrayContaining([ + expect.objectContaining({ docId: 'first' }), + expect.objectContaining({ docId: 'second' }), + expect.objectContaining({ docId: 'third' }), + ]), + }), + ]) + expect((storage.values().next().value as { documents: unknown[] }).documents).toHaveLength(3) + }) + + it('does not extend malformed recovery retention while quarantining it', async () => { + vi.useFakeTimers() + try { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('invalid', invalid, invalid) + await vi.advanceTimersByTimeAsync(6 * 24 * 60 * 60 * 1_000) + await expect(subject.load()).resolves.toBeNull() + await vi.advanceTimersByTimeAsync(2 * 24 * 60 * 60 * 1_000) + const valid = updateWith('new edits') + await subject.save('current', valid, valid) + + expect([...storage.values()]).toEqual([ + expect.objectContaining({ documents: [expect.objectContaining({ docId: 'current' })] }), + ]) + } finally { + vi.useRealTimers() + } + }) + it('distinguishes unavailable browser storage from a configured size limit', async () => { vi.mocked(updateValue).mockRejectedValueOnce(new Error('Storage denied')) const pendingUpdate = updateWith('pending') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts index b2f129674b3..40f12cdfc6d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts @@ -20,7 +20,11 @@ export interface PendingDocumentRecovery { interface PendingUpdateJournalRecord { version: typeof JOURNAL_VERSION - documents: PendingDocumentRecovery[] + documents: JournalDocument[] +} + +interface JournalDocument extends PendingDocumentRecovery { + quarantined?: boolean } interface PendingUpdateJournalScope { @@ -34,9 +38,9 @@ interface JournalSaveResult { status: 'saved' | 'limit-exceeded' | 'unavailable' } -function isRecovery(value: unknown): value is PendingDocumentRecovery { +function isRecovery(value: unknown): value is JournalDocument { if (typeof value !== 'object' || value === null) return false - const candidate = value as Partial + const candidate = value as Partial return ( typeof candidate.docId === 'string' && candidate.docId.length > 0 && @@ -48,26 +52,32 @@ function isRecovery(value: unknown): value is PendingDocumentRecovery { candidate.recoverySnapshot.byteLength > 0 && candidate.recoverySnapshot.byteLength <= RECOVERY_SNAPSHOT_MAX_BYTES)) && typeof candidate.updatedAt === 'number' && - Number.isFinite(candidate.updatedAt) + Number.isFinite(candidate.updatedAt) && + (candidate.quarantined === undefined || typeof candidate.quarantined === 'boolean') ) } -function liveDocuments(value: unknown, now: number): PendingDocumentRecovery[] { +function liveDocuments(value: unknown, now: number): JournalDocument[] { if (typeof value !== 'object' || value === null) return [] const candidate = value as Partial if (candidate.version !== JOURNAL_VERSION || !Array.isArray(candidate.documents)) return [] return candidate.documents .filter(isRecovery) .filter((document) => now - document.updatedAt <= JOURNAL_TTL_MS) - .sort((left, right) => right.updatedAt - left.updatedAt) + .sort( + (left, right) => + Number(left.quarantined === true) - Number(right.quarantined === true) || + right.updatedAt - left.updatedAt + ) .slice(0, MAX_DOCUMENTS) } -function record(documents: PendingDocumentRecovery[]): PendingUpdateJournalRecord { +function record(documents: JournalDocument[]): PendingUpdateJournalRecord { return { version: JOURNAL_VERSION, documents } } -function sameUpdate(left: Uint8Array, right: Uint8Array): boolean { +function sameUpdate(left: Uint8Array | null, right: Uint8Array | null): boolean { + if (left === null || right === null) return left === right if (left.byteLength !== right.byteLength) return false return left.every((byte, index) => byte === right[index]) } @@ -98,10 +108,25 @@ export class PendingFileDocUpdateJournal { async load(preferredDocId?: string): Promise { try { await this.mutationQueue - const documents = liveDocuments(await get(this.key), Date.now()) - return preferredDocId + const documents = liveDocuments(await get(this.key), Date.now()).filter( + (document) => !document.quarantined + ) + const recovered = preferredDocId ? (documents.find((document) => document.docId === preferredDocId) ?? null) : (documents[0] ?? null) + if (!recovered) return null + const validationDoc = new Y.Doc() + try { + if (recovered.recoverySnapshot) Y.applyUpdate(validationDoc, recovered.recoverySnapshot) + Y.applyUpdate(validationDoc, recovered.pendingUpdate) + return recovered + } catch (error) { + logger.warn('Isolating malformed pending file edits', { error }) + await this.quarantine(recovered) + return null + } finally { + validationDoc.destroy() + } } catch (error) { logger.warn('Failed to load pending file edits', { error }) return null @@ -126,7 +151,9 @@ export class PendingFileDocUpdateJournal { await updateValue(this.key, (value) => { const now = Date.now() const documents = liveDocuments(value, now) - const existing = documents.find((document) => document.docId === docId) + const existing = documents.find( + (document) => document.docId === docId && !document.quarantined + ) const merged = existing ? Y.mergeUpdates([existing.pendingUpdate, pendingUpdate]) : pendingUpdate @@ -145,7 +172,7 @@ export class PendingFileDocUpdateJournal { } const retained = [ next, - ...documents.filter((document) => document.docId !== docId), + ...documents.filter((document) => document.docId !== docId || document.quarantined), ].slice(0, MAX_DOCUMENTS) result = { pendingUpdate: merged, @@ -170,7 +197,9 @@ export class PendingFileDocUpdateJournal { return record( documents.filter( (document) => - document.docId !== docId || !sameUpdate(document.pendingUpdate, acknowledgedUpdate) + document.quarantined || + document.docId !== docId || + !sameUpdate(document.pendingUpdate, acknowledgedUpdate) ) ) }), @@ -178,6 +207,26 @@ export class PendingFileDocUpdateJournal { ) } + /** Retain invalid bytes within the journal's existing bounds without replaying or merging them. */ + private quarantine(recovered: PendingDocumentRecovery): Promise { + return this.enqueue( + () => + updateValue(this.key, (value) => + record( + liveDocuments(value, Date.now()).map((document) => + document.docId === recovered.docId && + document.updatedAt === recovered.updatedAt && + sameUpdate(document.pendingUpdate, recovered.pendingUpdate) && + sameUpdate(document.recoverySnapshot, recovered.recoverySnapshot) + ? { ...document, quarantined: true } + : document + ) + ) + ), + undefined + ) + } + private enqueue(operation: () => Promise, fallback: T): Promise { const result = this.mutationQueue.then(operation, operation) this.mutationQueue = result.then( diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts index 93121d6b4d9..d3c9805f47a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts @@ -195,7 +195,6 @@ function replaceMatch(transaction: EditorState['tr'], match: FindMatch, replacem ) } -/** Replaces the active match as one ordinary editor transaction. */ export function replaceActiveFindMatch( editor: Editor, replacement: string, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx index 677423eb73e..22582b4fca2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx @@ -16,7 +16,6 @@ interface ImageInspectorProps extends ImageDetails { onReturnFocus: () => void } -/** Selection-local controls for accessible image text, links, and explicit sizing. */ export function ImageInspector({ alt, href, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx index ba8b33fd482..0eed9d37670 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx @@ -23,16 +23,19 @@ vi.mock( vi.mock( '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector', - () => ({ ImageInspector: () => null }) + () => ({ ImageInspector: vi.fn(() => null) }) ) import { ResizableImageView } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image' +import { ImageInspector } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector' let host: HTMLDivElement let root: Root const editor = { isEditable: true, isDestroyed: false, commands: { focus: vi.fn() } } beforeEach(() => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }) + vi.clearAllMocks() editor.isEditable = true editor.isDestroyed = false host = document.createElement('div') @@ -89,6 +92,37 @@ function renderImage(updateAttributes: ReturnType): HTMLButtonElem } describe('ResizableImageView', () => { + it.each(['read-only', 'destroyed'] as const)( + 'rejects queued image detail and size changes after the editor becomes %s', + (state) => { + const updateAttributes = vi.fn() + renderImage(updateAttributes) + const inspector = vi.mocked(ImageInspector).mock.calls.at(-1)![0] + if (state === 'read-only') editor.isEditable = false + else editor.isDestroyed = true + + act(() => { + inspector.onApply({ alt: 'changed', href: 'https://example.com' }) + inspector.onResetSize() + }) + expect(updateAttributes).not.toHaveBeenCalled() + } + ) + + it('applies image details and resets dimensions while the editor remains editable', () => { + const updateAttributes = vi.fn() + renderImage(updateAttributes) + const inspector = vi.mocked(ImageInspector).mock.calls.at(-1)![0] + act(() => { + inspector.onApply({ alt: 'changed', href: 'https://example.com' }) + inspector.onResetSize() + }) + expect(updateAttributes.mock.calls).toEqual([ + [{ alt: 'changed', href: 'https://example.com' }], + [{ width: null, height: null }], + ]) + }) + it('keeps the width automatic when only an explicit height is stored', () => { renderImage(vi.fn()) const image = host.querySelector('img') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts index 38f39113f25..dabebe5bc4d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts @@ -45,13 +45,8 @@ function imageAttrsFromHtml(raw: string): Record | null { } /** - * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when - * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to - * preserve its dimensions. Unsized images stay clean `![alt](src)`. An image with an `href` is - * wrapped in a markdown link so a linked badge round-trips as `[![alt](src)](href)`. - * - * Sized linked images use the standard `[](href)` combination, preserving both dimensions and - * link semantics while remaining readable by ordinary Markdown renderers. + * Markdown has no image dimensions, so sized images use HTML. Links wrap either representation: + * `[![alt](src)](href)` or `[](href)`. */ function imageMarkdown(node: JSONContent): string { const attrs = node.attrs ?? {} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx index a7c3a2a0225..10a4bca99ff 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx @@ -234,8 +234,14 @@ export function ResizableImageView({ alt={attrs.alt ?? ''} href={typeof attrs.href === 'string' ? attrs.href : ''} hasCustomSize={Boolean(attrs.width || attrs.height)} - onApply={({ alt, href }) => updateAttributes({ alt, href: href || null })} - onResetSize={() => updateAttributes({ width: null, height: null })} + onApply={({ alt, href }) => { + if (!editor.isEditable || editor.isDestroyed) return + updateAttributes({ alt, href: href || null }) + }} + onResetSize={() => { + if (!editor.isEditable || editor.isDestroyed) return + updateAttributes({ width: null, height: null }) + }} onReturnFocus={() => editor.commands.focus()} /> )} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx index 02bec540177..b58cac3a261 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx @@ -164,10 +164,8 @@ export function EditorBubbleMenu({ }, [editor]) /** - * Linear-style reveal: the toolbar stays hidden while the pointer is down (the drag gate in - * `shouldShow`) and surfaces on release. `pointerup`/`pointercancel`/`blur` listen on `window` so a - * release outside the editor — or a cancelled touch gesture — still clears the drag flag; otherwise it - * could wedge `true` and suppress the toolbar for later keyboard selections. + * Window-level release/cancel/blur handlers clear the drag gate even outside the editor, + * preventing a lost pointer release from suppressing later keyboard selections. */ useEffect(() => { const dom = editor.view.dom diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts index 0504c2d1b9f..ca8aad1e177 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts @@ -45,10 +45,8 @@ const fidelityLexer = new Marked({ gfm: true }) const SUPPORTED_IMAGE_ATTRIBUTES = new Set(['src', 'alt', 'title', 'width', 'height']) /** - * The image node deliberately models only the attributes it can render and serialize. An HTML image - * carrying anything else must stay in source mode; comparing only its `src` would declare a stable but - * lossy conversion safe after the unsupported attribute had already disappeared. - * Duplicate attributes also stay in source mode rather than choosing between conflicting values. + * Unsupported or duplicate HTML image attributes require source mode; a stable serialization + * is not lossless if parsing already discarded those attributes. */ function hasUnsupportedHtmlImageAttribute(content: string): boolean { const tokenizer = new Tokenizer() diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index 705272dcb73..11066530896 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -44,7 +44,6 @@ export const FILE_DOC_EVENTS = { /** Schema assumed for peers from before schema negotiation was added. */ export const FILE_DOC_LEGACY_SCHEMA_VERSION = 1 -/** Current collaborative-document schema understood by this client and relay. */ export const FILE_DOC_SCHEMA_VERSION = 1 /** @@ -205,7 +204,6 @@ export interface FileDocUpdatePayload { update: Uint8Array } -/** Relay acknowledgement for a {@link FileDocUpdatePayload}. */ export type FileDocUpdateAck = | { status: 'accepted'; updateId: string } | { From b6feb867ea90b93765aaf3c7fda3e512e99ba0c9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 5 Sep 2026 12:46:26 -0700 Subject: [PATCH 5/9] fix(files): close editor recovery and rendering gaps --- .../src/handlers/file-doc-store.test.ts | 117 ++++++++++++++++-- apps/realtime/src/handlers/file-doc-store.ts | 103 ++++++++++----- apps/realtime/src/handlers/file-doc.test.ts | 3 +- .../collaboration/file-doc-provider.test.ts | 11 +- .../collaboration/file-doc-provider.ts | 4 +- .../image-resize.test.tsx | 30 ++++- .../rich-markdown-editor/image.tsx | 29 ++++- .../round-trip-safety.test.ts | 16 ++- .../rich-markdown-editor/round-trip-safety.ts | 19 ++- 9 files changed, 274 insertions(+), 58 deletions(-) diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index ab7e04f7819..c05b15c04de 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -138,6 +138,7 @@ function makeClient(): any { }, eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { const [key] = opts.keys + if (script.startsWith('for _, key in ipairs(KEYS)')) return 1 if (script.includes("redis.call('exists', KEYS[1])") && !b().streams.has(key)) { return script.includes('zscore') ? -1 : false } @@ -149,13 +150,16 @@ function makeClient(): any { return opts.arguments[0] } if (script.includes("redis.call('del', KEYS[1], KEYS[4], KEYS[5])")) { - const [, generationKey, versionKey, dedupeKey, agentKey] = opts.keys + const [, generationKey, versionKey, dedupeKey, agentKey, invalidationKey] = opts.keys const [version, , marker] = opts.arguments const current = b().kv.get(versionKey) + const invalidated = b().kv.get(invalidationKey) + if (invalidated && Number(invalidated) >= Number(version)) return 0 if (current && Number(current) > Number(version)) return 0 if (current === version && b().kv.get(generationKey) === marker) return 0 b().kv.set(generationKey, marker) b().kv.set(versionKey, version) + b().kv.set(invalidationKey, version) b().streams.delete(key) b().dedupe.delete(dedupeKey) b().kv.delete(agentKey) @@ -258,7 +262,7 @@ const REDIS_URL = 'redis://fake' const NAME = 'workspace-file-doc:file-1' interface StoreTestAccess { - localInvalidations: Map + localInvalidations: Map rooms: Map< string, { @@ -509,6 +513,41 @@ describe('FileDocStore', () => { await expect(store.getStreamState(NAME)).resolves.toBeNull() }) + it('does not repeat an invalidation after the same durable version is reseeded', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('old'), 10) + await expect(store.invalidateDocument(NAME, 20)).resolves.toBe(true) + await expect(store.seedIfEmpty(NAME, updateFor('replacement'), 20)).resolves.toBe(true) + const generation = await store.getDocumentGeneration(NAME) + const doc = new Y.Doc() + Y.applyUpdate(doc, (await store.getStreamState(NAME))!) + const before = Y.encodeStateVector(doc) + doc.getText('body').insert(11, ' accepted') + await store.publishClientUpdateAndWait( + NAME, + 'accepted-edit', + Y.encodeStateAsUpdate(doc, before), + generation + ) + + await expect(store.invalidateDocument(NAME, 20)).resolves.toBe(false) + expect(await store.getDocumentGeneration(NAME)).toBe(generation) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('replacement accepted') + expect(state.backing!.dedupe.get(`filedoc:updates:${NAME}`)).toHaveLength(1) + doc.destroy() + recovered.destroy() + }) + + it('applies the first invalidation even when its durable version was already seeded', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('same content, changed eligibility'), 20) + await expect(store.invalidateDocument(NAME, 20)).resolves.toBe(true) + await expect(store.invalidateDocument(NAME, 20)).resolves.toBe(false) + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + it('does not resurrect a tracked stream with a dependency-only update after Redis loses it', async () => { const store = await newStore() await store.seedIfEmpty(NAME, updateFor('base'), 10) @@ -576,6 +615,51 @@ describe('FileDocStore', () => { await expect(store.getStreamState(NAME)).rejects.toThrow('replaced') }) + it.each([true, false])( + 'validates a modern snapshot following a legacy seed (same identity: %s)', + async (sameIdentity) => { + const store = await newStore() + const seed = new Y.Doc() + seed.getMap('config').set('initialContentLoaded', true) + seed.getMap('config').set('docId', 'legacy-document') + seed.getText('body').insert(0, 'legacy') + seedLegacyStream(Y.encodeStateAsUpdate(seed)) + const backing = state.backing! + backing.kv.set( + `filedoc:generation:${NAME}`, + sameIdentity ? 'legacy-document' : 'different-document' + ) + backing.streams.get(`filedoc:stream:${NAME}`)!.push({ + id: `${++backing.seq}-0`, + message: { + u: Buffer.from(Y.encodeStateAsUpdate(seed)).toString('base64'), + s: '1', + g: sameIdentity ? 'legacy-document' : 'different-document', + }, + }) + const attached = new Y.Doc() + if (sameIdentity) { + await store.attachRoom(NAME, attached) + const before = Y.encodeStateVector(seed) + seed.getText('body').insert(6, ' peer') + await store.publishClientUpdateAndWait( + NAME, + 'peer-edit', + Y.encodeStateAsUpdate(seed, before), + 'legacy-document' + ) + await store.catchUp(NAME) + expect(attached.getText('body').toString()).toBe('legacy peer') + store.detachRoom(NAME) + } else { + await expect(store.attachRoom(NAME, attached)).rejects.toThrow('replaced') + expect(storeInternals(store).rooms.has(NAME)).toBe(false) + } + seed.destroy() + attached.destroy() + } + ) + it('replays stream history in bounded pages', async () => { const streamKey = `filedoc:stream:${NAME}` const noop = Buffer.from(updateFor('')).toString('base64') @@ -1083,26 +1167,43 @@ describe('FileDocStore', () => { doc.destroy() }) - it('bounds single-replica invalidation markers to the lifetime of active rooms', async () => { + it('expires idle single-replica invalidation watermarks', async () => { + vi.useFakeTimers() const store = new FileDocStore(undefined) - for (let index = 0; index < 100; index++) { - await store.invalidateDocument(`closed-${index}`, 10) + try { + for (let index = 0; index < 100; index++) { + await store.invalidateDocument(`closed-${index}`, 10) + } + expect(storeInternals(store).localInvalidations.size).toBe(100) + await vi.advanceTimersByTimeAsync(660_000) + expect(storeInternals(store).localInvalidations.size).toBe(0) + } finally { + await store.shutdown() + vi.useRealTimers() } - expect(storeInternals(store).localInvalidations.size).toBe(0) + }) + + it('deduplicates single-replica invalidations across same-version seeds and room reopen', async () => { + const store = new FileDocStore(undefined) const staleDoc = new Y.Doc() await store.attachRoom(NAME, staleDoc) await store.invalidateDocument(NAME, 20) await expect(store.seedIfEmpty(NAME, updateFor('stale fetched seed'), 10)).resolves.toBe(false) await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(false) expect(storeInternals(store).localInvalidations.size).toBe(1) + await expect(store.seedIfEmpty(NAME, updateFor('same-version seed'), 20)).resolves.toBe(true) + await expect(store.invalidateDocument(NAME, 20)).resolves.toBe(false) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(true) store.detachRoom(NAME) - expect(storeInternals(store).localInvalidations.size).toBe(0) + expect(storeInternals(store).localInvalidations.size).toBe(1) const freshDoc = new Y.Doc() await store.attachRoom(NAME, freshDoc) - await expect(store.seedIfEmpty(NAME, updateFor('fresh authoritative seed'), 30)).resolves.toBe( + await expect(store.seedIfEmpty(NAME, updateFor('fresh authoritative seed'), 20)).resolves.toBe( true ) + await expect(store.invalidateDocument(NAME, 20)).resolves.toBe(false) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(true) await store.invalidateDocument(NAME, 40) await store.shutdown() expect(storeInternals(store).localInvalidations.size).toBe(0) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index bd872b100b3..23b430d127e 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -63,11 +63,11 @@ const RELEASE_LOCK_SCRIPT = * Returns 1 if THIS call wrote the seed, 0 if the stream already had content. */ const SEED_IF_EMPTY_SCRIPT = - "local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[6]) then return 0 end; if redis.call('xlen', KEYS[1]) == 0 then redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[4]); if ARGV[6] ~= '0' then redis.call('set', KEYS[3], ARGV[6], 'EX', ARGV[4]) end; redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[5], ARGV[3]); redis.call('expire', KEYS[1], ARGV[4]); return 1 else return 0 end" + "local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[6]) then return 0 end; if redis.call('xlen', KEYS[1]) == 0 then redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[4]); if ARGV[6] ~= '0' then redis.call('set', KEYS[3], ARGV[6], 'EX', ARGV[4]) end; redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[5], ARGV[3]); redis.call('expire', KEYS[1], ARGV[4]); redis.call('expire', KEYS[4], ARGV[4]); return 1 else return 0 end" /** Orders a durable replacement with seeds and merges, and fences publishers in the same transaction. */ const INVALIDATE_DOCUMENT_SCRIPT = - "local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[1]) then return 0 end; if version == ARGV[1] and redis.call('get', KEYS[2]) == ARGV[3] then return 0 end; redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[2]); redis.call('set', KEYS[3], ARGV[1], 'EX', ARGV[2]); redis.call('del', KEYS[1], KEYS[4], KEYS[5]); return 1" + "local invalidated = redis.call('get', KEYS[6]); if invalidated and tonumber(invalidated) >= tonumber(ARGV[1]) then return 0 end; local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[1]) then return 0 end; if version == ARGV[1] and redis.call('get', KEYS[2]) == ARGV[3] then return 0 end; redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[2]); redis.call('set', KEYS[3], ARGV[1], 'EX', ARGV[2]); redis.call('set', KEYS[6], ARGV[1], 'EX', ARGV[2]); redis.call('del', KEYS[1], KEYS[4], KEYS[5]); return 1" /** Upgrades an existing pre-negotiation stream without ever resurrecting a missing stream. */ const ADOPT_GENERATION_SCRIPT = @@ -75,7 +75,11 @@ const ADOPT_GENERATION_SCRIPT = /** Atomically fence XADD so stale rooms cannot recreate a replaced or expired stream. Returns false when fenced. */ const APPEND_UPDATE_SCRIPT = - "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; if ARGV[3] ~= '' then return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1') else return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]) end" + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; for _, key in ipairs(KEYS) do redis.call('expire', key, ARGV[5]) end; if ARGV[3] ~= '' then return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1') else return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]) end" + +/** Renew stream metadata atomically, so an invalidation watermark cannot expire ahead of its stream. */ +const REFRESH_DOCUMENT_TTLS_SCRIPT = + "for _, key in ipairs(KEYS) do redis.call('expire', key, ARGV[1]) end; return 1" /** A compacted snapshot replaces the seed entry, so it must carry that seed's generation forward. */ const APPEND_SNAPSHOT_SCRIPT = @@ -86,7 +90,7 @@ const APPEND_SNAPSHOT_SCRIPT = * lost, so a retry with the same id must not inflate the stream or its compaction counters. */ const APPEND_CLIENT_UPDATE_SCRIPT = - "local generation = redis.call('get', KEYS[3]) or ''; if generation ~= ARGV[6] or redis.call('exists', KEYS[1]) == 0 then return -1 end; if redis.call('zscore', KEYS[2], ARGV[1]) then redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 0 end; local id = redis.call('xadd', KEYS[1], '*', ARGV[2], ARGV[3]); local score = string.match(id, '^(%d+)'); redis.call('zadd', KEYS[2], score, ARGV[1]); local excess = redis.call('zcard', KEYS[2]) - tonumber(ARGV[4]); if excess > 0 then redis.call('zpopmin', KEYS[2], excess) end; if redis.call('ttl', KEYS[2]) < 0 then redis.call('expire', KEYS[2], ARGV[5]) end; redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 1" + "local generation = redis.call('get', KEYS[3]) or ''; if generation ~= ARGV[6] or redis.call('exists', KEYS[1]) == 0 then return -1 end; redis.call('expire', KEYS[4], ARGV[5]); if redis.call('zscore', KEYS[2], ARGV[1]) then redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 0 end; local id = redis.call('xadd', KEYS[1], '*', ARGV[2], ARGV[3]); local score = string.match(id, '^(%d+)'); redis.call('zadd', KEYS[2], score, ARGV[1]); local excess = redis.call('zcard', KEYS[2]) - tonumber(ARGV[4]); if excess > 0 then redis.call('zpopmin', KEYS[2], excess) end; if redis.call('ttl', KEYS[2]) < 0 then redis.call('expire', KEYS[2], ARGV[5]) end; redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 1" /** * Monotonic set of the synced-version token: overwrite ONLY when the new value is greater than the @@ -130,6 +134,8 @@ export const REDIS_AGENT_ORIGIN = Symbol('file-doc-redis-agent') const STREAM_PREFIX = 'filedoc:stream:' const CLIENT_UPDATE_PREFIX = 'filedoc:updates:' const GENERATION_PREFIX = 'filedoc:generation:' +/** Retries must remain idempotent after a seed replaces the generation tombstone. */ +const INVALIDATION_VERSION_PREFIX = 'filedoc:invalidatedver:' /** Cluster-wide "durable version the live doc is synced to" (the persist If-Match token). */ const SYNC_VERSION_PREFIX = 'filedoc:syncver:' const SEED_LOCK_PREFIX = 'filedoc:seedlock:' @@ -210,6 +216,12 @@ const CLIENT_UPDATE_DEDUPE_CAPACITY = 16_384 const streamKey = (name: string) => `${STREAM_PREFIX}${name}` const generationKey = (name: string) => `${GENERATION_PREFIX}${name}` +const documentKeys = (name: string) => [ + streamKey(name), + generationKey(name), + `${SYNC_VERSION_PREFIX}${name}`, + `${INVALIDATION_VERSION_PREFIX}${name}`, +] export class FileDocInvalidatedError extends Error { constructor() { @@ -314,7 +326,7 @@ export class FileDocStore { /** Dedicated connection for blocking XREAD (a blocking command monopolizes its connection). */ private read: RedisClientType | null = null private readonly rooms = new Map() - private readonly localInvalidations = new Map() + private readonly localInvalidations = new Map() private running = false private heartbeat: ReturnType | null = null @@ -417,6 +429,7 @@ export class FileDocStore { }) if (this.rooms.get(name) !== room) return for (const entry of entries) this.applyEntry(name, room, entry.id, entry.message) + if (room.generationInvalidated) throw new FileDocInvalidatedError() const docId = room.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) if (room.generation === null && typeof docId === 'string') { const adopted = await this.write.eval(ADOPT_GENERATION_SCRIPT, { @@ -426,7 +439,7 @@ export class FileDocStore { if (adopted !== docId) throw new FileDocInvalidatedError() room.generation = docId } - await this.write.expire(streamKey(name), STREAM_TTL_SEC) + await this.refreshDocumentTtls(name) } catch (error) { logger.warn(`FileDocStore catch-up failed for ${name}`, { error: getErrorMessage(error), @@ -438,14 +451,13 @@ export class FileDocStore { /** Deregister a room the relay is destroying, so the tailer stops touching its (about-to-be-destroyed) doc. */ detachRoom(name: string): void { this.rooms.delete(name) - this.localInvalidations.delete(name) } /** * Append a locally-applied update to the shared stream so every task converges, AWAITING the write * and retrying a transient failure ({@link PUBLISH_MAX_RETRIES}) so a Redis blip can't silently drop - * an edit from the shared log. Only the `xAdd` is retried; the TTL refresh + compaction check are - * post-write best-effort and never re-trigger the append. Throws if the append ultimately fails. + * an edit from the shared log. The append and metadata TTL renewal are atomic; post-write + * compaction never re-triggers the append. Throws if the append ultimately fails. */ private async appendUpdate( name: string, @@ -471,8 +483,8 @@ export class FileDocStore { for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { const id = await this.write.eval(APPEND_UPDATE_SCRIPT, { - keys: [streamKey(name), generationKey(name)], - arguments: [UPDATE_FIELD, encoded, marker, expectedGeneration], + keys: documentKeys(name), + arguments: [UPDATE_FIELD, encoded, marker, expectedGeneration, String(STREAM_TTL_SEC)], }) if (id === null || id === false) throw new FileDocInvalidatedError() break @@ -489,8 +501,6 @@ export class FileDocStore { await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) } } - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) - await this.write.expire(generationKey(name), STREAM_TTL_SEC).catch(() => {}) const room = this.rooms.get(name) if (room) { room.publishes += 1 @@ -557,7 +567,12 @@ export class FileDocStore { for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { const appended = await this.write.eval(APPEND_CLIENT_UPDATE_SCRIPT, { - keys: [streamKey(name), `${CLIENT_UPDATE_PREFIX}${name}`, generationKey(name)], + keys: [ + streamKey(name), + `${CLIENT_UPDATE_PREFIX}${name}`, + generationKey(name), + `${INVALIDATION_VERSION_PREFIX}${name}`, + ], arguments: [ dedupeMember, UPDATE_FIELD, @@ -605,8 +620,11 @@ export class FileDocStore { */ async seedIfEmpty(name: string, update: Uint8Array, version = 0): Promise { if (!this.enabled) { - if ((this.localInvalidations.get(name) ?? 0) > version) return false - this.localInvalidations.delete(name) + const invalidation = this.localInvalidations.get(name) + if (invalidation && invalidation.expiresAt > Date.now() && invalidation.version > version) + return false + const room = this.rooms.get(name) + if (room) room.generationInvalidated = false return true } if (!this.write) throw new Error('FileDocStore is not initialized') @@ -616,7 +634,7 @@ export class FileDocStore { for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { const wrote = await this.write.eval(SEED_IF_EMPTY_SCRIPT, { - keys: [streamKey(name), generationKey(name), `${SYNC_VERSION_PREFIX}${name}`], + keys: documentKeys(name), arguments: [ UPDATE_FIELD, encoded, @@ -626,7 +644,7 @@ export class FileDocStore { String(version), ], }) - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + await this.refreshDocumentTtls(name).catch(() => {}) const room = this.rooms.get(name) if (wrote === 1 && room) room.generation = generation return wrote === 1 @@ -645,13 +663,21 @@ export class FileDocStore { /** * Fences an unsupported durable replacement before deleting its stream. The next authoritative - * seed replaces the tombstone; abandoned tombstones expire with the stream TTL. + * seed replaces the tombstone. A separate version watermark deduplicates retries across reseeds; + * both expire with the stream TTL once the document is idle. */ async invalidateDocument(name: string, version: number): Promise { if (!this.enabled) { - if (!this.rooms.has(name)) return true - if ((this.localInvalidations.get(name) ?? 0) >= version) return false - this.localInvalidations.set(name, version) + const now = Date.now() + const previous = this.localInvalidations.get(name) + if (previous && previous.expiresAt > now && previous.version >= version) return false + this.localInvalidations.set(name, { version, expiresAt: now + STREAM_TTL_SEC * 1_000 }) + const room = this.rooms.get(name) + if (room) room.generationInvalidated = true + if (!this.heartbeat) { + this.heartbeat = setInterval(() => void this.refreshTtls(), HEARTBEAT_MS) + this.heartbeat.unref() + } return true } if (!this.write) throw new Error('FileDocStore is not initialized') @@ -663,6 +689,7 @@ export class FileDocStore { `${SYNC_VERSION_PREFIX}${name}`, `${CLIENT_UPDATE_PREFIX}${name}`, `${AGENT_STREAM_PREFIX}${name}`, + `${INVALIDATION_VERSION_PREFIX}${name}`, ], arguments: [String(version), String(STREAM_TTL_SEC), INVALIDATED_GENERATION], })) === 1 @@ -676,7 +703,7 @@ export class FileDocStore { } async isDocumentGenerationCurrent(name: string, generation?: string): Promise { - if (!this.enabled) return !this.localInvalidations.has(name) + if (!this.enabled) return !this.rooms.get(name)?.generationInvalidated if (!this.write) throw new Error('FileDocStore is not initialized') const current = await this.write.get(generationKey(name)) return current === null ? !generation : current === generation @@ -964,7 +991,9 @@ export class FileDocStore { if ( room.generationInvalidated || (room.generation !== null && room.generation !== generation) || - (room.generation === null && room.seededObserved) + (room.generation === null && + room.seededObserved && + room.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) !== generation) ) { room.generationInvalidated = true return @@ -1146,14 +1175,28 @@ export class FileDocStore { } } + private async refreshDocumentTtls(name: string): Promise { + await this.write?.eval(REFRESH_DOCUMENT_TTLS_SCRIPT, { + keys: documentKeys(name), + arguments: [String(STREAM_TTL_SEC)], + }) + } + private async refreshTtls(): Promise { - if (!this.write) return + if (!this.write) { + const now = Date.now() + for (const [name, invalidation] of this.localInvalidations) { + if (this.rooms.has(name)) invalidation.expiresAt = now + STREAM_TTL_SEC * 1_000 + else if (invalidation.expiresAt <= now) this.localInvalidations.delete(name) + } + if (this.localInvalidations.size === 0 && this.heartbeat) { + clearInterval(this.heartbeat) + this.heartbeat = null + } + return + } for (const name of this.rooms.keys()) { - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) - // Keep the synced-version key alive as long as its stream, so an open-but-idle doc's persist - // If-Match token can't expire out from under it (which would force a needless reconcile). - await this.write.expire(`${SYNC_VERSION_PREFIX}${name}`, STREAM_TTL_SEC).catch(() => {}) - await this.write.expire(generationKey(name), STREAM_TTL_SEC).catch(() => {}) + await this.refreshDocumentTtls(name).catch(() => {}) } } } diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index 9e0223e0713..42dcaa1cf8d 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -199,13 +199,14 @@ describe('setupWorkspaceFileDocHandlers', () => { mockFetchFileDocPersist.mockResolvedValue({ status: 'persisted', version: 1 }) }) - afterEach(() => { + afterEach(async () => { // The room store is module-global; drop every room the test's sockets opened. const { io } = createIo() // Simulate a full disconnect between tests (`endOfLife`) so the module-global join-generation // map is cleared and never bleeds a counter into the next test. for (const id of createdSocketIds) cleanupFileDocForSocket(id, io, true) createdSocketIds.clear() + await getFileDocStore().shutdown() }) it('rejects join when the socket is not authenticated', async () => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index a85c85c7ddc..e7d74fed83b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -566,14 +566,21 @@ describe('FileDocProvider', () => { pendingUpdate: firstPendingUpdate!, status: 'saved', }) - await vi.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(save).toHaveBeenCalledTimes(2) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, save.mock.calls[1][2]) + expect(recovered.getText('default').toString()).toBe('first second') + recovered.destroy() + await vi.advanceTimersByTimeAsync(1_000) + expect(save).toHaveBeenCalledTimes(2) const firstUpdate = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) const firstPayload = firstUpdate?.[1] as { updateId: string } const acknowledge = firstUpdate?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void acknowledge(null, { status: 'accepted', updateId: firstPayload.updateId }) await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) - expect(save).toHaveBeenCalledTimes(2) + expect(save).toHaveBeenCalledTimes(3) expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toHaveLength(2) provider.destroy() } finally { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index b078c237647..5e6b5d3e91e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -730,6 +730,7 @@ export class FileDocProvider extends ObservableV2 { if (!docId) return this.updateFlushInProgress = true + let hasUnjournaledUpdates = false try { const update = Y.mergeUpdates(this.pendingUpdateBatch) this.pendingUpdateBatch = [] @@ -737,6 +738,7 @@ export class FileDocProvider extends ObservableV2 { ? Y.mergeUpdates([this.inFlightUpdate.update, update]) : update const saved = await this.journal?.save(docId, journalUpdate, Y.encodeStateAsUpdate(this.doc)) + hasUnjournaledUpdates = this.pendingUpdateBatch.length > 0 if (this.disposed || this.fatal) { this.queuePendingUpdate(update) return @@ -758,7 +760,7 @@ export class FileDocProvider extends ObservableV2 { } finally { this.updateFlushInProgress = false this.updateBeforeUnloadProtection() - if (this.pendingUpdateBatch.length > 0 && !this.inFlightUpdate) { + if (this.pendingUpdateBatch.length > 0 && (hasUnjournaledUpdates || !this.inFlightUpdate)) { this.scheduleUpdateFlush(0) } } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx index 0eed9d37670..4089e298d17 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx @@ -62,7 +62,10 @@ function pointerEvent( return event } -function renderImage(updateAttributes: ReturnType): HTMLButtonElement { +function renderImage( + updateAttributes: ReturnType, + dimensions: { width?: string | null; height?: string | null } = {} +): HTMLButtonElement { const props = { node: { attrs: { @@ -71,6 +74,7 @@ function renderImage(updateAttributes: ReturnType): HTMLButtonElem title: null, width: null, height: '100', + ...dimensions, href: null, }, }, @@ -123,7 +127,7 @@ describe('ResizableImageView', () => { ]) }) - it('keeps the width automatic when only an explicit height is stored', () => { + it('renders a height-only image proportionally without fixing its responsive height', () => { renderImage(vi.fn()) const image = host.querySelector('img') if (!image) throw new Error('Missing image') @@ -133,8 +137,28 @@ describe('ResizableImageView', () => { }) act(() => image.dispatchEvent(new Event('load'))) + expect(image.style.height).toBe('') + expect(image.style.width).toBe('calc(200px)') + expect(image.style.aspectRatio).toBe('400 / 200') + }) + + it.each([ + { width: '600', height: '400' }, + { width: '600px', height: '400px' }, + { width: '600', height: '400px' }, + ])('uses the authored ratio for responsive pixel dimensions: %j', (dimensions) => { + renderImage(vi.fn(), dimensions) + const image = host.querySelector('img')! + expect(image.style.width).toBe('600px') + expect(image.style.height).toBe('') + expect(image.style.aspectRatio).toBe('600 / 400') + }) + + it('preserves relative dimensions instead of assuming they are pixel ratios', () => { + renderImage(vi.fn(), { width: '50%', height: '100px' }) + const image = host.querySelector('img')! + expect(image.style.width).toBe('50%') expect(image.style.height).toBe('100px') - expect(image.style.width).toBe('') }) it('commits one proportional width change and clears a stale explicit height', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx index 10a4bca99ff..9659486935b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx @@ -13,6 +13,7 @@ const MIN_WIDTH = 64 /** A bare pixel count (`"640"`) that needs a `px` suffix, vs. an already-unit'd size (`"50%"`). */ const BARE_PIXEL_SIZE = /^\d+$/ +const PIXEL_SIZE = /^\d+(?:\.\d+)?px$/ /** * Drag-to-resize image node view (handle at the bottom-right, revealed on selection). Dragging @@ -134,21 +135,37 @@ export function ResizableImageView({ // stored value is stale (e.g. left over after the file's content was replaced) — so it wins once // available; stored metadata only reserves the box pre-load. Equal in the common case, so no shift. const intrinsicDimensions = measuredDimensions ?? storedDimensions + const authoredDimensions = + committedWidth && + committedHeight && + PIXEL_SIZE.test(committedWidth) && + PIXEL_SIZE.test(committedHeight) && + Number.parseFloat(committedWidth) > 0 && + Number.parseFloat(committedHeight) > 0 + ? { width: Number.parseFloat(committedWidth), height: Number.parseFloat(committedHeight) } + : null + const displayDimensions = + dragWidth === null ? (authoredDimensions ?? intrinsicDimensions) : intrinsicDimensions const displayWidth = dragWidth !== null ? `${dragWidth}px` : (committedWidth ?? - (!committedHeight && intrinsicDimensions ? `${intrinsicDimensions.width}px` : undefined)) + (intrinsicDimensions + ? committedHeight + ? `calc(${committedHeight} * ${intrinsicDimensions.width / intrinsicDimensions.height})` + : `${intrinsicDimensions.width}px` + : undefined)) // width + aspect-ratio (with `max-w-full`/`h-auto` from the class list) reserves a responsive box the // image can't reflow into, per the CLS-avoidance pattern for known-ratio responsive images. React drops // the undefined keys, so an unmeasured image simply gets no reservation (its prior behavior). const imageStyle: CSSProperties = { width: displayWidth, - height: dragWidth === null ? committedHeight : undefined, - aspectRatio: - intrinsicDimensions && !committedHeight - ? `${intrinsicDimensions.width} / ${intrinsicDimensions.height}` - : undefined, + height: + dragWidth === null && committedWidth && !authoredDimensions ? committedHeight : undefined, + maxHeight: dragWidth === null && !committedWidth ? committedHeight : undefined, + aspectRatio: displayDimensions + ? `${displayDimensions.width} / ${displayDimensions.height}` + : undefined, } // Sanitize the linked-image target before rendering the anchor — a parsed markdown href is diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts index 0ba8ef68884..0250aa84806 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts @@ -6,6 +6,21 @@ import { normalizeMarkdownContent } from '@/app/workspace/[workspaceId]/files/co import { isRoundTripSafe } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety' describe('isRoundTripSafe', () => { + it.each([ + '
\n\n
', + '', + '', + '---\nexample: \'\'\n---\n# Heading', + ])('allows image attributes preserved verbatim in raw content: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(true) + expect(normalizeMarkdownContent(source).trim()).toBe(source) + }) + + it('does not let a preserved raw tag hide an identical image tag that loses attributes', () => { + const tag = '' + expect(isRoundTripSafe(`
\n${tag}\n
\n\n${tag}`)).toBe(false) + }) + it('passes ordinary markdown and lossless normalizations', () => { expect(isRoundTripSafe('# Title\n\nA **bold** word and a [link](https://sim.ai).')).toBe(true) expect(isRoundTripSafe('- one\n- two\n\n```js\nconst x = 1\n```')).toBe(true) @@ -161,7 +176,6 @@ describe('isRoundTripSafe', () => { }) it.each([ - '', '[](/link)', '[](/link)', "[](/link)", diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts index ca8aad1e177..bc69ce942b0 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts @@ -45,10 +45,11 @@ const fidelityLexer = new Marked({ gfm: true }) const SUPPORTED_IMAGE_ATTRIBUTES = new Set(['src', 'alt', 'title', 'width', 'height']) /** - * Unsupported or duplicate HTML image attributes require source mode; a stable serialization - * is not lossless if parsing already discarded those attributes. + * Count tags that image parsing would lose attributes from, including duplicates. Raw snippets + * may preserve these verbatim, so compare the counts before and after the first serialization. */ -function hasUnsupportedHtmlImageAttribute(content: string): boolean { +function unsupportedHtmlImages(content: string): Map { + const images = new Map() const tokenizer = new Tokenizer() new Lexer({ gfm: true, tokenizer }) const imagePattern = /])/gi @@ -61,11 +62,14 @@ function hasUnsupportedHtmlImageAttribute(content: string): boolean { const pattern = /(?:^|\s)([^\s=/>]+)(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>`]+))?/g for (const attribute of attributes.matchAll(pattern)) { const name = attribute[1].toLowerCase() - if (!SUPPORTED_IMAGE_ATTRIBUTES.has(name) || seen.has(name)) return true + if (!SUPPORTED_IMAGE_ATTRIBUTES.has(name) || seen.has(name)) { + images.set(tag.raw, (images.get(tag.raw) ?? 0) + 1) + break + } seen.add(name) } } - return false + return images } function imageSources(token: Token): string[] { @@ -168,11 +172,14 @@ export function isRoundTripSafe(content: string): boolean { const stripped = stripCode(content) if (STABLE_LOSS_PATTERNS.some((pattern) => pattern.test(stripped))) return false if (hasOrphanReferenceDefinition(stripped)) return false - if (hasUnsupportedHtmlImageAttribute(stripped)) return false try { const source = inspectMarkdownFidelity(content) if (source.hasTaskReference || source.hasTableHtmlImage) return false const once = serializeMarkdownDocument(content) + const preservedImages = unsupportedHtmlImages(stripCode(once)) + for (const [tag, count] of unsupportedHtmlImages(stripped)) { + if ((preservedImages.get(tag) ?? 0) < count) return false + } const serialized = inspectMarkdownFidelity(once) for (const [target, count] of source.targets) { if ((serialized.targets.get(target) ?? 0) < count) return false From da63a473e5df5b2012ab3f2dbf7826fc912b0643 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 5 Sep 2026 13:05:25 -0700 Subject: [PATCH 6/9] fix(files): preserve native non-pixel image heights --- .../image-resize.test.tsx | 21 +++++++++++++++++++ .../rich-markdown-editor/image.tsx | 10 ++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx index 4089e298d17..f046c168979 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx @@ -161,6 +161,27 @@ describe('ResizableImageView', () => { expect(image.style.height).toBe('100px') }) + it.each(['50%', 'auto', '10em', 'calc(50% - 10px)', 'min-content', 'inherit'])( + 'preserves the native height-only CSS value %s before and after loading', + (height) => { + renderImage(vi.fn(), { height }) + const image = host.querySelector('img')! + expect(image.style.width).toBe('') + expect(image.style.height).toBe(height) + expect(image.style.maxHeight).toBe('') + + Object.defineProperties(image, { + naturalWidth: { configurable: true, value: 400 }, + naturalHeight: { configurable: true, value: 200 }, + }) + act(() => image.dispatchEvent(new Event('load'))) + + expect(image.style.width).toBe('') + expect(image.style.height).toBe(height) + expect(image.style.maxHeight).toBe('') + } + ) + it('commits one proportional width change and clears a stale explicit height', () => { const updateAttributes = vi.fn() const handle = renderImage(updateAttributes) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx index 9659486935b..6e5e7d6c32d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx @@ -135,6 +135,7 @@ export function ResizableImageView({ // stored value is stale (e.g. left over after the file's content was replaced) — so it wins once // available; stored metadata only reserves the box pre-load. Equal in the common case, so no shift. const intrinsicDimensions = measuredDimensions ?? storedDimensions + const hasPixelHeight = committedHeight !== undefined && PIXEL_SIZE.test(committedHeight) const authoredDimensions = committedWidth && committedHeight && @@ -150,7 +151,7 @@ export function ResizableImageView({ dragWidth !== null ? `${dragWidth}px` : (committedWidth ?? - (intrinsicDimensions + (intrinsicDimensions && (!committedHeight || hasPixelHeight) ? committedHeight ? `calc(${committedHeight} * ${intrinsicDimensions.width / intrinsicDimensions.height})` : `${intrinsicDimensions.width}px` @@ -161,8 +162,11 @@ export function ResizableImageView({ const imageStyle: CSSProperties = { width: displayWidth, height: - dragWidth === null && committedWidth && !authoredDimensions ? committedHeight : undefined, - maxHeight: dragWidth === null && !committedWidth ? committedHeight : undefined, + dragWidth === null && !authoredDimensions && (committedWidth || !hasPixelHeight) + ? committedHeight + : undefined, + maxHeight: + dragWidth === null && !committedWidth && hasPixelHeight ? committedHeight : undefined, aspectRatio: displayDimensions ? `${displayDimensions.width} / ${displayDimensions.height}` : undefined, From dbe426d7d47f0c48bfa926b2f56030b2669b0b87 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 5 Sep 2026 13:15:38 -0700 Subject: [PATCH 7/9] chore(files): remove redundant find-bar comment --- .../app/workspace/[workspaceId]/components/find-bar/find-bar.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.tsx b/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.tsx index 7bd4dc8b52f..282df8612cb 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.tsx @@ -144,7 +144,6 @@ export const FindBar = memo(function FindBar({ className={replace ? 'min-w-0 flex-1' : 'w-[200px]'} onChange={(e) => onQueryChange(e.target.value)} onKeyDown={handleKeyDown} - /** Whitespace may not match, but the user must still be able to clear it. */ endAdornment={ query.length > 0 ? (