fix(db): open a database whose keyword index was built by an older version

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to MeMesh are documented here.
 
 ### Fixed
 
+- A small database last opened by an older release (before 4.2.11, or with Thai, Lao, Khmer, half-width katakana or rare CJK characters indexed by an older build) opens again after upgrading. Before, every open failed with "database disk image is malformed" although the database was not damaged, because an upgrade step removed keyword-index rows using the new text splitting before the index was rebuilt (#522).
 - Memories about a file are shown before an edit in every session again. Once any session (or another agent on the same machine) had been shown them, every other session editing that file got nothing, and the hook record said "nothing to recall". Each session now keeps its own list (a subagent keeps one of its own), a repeat is recorded as throttled, and a list that cannot be written or cleaned up is recorded as an error instead of failing silently (#521).
 - Importing with `overwrite` (MCP `import`, `POST /v1/import`, `memesh import --merge overwrite`) now keeps the memory's previous observations, tags and title in its `replaced_history`, the same way `remember` with `replace: true` does. Before, the old content was erased with no way back. Importing a file that names one memory more than once is refused before anything is written (#530).
 - A message sent through the MCP `message` tool keeps every `null` in its JSON payload. Before, the MCP boundary removed each null-valued key at any depth of the payload while `send` still reported success, so the recipient got different data than the CLI or HTTP would have stored (#517). A null-valued top-level tool parameter still means "left blank", except `payload`, whose null is a value (#553).

dist/mcp/THIRD_PARTY_NOTICES.txt

@@ -1,7 +1,7 @@
 Third-party software bundled in dist/mcp/server.js
 
-Bundle SHA-256: 2052e8e6178ebae0baa24f1a8f9e5ee364cc827f17de5edf9aa23d8cefb29346
-Source map SHA-256: b8ea27c8208e263bc933526350f1c3c1b833cbacf13d44931421eec9f9cca407
+Bundle SHA-256: 48878b89f47877c89f92b6234f66b9d91ff93176696922769c3a5f0a645ffe1d
+Source map SHA-256: 8f8532077e6e849de85c2fe4bf2c037c970780912e63a9912f2ee40e057da491
 
 Generated by scripts/build-mcp-bundle.mjs from the esbuild input closure.
 

dist/mcp/server.js

@@ -27220,6 +27220,7 @@ function migrateToCurrentSchema(db2) {
   migrateEntitiesSchema(db2);
   runAutoDecay(db2);
   backfillSignalScores(db2);
+  ensureFtsSegmentation(db2);
   backfillTitles(db2);
   backfillAcceptedProposalTrust(db2);
   dedupeObservations(db2);
@@ -27229,7 +27230,6 @@ function migrateToCurrentSchema(db2) {
   splitFusedLessons(db2, { deriveTitle: deriveHeuristicTitle });
   repairFusedLessonShellHistory(db2);
   ensureDreamProposalsTable(db2);
-  ensureFtsSegmentation(db2);
   dropArchivedIndexRows(db2);
 }
 function ensureDreamProposalsTable(db2) {

dist/transports/cli/cli.js

@@ -6033,6 +6033,7 @@ function migrateToCurrentSchema(db2) {
   migrateEntitiesSchema(db2);
   runAutoDecay(db2);
   backfillSignalScores(db2);
+  ensureFtsSegmentation(db2);
   backfillTitles(db2);
   backfillAcceptedProposalTrust(db2);
   dedupeObservations(db2);
@@ -6042,7 +6043,6 @@ function migrateToCurrentSchema(db2) {
   splitFusedLessons(db2, { deriveTitle: deriveHeuristicTitle });
   repairFusedLessonShellHistory(db2);
   ensureDreamProposalsTable(db2);
-  ensureFtsSegmentation(db2);
   dropArchivedIndexRows(db2);
 }
 function reindexFts() {

src/db.ts

@@ -215,6 +215,18 @@ function migrateToCurrentSchema(db: MemeshDatabase): void {
   // marker key.
   backfillSignalScores(db);
 
+  // Rebuild entities_fts once when the way text is segmented changes.
+  // Databases written before CJK segmentation hold whole-run tokens that no
+  // segmented query can match, so without this the change would take Chinese
+  // recall from bad to zero while English kept working — a silent regression.
+  //
+  // It runs BEFORE every pass that removes FTS rows one by one (the title
+  // backfill below is the first): a contentless delete must repeat the exact
+  // tokens that were indexed, and on an old index today's segmentation
+  // produces different ones. On a small database that mismatch makes the
+  // delete fail with "database disk image is malformed" on every open (#522).
+  ensureFtsSegmentation(db);
+
   // UX-1: give pre-title rows a human-readable heuristic title. Same
   // marker + fill-only discipline as backfillSignalScores above.
   backfillTitles(db);
@@ -253,12 +265,6 @@ function migrateToCurrentSchema(db: MemeshDatabase): void {
   // Agent work packages stage proposals here for explicit human review.
   ensureDreamProposalsTable(db);
 
-  // Rebuild entities_fts once when the way text is segmented changes.
-  // Databases written before CJK segmentation hold whole-run tokens that no
-  // segmented query can match, so without this the change would take Chinese
-  // recall from bad to zero while English kept working — a silent regression.
-  ensureFtsSegmentation(db);
-
   // One-shot repair for FTS rows written before archived entities were removed
   // from the keyword index. It rebuilds from active entities only and has no
   // dependency on an optional secondary index.

tests/db-old-index-opens.test.ts

@@ -0,0 +1,58 @@
+/**
+ * #522: a small database last opened by an older build holds keyword-index
+ * rows split the old way (whole runs of CJK / Thai text). Opening it must
+ * rebuild that index before any step removes rows from it one by one with
+ * today's splitting, or the removal fails and every open reports
+ * "database disk image is malformed" for a database that is not corrupt.
+ */
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { closeDatabase, openDatabase } from '../src/db.js';
+import { KnowledgeGraph } from '../src/knowledge-graph.js';
+import { MemeshDatabase } from '../src/storage/sqlite.js';
+import { FTS_SQL, SCHEMA_SQL, migrateEntitiesSchema } from '../src/storage/schema.js';
+
+function writeLegacyDatabase(dbPath: string, name: string, observation: string, segmentationVersion?: string): void {
+  const db = new MemeshDatabase(dbPath);
+  db.pragma('journal_mode = WAL');
+  db.exec(SCHEMA_SQL);
+  db.exec(FTS_SQL);
+  migrateEntitiesSchema(db);
+  db.prepare("INSERT INTO entities (name, type) VALUES (?, 'decision')").run(name);
+  db.prepare('INSERT INTO observations (entity_id, content) VALUES (1, ?)').run(observation);
+  // Indexed the way an older build did: the whole unspaced run as one token.
+  db.prepare('INSERT INTO entities_fts (rowid, name, observations) VALUES (1, ?, ?)').run(name, observation);
+  if (segmentationVersion) {
+    db.prepare("INSERT INTO memesh_metadata (key, value) VALUES ('fts_segmentation_version', ?)").run(segmentationVersion);
+  }
+  db.close();
+}
+
+describe('Feature: #522 an old small database opens after upgrading', () => {
+  let dir: string;
+  let dbPath: string;
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'memesh-old-index-'));
+    dbPath = path.join(dir, 'knowledge-graph.db');
+  });
+
+  afterEach(() => {
+    try { closeDatabase(); } catch { /* not open */ }
+    fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  });
+
+  it('a pre-segmentation database with Chinese text opens and finds it', () => {
+    writeLegacyDatabase(dbPath, '資料庫遷移', '資料庫遷移前一定要先備份');
+    const db = openDatabase(dbPath);
+    expect(new KnowledgeGraph(db).search('資料庫').map((e) => e.name)).toContain('資料庫遷移');
+  });
+
+  it('a database indexed before Thai was segmented opens and finds it', () => {
+    writeLegacyDatabase(dbPath, 'ฐานข้อมูล', 'สำรองข้อมูลก่อนย้ายฐานข้อมูลเสมอ', '2');
+    const db = openDatabase(dbPath);
+    expect(new KnowledgeGraph(db).search('ฐานข้อมูล').map((e) => e.name)).toContain('ฐานข้อมูล');
+  });
+});