From 7b8b7038f767c84204b11feaf56b2028dd54149e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 7 Jul 2026 10:34:06 +0200 Subject: [PATCH] fix: TypeScript-at-scale indexing cascade and parallel-only crash recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An 81k-file TypeScript corpus previously never finished indexing. Three stacked defects, found by stack-sampling the live grind and reading the supervisor's quarantine records: 1) walk_defs pushed children via index-based ts_node_child(i), which is O(i) per call in tree-sitter — O(n^2) per wide node. A 3.5 MB file with ~580k flat comment siblings needed ~1.7e11 iterator steps and hung extraction past the supervisor's quiet-timeout. All six descent sites (including the default loop) now collect children linearly with a TSTreeCursor (wd_collect_children); small nodes keep the direct index path. Guard: extract_wide_flat_file_is_linear (400k comment siblings — the monster's exact shape — bounded at 30s). 2) The sequential cross-LSP driver handed the FULL def list to a full per-file registry build+finalize — O(files x defs); 74% of samples sat in build_qn_index. The parallel resolve worker's dispatch (module-def filter -> shared prebuilt registries -> filtered fallback) is now extracted into cbm_pxc_dispatch_file and drives BOTH pipelines — one path, one semantics. The shared registries live in a caller-owned arena that outlives the calls pass (resolved_calls borrow registry strings; freeing at pass end was a use-after-free ASan caught in pass_calls). Rust is exempt from the def filter: Cargo-manifest cross-crate resolution needs defs from OTHER workspace crates that the own+imports filter drops (this starvation predated the unification on the parallel path). Guard: pipeline_seq_ts_cross_uses_shared_registry (full builds 40 -> 0, cross-file edge preserved). 3) The crash supervisor's recovery re-ran the worker SINGLE-THREADED to keep one exact marker. At scale that fell into the sequential crawl, was killed as a hang mid-pass, and the stale extraction marker got four innocent files quarantined, one 15-minute retry at a time. Recovery (and the terminal partial run) now re-run PARALLEL; the marker is an append journal ("S " / "D ", written by extraction and both resolve drivers), the suspect set is the open-S in-flight set, and a file is quarantined only when it recurs across two consecutive failed runs — one per round, oldest open S first; disjoint consecutive sets stop the loop rather than blame an innocent. Guard: index_recovery_parallel_quarantines_crasher (single-threaded spawn count must stay zero; verified RED against the old loop). Also: parse_ts_type_text gains a dynamic work budget proportional to the input (1M + 64x source bytes, CBM_TS_TYPE_BUDGET override, warn once then degrade to unknown) replacing unbounded recursion cost. Measured on microsoft/TypeScript (81,398 files): infinite -> 33s cold full index, 295k nodes / 779k edges, zero dangling edges, zero files skipped — the monster file itself now indexes cleanly. Full 9-language sweep green with no quality flags; kernel unchanged at 390s / 8.5M nodes / 0 dangling. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 58 +++++-- internal/cbm/cbm.h | 9 ++ internal/cbm/extract_defs.c | 86 +++++++--- internal/cbm/lsp/ts_lsp.c | 62 +++++++ internal/cbm/lsp/ts_lsp.h | 5 + src/mcp/index_supervisor.c | 13 ++ src/mcp/index_supervisor.h | 4 + src/mcp/mcp.c | 192 +++++++++++++++++----- src/pipeline/pass_lsp_cross.c | 185 +++++++++++++++++++-- src/pipeline/pass_lsp_cross.h | 14 ++ src/pipeline/pass_parallel.c | 153 +++--------------- src/pipeline/pipeline.c | 6 + src/pipeline/pipeline_internal.h | 11 ++ tests/test_extraction.c | 87 ++++++++-- tests/test_mcp.c | 270 ++++++++++++++++++++++++++----- tests/test_pipeline.c | 74 +++++++++ 16 files changed, 957 insertions(+), 272 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 8401de4ec..f38eaad69 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -533,25 +533,41 @@ static int count_params_from_signature(const char *sig) { * or spins forever (an external-scanner infinite loop the quiet-timeout kills). * This gives an honest guard — green iff the supervisor actually contains a real * fault — instead of a fixture that may stop faulting once a root cause is fixed. */ -/* Crash-supervisor per-file marker (Stage 3c skip-and-continue). In the - * supervisor's single-threaded recovery re-run, cbm_extract_file records the - * file it is ABOUT to process here (CBM_INDEX_MARKER_FILE) before touching it, - * so that if this file hard-crashes the worker the parent can read the marker - * back and learn the EXACT crasher to quarantine. Only meaningful single- - * threaded (parallel would race the marker); the env var is set solely by the - * supervisor during recovery, so it is a no-op on every normal run. */ -static void cbm_index_write_marker(const char *rel_path) { +/* Crash-supervisor per-file marker JOURNAL (Stage 3c skip-and-continue, + * parallel-safe). Recovery re-runs are PARALLEL (there are no sequential + * production runs), so a single overwrite-style marker would race across + * workers and — worse — go stale during non-extract phases, blaming + * whatever file was extracted LAST (that mis-quarantined four innocent + * ms-typescript fixtures, one 15-minute retry at a time). Instead every + * worker APPENDS one short line per event: "S " when it STARTS + * work on a file, "D " when it finishes it. A single short + * append of one line is atomic in practice on every target platform, and + * the parent discards a torn final line by design. The parent's suspect + * set after a crash/hang = files with an S but no D — exactly the + * in-flight set; a file is only quarantined after appearing in the + * suspect set of TWO CONSECUTIVE failed runs, so a stale or merely + * unlucky in-flight file is never quarantined alone. The env var is set + * solely by the supervisor during recovery — a no-op on normal runs. */ +static void cbm_index_mark(const char *rel_path, char event) { const char *mf = getenv("CBM_INDEX_MARKER_FILE"); if (!mf || !mf[0] || !rel_path || !rel_path[0]) { return; } - FILE *f = cbm_fopen(mf, "wb"); + FILE *f = cbm_fopen(mf, "ab"); if (f) { - (void)fputs(rel_path, f); + (void)fprintf(f, "%c %s\n", event, rel_path); (void)fclose(f); } } +void cbm_index_mark_start(const char *rel_path) { + cbm_index_mark(rel_path, 'S'); +} + +void cbm_index_mark_done(const char *rel_path) { + cbm_index_mark(rel_path, 'D'); +} + /* ── Crash-quarantine set (Stage 3c skip-and-continue) ────────────────────── * After a crash the supervisor re-runs the worker single-threaded, passing * CBM_INDEX_QUARANTINE_FILE — a newline-delimited list of repo-relative paths @@ -667,9 +683,29 @@ static void cbm_test_fault_inject(const char *rel_path) { } } +static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len, + CBMLanguage language, const char *project, + const char *rel_path, int64_t timeout_micros, + const char **extra_defines, const char **include_paths); + +/* Public entry: run the extraction and journal completion. The DONE mark on + * every ordinary return (including error/timeout results) tells the crash + * supervisor this file did NOT kill the worker — only a file whose S has no + * D is a crash/hang suspect. */ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language, const char *project, const char *rel_path, int64_t timeout_micros, const char **extra_defines, const char **include_paths) { + CBMFileResult *r = cbm_extract_file_impl(source, source_len, language, project, rel_path, + timeout_micros, extra_defines, include_paths); + cbm_index_mark_done(rel_path); + return r; +} + +static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len, + CBMLanguage language, const char *project, + const char *rel_path, int64_t timeout_micros, + const char **extra_defines, + const char **include_paths) { // Allocate result on heap (arena inside for all string data) enum { SINGLE = 1 }; CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult)); @@ -691,7 +727,7 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage return result; } - cbm_index_write_marker(rel_path); + cbm_index_mark_start(rel_path); cbm_test_fault_inject(rel_path); // Get language spec diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index e184fe1fa..6b4228131 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -532,6 +532,15 @@ bool cbm_index_is_quarantined(const char *rel_path); // extract loops to report the skip's phase in skipped[] (falls back to "crash"). const char *cbm_index_quarantine_phase(const char *rel_path); +// Crash-supervisor marker journal (parallel-safe): appends "S " / +// "D " to CBM_INDEX_MARKER_FILE. Files with an S but no D form the +// parent's crash/hang suspect set. No-ops when the env var is unset. +// cbm_extract_file journals its own start/done; long-running per-file phases +// (cross-LSP resolve) call these around their per-file work so a hang there +// is attributed to the RIGHT file instead of a stale extraction marker. +void cbm_index_mark_start(const char *rel_path); +void cbm_index_mark_done(const char *rel_path); + // Extract all data from one file. Caller must call cbm_free_result(). // source must remain valid for the duration of the call. // timeout_micros: per-file parse timeout in microseconds (0 = no timeout). diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 1e6ee13db..2de63ffbb 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -5691,6 +5691,58 @@ static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) { s->data[s->top++] = (walk_defs_frame_t){node, enclosing_qn}; } +/* Push all children of `node` in REVERSE order (so they pop in source order) + * using a LINEAR traversal. Index-based ts_node_child(i) is O(i) per call in + * tree-sitter, so a reverse index loop is O(n^2) per node — a file whose + * root has ~580k flat siblings (ms-typescript's reallyLargeFile.ts fourslash + * fixture) needed ~1.7e11 child-iterator steps and hung extraction for + * hours; the supervisor then killed the silent worker as a hang and + * quarantined innocent files off the stale marker. Collect the children + * FORWARD with a TSTreeCursor (O(1) amortized per step) into a scratch + * buffer, then push in reverse. Small nodes keep the direct index loop — + * no cursor/buffer overhead on the overwhelmingly common case. */ +enum { WD_CURSOR_MIN_CHILDREN = 64 }; + +/* Collect all `cc` children of `node` linearly via a TSTreeCursor into a + * malloc'd array (caller frees). Returns NULL for small nodes and on OOM — + * the caller then uses indexed ts_node_child access, which is fine (and + * cheaper) at small child counts and merely quadratic-but-correct on OOM. */ +static TSNode *wd_collect_children(TSNode node, uint32_t cc) { + if (cc < WD_CURSOR_MIN_CHILDREN) { + return NULL; + } + TSNode *buf = (TSNode *)malloc((size_t)cc * sizeof(TSNode)); + if (!buf) { + return NULL; + } + TSTreeCursor cur = ts_tree_cursor_new(node); + uint32_t got = 0; + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + buf[got++] = ts_tree_cursor_current_node(&cur); + } while (got < cc && ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + if (got != cc) { + /* Defensive: cursor and child_count disagree — fall back to indexed. */ + free(buf); + return NULL; + } + return buf; +} + +static void wd_push_children_reverse(wd_stack_t *s, TSNode node, const char *enclosing_qn) { + uint32_t cc = ts_node_child_count(node); + if (cc == 0) { + return; + } + TSNode *kids = wd_collect_children(node, cc); + for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) { + wd_push(s, kids ? kids[i] : ts_node_child(node, (uint32_t)i), enclosing_qn); + } + free(kids); +} + // Push nested class nodes from a class body container onto the defs stack. // Iteratively walks into wrapper nodes (field_declaration, template_declaration). static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_stack_t *s, @@ -5702,8 +5754,10 @@ static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_sta while (nc_stack.count > 0) { TSNode cur = ts_nstack_pop(&nc_stack); uint32_t nc = ts_node_child_count(cur); + /* Linear child access for wide class bodies (see wd_collect_children). */ + TSNode *kids = wd_collect_children(cur, nc); for (int i = (int)nc - SKIP_CHAR; i >= 0; i--) { - TSNode child = ts_node_child(cur, (uint32_t)i); + TSNode child = kids ? kids[i] : ts_node_child(cur, (uint32_t)i); if (cbm_kind_in_set(child, spec->class_node_types)) { wd_push(s, child, enclosing_qn); } else { @@ -5714,6 +5768,7 @@ static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_sta } } } + free(kids); } } @@ -6220,10 +6275,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, // resolve a null name and, for grammars where the kind has a `name` // field, double-mint). Push children so nested tags/defs are still // traversed, then skip the generic func path. - uint32_t cc = ts_node_child_count(node); - for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) { - wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn); - } + wd_push_children_reverse(&s, node, frame.enclosing_class_qn); continue; } @@ -6234,10 +6286,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, // generic extract_func_def below would double-mint a def whose name // still carries the quotes. Push children so nested defines are still // traversed, then skip the generic func path. - uint32_t cc = ts_node_child_count(node); - for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) { - wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn); - } + wd_push_children_reverse(&s, node, frame.enclosing_class_qn); continue; } @@ -6262,10 +6311,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, (strcmp(kind, "export_item") == 0 || strcmp(kind, "import_item") == 0) && ts_node_is_null( find_first_descendant_by_kind(node, "func_type", CBM_DESCENDANT_MAX_DEPTH))) { - uint32_t cc = ts_node_child_count(node); - for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) { - wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn); - } + wd_push_children_reverse(&s, node, frame.enclosing_class_qn); continue; } @@ -6303,10 +6349,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, * the class/func paths on the namespace node itself. */ if (is_namespace_scope_kind(ctx->language, kind)) { const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn); - uint32_t nsc = ts_node_child_count(node); - for (int i = (int)nsc - SKIP_CHAR; i >= 0; i--) { - wd_push(&s, ts_node_child(node, (uint32_t)i), new_enclosing); - } + wd_push_children_reverse(&s, node, new_enclosing); continue; } @@ -6317,10 +6360,11 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, continue; } - uint32_t count = ts_node_child_count(node); - for (int i = (int)count - SKIP_CHAR; i >= 0; i--) { - wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn); - } + /* Default descent — THE hot loop: a file whose root has hundreds of + * thousands of flat siblings (580k comment lines in ms-typescript's + * reallyLargeFile.ts) lands here every visit, so linear child + * collection is mandatory (see wd_push_children_reverse). */ + wd_push_children_reverse(&s, node, frame.enclosing_class_qn); } free(s.data); } diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index cc233c700..556ae8f2e 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -27,10 +27,49 @@ */ #include "ts_lsp.h" +#include #include #include #include +/* TEST HOOK: full per-file cross-registry builds (stdlib + every cross-file + * def registered + finalized, per file). This is the sequential-mode quadratic + * that ground an 81k-file TS corpus for hours — the shared-registry dispatch + * must keep it at ZERO; any nonzero count means a resolve path regressed to + * the per-file build. */ +static _Atomic long g_ts_full_reg_builds; +long cbm_ts_full_registry_builds(void) { + return atomic_load(&g_ts_full_reg_builds); +} +void cbm_ts_full_registry_builds_reset(void) { + atomic_store(&g_ts_full_reg_builds, 0); +} + +/* Dynamic work budget for parse_ts_type_text (thread-local, reset at every + * per-file/per-build entry point). The type-text parser is self-recursive + * over string SLICES and re-derives lengths per call, so an adversarial or + * generated type text can cost far more than its bytes suggest. Instead of + * a fixed depth cap, the total work SCALES WITH THE INPUT: budget = + * 1M + 64 x source_len units, each call charging 1 + slice_len/16 — i.e. + * total scanned bytes are bounded at ~1024x the source size, generous + * enough that no real-world file ever hits it. CBM_TS_TYPE_BUDGET (absolute + * units) overrides. On exhaustion: WARN once per window, then return + * UNKNOWN — the same graceful degradation the parser already uses for + * constructs it does not model. -1 = unlimited (no entry point armed it). */ +static _Thread_local long g_ts_type_budget = -1; +static _Thread_local bool g_ts_type_budget_warned; + +static void ts_type_budget_reset(size_t source_len) { + const char *e = getenv("CBM_TS_TYPE_BUDGET"); + if (e && e[0]) { + long v = atol(e); + g_ts_type_budget = (v > 0) ? v : -1; + } else { + g_ts_type_budget = 1000000 + (long)source_len * 64; + } + g_ts_type_budget_warned = false; +} + #define TS_LSP_MAX_EVAL_DEPTH 64 #define TS_LSP_FIELD_LEN(s) ((uint32_t)(sizeof(s) - 1)) @@ -156,6 +195,22 @@ static const CBMType *parse_ts_type_text(CBMArena *arena, const char *text, cons if (len == 0) return cbm_type_unknown(); + /* Dynamic budget (see g_ts_type_budget): charge proportional to this + * slice; on exhaustion degrade to UNKNOWN instead of grinding. */ + if (g_ts_type_budget >= 0) { + g_ts_type_budget -= 1 + (long)(len / 16); + if (g_ts_type_budget < 0) { + if (!g_ts_type_budget_warned) { + g_ts_type_budget_warned = true; + fprintf(stderr, + " [tslsp] type-text parse budget exhausted (module %s); " + "returning unknown\n", + module_qn ? module_qn : "?"); + } + return cbm_type_unknown(); + } + } + // Function type `(params) => returnType` (TS function type literal). if (text[0] == '(') { int depth = 0; @@ -4784,6 +4839,7 @@ void cbm_run_ts_lsp(CBMArena *arena, CBMFileResult *result, const char *source, TSNode root, bool js_mode, bool jsx_mode, bool dts_mode) { if (!arena || !result || !source || ts_node_is_null(root)) return; + ts_type_budget_reset((size_t)source_len); // Diagnostic / benchmarking knob: setting `CBM_LSP_DISABLED=1` skips the resolver. // This is used by the baseline-vs-LSP comparison tests to measure how many calls @@ -4977,6 +5033,8 @@ CBMTypeRegistry *cbm_ts_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i return NULL; cbm_registry_init(reg, arena); cbm_ts_stdlib_register(reg, arena); + /* Budget scales with the def volume this build parses type texts for. */ + ts_type_budget_reset((size_t)(def_count > 0 ? def_count : 1) * 1024); for (int i = 0; i < def_count; i++) { CBMLSPDef *d = &defs[i]; if (d->lang != CBM_LANG_JAVASCRIPT && d->lang != CBM_LANG_TYPESCRIPT && @@ -5006,6 +5064,8 @@ void cbm_run_ts_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!arena || !out || !reg) return; + ts_type_budget_reset((size_t)source_len); + /* Per-file overlay: register only the file's own-module defs so the * AST passes can refine them. Imports/stdlib resolve via fallback. */ CBMTypeRegistry overlay; @@ -5075,6 +5135,8 @@ void cbm_run_ts_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!arena || !out) return; + atomic_fetch_add(&g_ts_full_reg_builds, 1); + ts_type_budget_reset((size_t)source_len); CBMTypeRegistry reg; cbm_registry_init(®, arena); cbm_ts_stdlib_register(®, arena); diff --git a/internal/cbm/lsp/ts_lsp.h b/internal/cbm/lsp/ts_lsp.h index 154d38ee7..1473ce1bd 100644 --- a/internal/cbm/lsp/ts_lsp.h +++ b/internal/cbm/lsp/ts_lsp.h @@ -123,6 +123,11 @@ void cbm_run_ts_lsp_cross_with_registry(CBMArena *arena, const char *source, int // will replace this in v1.3. void cbm_ts_stdlib_register(CBMTypeRegistry *reg, CBMArena *arena); +// TEST HOOKS: count of full per-file cross-registry builds (the quadratic the +// shared-registry dispatch eliminates; must stay 0 on the shared path). +long cbm_ts_full_registry_builds(void); +void cbm_ts_full_registry_builds_reset(void); + // --- Batch cross-file LSP --- // Per-file input for batch TS LSP processing. diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index ae71f163b..976c10d54 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -53,6 +53,16 @@ int cbm_index_supervisor_spawn_count(void) { return g_spawn_count; } +/* Test hook: counts SINGLE-THREADED spawns. Production recovery is parallel- + * only (there are no sequential production runs); this must stay ZERO on + * every supervised path — any nonzero count means a recovery/probe regressed + * to the sequential crawl that ground an 81k-file TS corpus for hours. */ +static int g_spawn_st_count = 0; + +int cbm_index_supervisor_spawn_st_count(void) { + return g_spawn_st_count; +} + /* #845: opt-in host mark — see the header. Set once from the real binary's * main(); embedders never set it, so should_wrap() stays false for them. */ static bool g_host_marked = false; @@ -137,6 +147,9 @@ static void worker_tmp_path(char *out, size_t out_sz, int pid, const char *suffi int cbm_index_spawn_worker(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_result_t *result) { g_spawn_count++; /* test hook (#845) — see cbm_index_supervisor_spawn_count */ + if (single_thread) { + g_spawn_st_count++; /* test hook — must stay 0: recovery is parallel-only */ + } result->outcome = CBM_PROC_SPAWN_FAILED; result->exit_code = -1; result->term_signal = 0; diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index c976311e6..e15526140 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -54,6 +54,10 @@ bool cbm_index_supervisor_should_wrap(void); * across an index_repository call to prove indexing ran IN-PROCESS. */ int cbm_index_supervisor_spawn_count(void); +/* Test hook: single-threaded spawn count — must stay ZERO (production + * recovery is parallel-only; no sequential runs). */ +int cbm_index_supervisor_spawn_st_count(void); + typedef struct { cbm_proc_outcome_t outcome; /* how the worker ended */ int exit_code; /* worker exit code (-1 if signalled) */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 070196f1b..9330f31bd 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -3529,23 +3529,93 @@ static void supervisor_tmp_path(char *out, size_t out_sz, const char *suffix) { } } -/* Read the single-line crash marker (the rel_path the worker was processing when - * it died). Returns a trimmed heap string (caller frees), or NULL when the marker - * is empty/unreadable (i.e. the crash could not be attributed to a file). */ -static char *supervisor_read_marker(const char *path) { +/* Parse the worker's marker JOURNAL ("S " / "D " lines, one event + * per line — see cbm_index_mark_start/done) into the crash/hang SUSPECT set: + * files whose last event is an S with no closing D, i.e. the in-flight set + * at kill time. Recovery runs are PARALLEL, so there are up to worker_count + * suspects; a torn final line (no trailing newline) is discarded by design. + * Returns a malloc'd array of malloc'd rel paths, OLDEST OPEN S FIRST (for a + * hang, the oldest still-open file IS the stuck one). Caller frees via + * supervisor_free_suspects. */ +static char **supervisor_read_suspects(const char *path, int *out_n) { + *out_n = 0; FILE *f = cbm_fopen(path, "rb"); if (!f) { return NULL; } - char buf[CBM_SZ_1K]; - size_t n = fread(buf, 1, sizeof(buf) - 1, f); + char **open_paths = NULL; /* open (S-without-D) files in first-S order */ + int open_n = 0; + int open_cap = 0; + char line[CBM_SZ_1K]; + while (fgets(line, sizeof(line), f)) { + size_t len = strlen(line); + if (len == 0 || line[len - 1] != '\n') { + break; /* torn final line — discard and stop */ + } + line[--len] = '\0'; + if (len > 0 && line[len - 1] == '\r') { + line[--len] = '\0'; + } + if (len < 3 || (line[0] != 'S' && line[0] != 'D') || line[1] != ' ') { + continue; + } + const char *rel = line + 2; + if (line[0] == 'S') { + bool already = false; + for (int i = 0; i < open_n && !already; i++) { + already = strcmp(open_paths[i], rel) == 0; + } + if (already) { + continue; + } + if (open_n == open_cap) { + int ncap = open_cap ? open_cap * 2 : 16; + char **np = (char **)realloc(open_paths, (size_t)ncap * sizeof(char *)); + if (!np) { + break; + } + open_paths = np; + open_cap = ncap; + } + open_paths[open_n++] = cbm_strdup(rel); + } else { + for (int i = 0; i < open_n; i++) { + if (strcmp(open_paths[i], rel) == 0) { + free(open_paths[i]); + memmove(&open_paths[i], &open_paths[i + 1], + (size_t)(open_n - i - 1) * sizeof(char *)); + open_n--; + break; + } + } + } + } (void)fclose(f); - buf[n] = '\0'; - size_t len = strlen(buf); - while (len > 0 && (buf[len - 1] == '\n' || buf[len - 1] == '\r' || buf[len - 1] == ' ')) { - buf[--len] = '\0'; + if (open_n == 0) { + free(open_paths); + return NULL; + } + *out_n = open_n; + return open_paths; +} + +static void supervisor_free_suspects(char **s, int n) { + if (!s) { + return; + } + for (int i = 0; i < n; i++) { + free(s[i]); + } + free(s); +} + +static bool supervisor_suspect_contains(char **s, int n, const char *rel) { + for (int i = 0; i < n; i++) { + if (s[i] && strcmp(s[i], rel) == 0) { + return true; + } } - return len > 0 ? cbm_strdup(buf) : NULL; + return false; } /* Append one quarantine entry "rel\tphase\n" (phase = "crash"|"hang") to the @@ -3597,12 +3667,22 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { return resp; } - /* Crash / hang / nonzero exit → skip-and-continue recovery. Re-run the worker - * single-threaded with a per-file marker; each time it crashes/hangs the - * marker names the exact crasher, which we append to the quarantine file - * before re-spawning. A clean single-threaded run then indexes the good files - * and reports the quarantined ones as phase="crash" skips (via the ordinary - * Stage-2 skip plumbing — no JSON merge needed). */ + /* Crash / hang / nonzero exit → skip-and-continue recovery. Re-run the + * worker PARALLEL (there are no sequential production runs) with the + * per-file marker JOURNAL armed; after each failed run the journal's + * open-S set is the in-flight SUSPECT set. A file is quarantined only + * when it appears in the suspect sets of TWO CONSECUTIVE failed runs + * (intersection — a stale or merely unlucky in-flight file rotates out), + * and only ONE file per round: the OLDEST open S in the intersection + * (for a hang the oldest still-open file IS the stuck one; for a crash + * it is the longest-running suspect — the best single deterministic + * pick). A clean run then indexes the good files and reports the + * quarantined ones as phase="crash"/"hang" skips via the ordinary + * Stage-2 skip plumbing. The old design re-ran SINGLE-THREADED to keep + * one exact marker; at scale that fell into the sequential crawl, went + * quiet, was killed as a hang mid-pass, and the stale marker got FOUR + * innocent ms-typescript fixtures quarantined one 15-minute retry at a + * time. */ cbm_proc_outcome_t last_outcome = wr.outcome; cbm_index_worker_result_free(&wr); @@ -3627,10 +3707,13 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { } char *resp = NULL; - int quarantined = 0; /* files pinned + added to the quarantine list so far */ + int quarantined = 0; /* files pinned + added to the quarantine list so far */ + char **prev_suspects = NULL; /* previous failed round's in-flight set */ + int prev_n = 0; for (int i = 0; i < cap; i++) { cbm_index_worker_result_t wr2; - int rc2 = cbm_index_spawn_worker(args, true, marker_path, quarantine_path, &wr2); + int rc2 = cbm_index_spawn_worker(args, /*single_thread=*/false, marker_path, + quarantine_path, &wr2); if (rc2 != 0) { last_outcome = wr2.outcome; cbm_index_worker_result_free(&wr2); @@ -3645,30 +3728,49 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { if (wr2.outcome == CBM_PROC_CRASH || wr2.outcome == CBM_PROC_HANG) { last_outcome = wr2.outcome; cbm_index_worker_result_free(&wr2); - /* crash vs hang: the phase this file is quarantined under and reported - * as in skipped[]. A fault signal → "crash"; a no-progress kill → - * "hang". */ + /* crash vs hang: the phase this file is quarantined under and + * reported as in skipped[]. A fault signal → "crash"; a + * no-progress kill → "hang". */ const char *phase = (last_outcome == CBM_PROC_HANG) ? "hang" : "crash"; - /* Attribute the failure to the file the marker pinned and quarantine it. - * If the marker is empty/unreadable we cannot attribute it to a single - * file → stop and report a contained failure. */ - char *crasher = supervisor_read_marker(marker_path); - if (!crasher) { + int sus_n = 0; + char **suspects = supervisor_read_suspects(marker_path, &sus_n); + (void)remove(marker_path); /* fresh journal for the next re-run */ + if (!suspects || sus_n == 0) { + supervisor_free_suspects(suspects, sus_n); cbm_log_warn("index.supervisor.unattributable", "action", "give_up"); break; } - if (!supervisor_append_quarantine(quarantine_path, crasher, phase)) { - cbm_log_warn("index.supervisor.quarantine_write_fail", "path", crasher); - free(crasher); - break; + if (prev_suspects) { + /* Two-consecutive-strikes: quarantine the OLDEST open S that + * was also in flight in the previous failed round. */ + const char *pick = NULL; + for (int k = 0; k < sus_n && !pick; k++) { + if (supervisor_suspect_contains(prev_suspects, prev_n, suspects[k])) { + pick = suspects[k]; + } + } + if (!pick) { + /* Disjoint consecutive in-flight sets: the failure is not + * attributable to a recurring file (systemic) — stop + * rather than quarantine an innocent. */ + supervisor_free_suspects(suspects, sus_n); + cbm_log_warn("index.supervisor.unattributable", "action", "give_up"); + break; + } + if (!supervisor_append_quarantine(quarantine_path, pick, phase)) { + cbm_log_warn("index.supervisor.quarantine_write_fail", "path", pick); + supervisor_free_suspects(suspects, sus_n); + break; + } + quarantined++; + char attempt_buf[MCP_FIELD_SIZE]; + snprintf(attempt_buf, sizeof(attempt_buf), "%d", i + 1); + cbm_log_warn("index.file_quarantined", "path", pick, "outcome", phase, "attempt", + attempt_buf); } - quarantined++; - char attempt_buf[MCP_FIELD_SIZE]; - snprintf(attempt_buf, sizeof(attempt_buf), "%d", i + 1); - cbm_log_warn("index.file_quarantined", "path", crasher, "outcome", phase, "attempt", - attempt_buf); - free(crasher); - (void)remove(marker_path); /* fresh marker for the next re-run */ + supervisor_free_suspects(prev_suspects, prev_n); + prev_suspects = suspects; + prev_n = sus_n; continue; } /* SPAWN_FAILED / nonzero exit / non-fault kill → not a crash we can @@ -3677,19 +3779,21 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { cbm_index_worker_result_free(&wr2); break; } + supervisor_free_suspects(prev_suspects, prev_n); (void)remove(marker_path); /* marker no longer needed */ /* Terminal best-effort-partial: the loop exited WITHOUT a clean run (cap * exhausted, or an unattributable failure) but at least one file was already - * quarantined. Try ONE final single-threaded spawn with the accumulated - * quarantine and NO marker — every known-bad file short-circuits, so a clean - * run yields a PARTIAL index (all good files indexed, all known crashers/hangs - * reported as skips) rather than a hard failure. Bounded by the same quiet- - * timeout, so it cannot itself hang. Rare given monotonic progress. */ + * quarantined. Try ONE final PARALLEL spawn with the accumulated quarantine + * and NO marker — every known-bad file short-circuits, so a clean run yields + * a PARTIAL index (all good files indexed, all known crashers/hangs reported + * as skips) rather than a hard failure. Bounded by the same quiet-timeout, + * so it cannot itself hang. Rare given monotonic progress. */ if (!resp && quarantined > 0) { cbm_index_worker_result_t wrp; - int rcp = cbm_index_spawn_worker(args, true, NULL, quarantine_path, &wrp); + int rcp = + cbm_index_spawn_worker(args, /*single_thread=*/false, NULL, quarantine_path, &wrp); if (rcp == 0 && wrp.outcome == CBM_PROC_CLEAN && wrp.response) { resp = wrp.response; /* transfer ownership to caller */ wrp.response = NULL; diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 40ec8e264..7194bd5b5 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -621,6 +621,147 @@ void cbm_pxc_run_one_ts(CBMFileResult *r, const char *source, int source_len, co * there is no readable Cargo.toml, leaving *out_m untouched. The resulting * manifest feeds cross-CRATE Rust resolution (#56): its [workspace].members * map lets `crate_a::foo` route to the member crate's def. */ +/* Per-file cross-LSP dispatch, shared by the PARALLEL resolve worker and the + * SEQUENTIAL driver. One code path = one semantics: filter the global defs + * down to the file's own+imported modules via the module-def index, resolve + * through the shared prebuilt registry when the language has one (per-file + * OVERLAY pattern — no registry build, no finalize), and only fall back to + * the per-file registry build (with the FILTERED defs) for languages without + * a shared-registry variant. Before this helper existed the sequential + * driver fed the FULL def list into full per-file registry builds — + * O(files x defs), which ground an 81k-file TS corpus for hours. + * + * `rust_shared_get` supplies the lazily-built shared Rust all-defs registry + * (the parallel resolver owns its once-guard); NULL means "no shared rust + * registry available" and rust NULL-filter files take the per-file build. */ +void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *source, + int source_len, const char *rel, const char *def_module, + const CBMCrossLspRegistries *cross_registries, + const CBMModuleDefIndex *module_def_index, CBMLSPDef *all_defs, + int all_def_count, const char **imp_keys, const char **imp_vals, + int imp_count, CBMTypeRegistry *(*rust_shared_get)(void *), + void *rust_shared_ctx) { + if (!result) { + return; + } + bool used_prebuilt = false; + CBMTypeRegistry *prebuilt = + cross_registries ? cbm_pxc_registry_for_lang(cross_registries, lang) : NULL; + if (prebuilt) { + switch (lang) { + case CBM_LANG_GO: + /* Tier 3 (metadata-driven): pure lookup over the Tier-1 + * lsp_unresolved entries — no parse, no AST walk. */ + cbm_go_fast_resolve_qualified_calls(result, prebuilt, imp_keys, imp_vals, imp_count); + used_prebuilt = true; + break; + case CBM_LANG_PYTHON: + cbm_run_py_lsp_cross_with_registry(&result->arena, source, source_len, def_module, + prebuilt, imp_keys, imp_vals, imp_count, + result->cached_tree, &result->resolved_calls); + used_prebuilt = true; + break; + case CBM_LANG_C: + case CBM_LANG_CPP: + case CBM_LANG_CUDA: + cbm_run_c_lsp_cross_with_registry( + &result->arena, source, source_len, def_module, (lang != CBM_LANG_C), prebuilt, + imp_keys, imp_vals, imp_count, result->cached_tree, &result->resolved_calls); + used_prebuilt = true; + break; + case CBM_LANG_CSHARP: + cbm_run_cs_lsp_cross_with_registry(&result->arena, source, source_len, def_module, + prebuilt, imp_vals, imp_count, result->cached_tree, + &result->resolved_calls); + used_prebuilt = true; + break; + case CBM_LANG_JAVASCRIPT: + case CBM_LANG_TYPESCRIPT: + case CBM_LANG_TSX: { + /* TS: per-file OVERLAY chained to the shared base. Filter to + * own+imports so the overlay builder can pick out own-module + * defs without scanning the whole project. */ + bool js; + bool jsx; + bool dts; + cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); + CBMLSPDef *ts_defs = all_defs; + int ts_def_count = all_def_count; + CBMLSPDef *ts_filtered = NULL; + if (module_def_index) { + int fc = 0; + ts_filtered = cbm_pxc_filter_defs_for_file(module_def_index, all_defs, lang, + result->namespace_name, def_module, + imp_vals, imp_count, &fc); + if (ts_filtered) { + ts_defs = ts_filtered; + ts_def_count = fc; + } + } + cbm_run_ts_lsp_cross_with_registry(&result->arena, source, source_len, def_module, js, + jsx, dts, prebuilt, ts_defs, ts_def_count, imp_keys, + imp_vals, imp_count, result->cached_tree, + &result->resolved_calls); + free(ts_filtered); + used_prebuilt = true; + break; + } + /* PHP falls through to the per-file build path below until its + * overlay variant lands. */ + default: + break; + } + } + + if (used_prebuilt) { + return; + } + /* Fallback: gopls per-file filter + per-file registry build. RUST is + * exempt from the module filter: its resolution is Cargo-manifest-aware + * and a `crate_a::foo` reference routes to defs in ANOTHER workspace + * crate — a module that is in neither own_module nor the import map, so + * the filter starves cross-crate resolution (#56 repro red). Rust + * therefore always resolves against the FULL def universe: the lazily + * built shared registry when available, else a full per-file build. */ + CBMLSPDef *filtered = NULL; + CBMLSPDef *file_defs = all_defs; + int file_def_count = all_def_count; + if (module_def_index && lang != CBM_LANG_RUST) { + int filtered_count = 0; + filtered = + cbm_pxc_filter_defs_for_file(module_def_index, all_defs, lang, result->namespace_name, + def_module, imp_vals, imp_count, &filtered_count); + if (filtered) { + file_defs = filtered; + file_def_count = filtered_count; + } + } + if (lang == CBM_LANG_RUST) { + CBMTypeRegistry *shared = rust_shared_get ? rust_shared_get(rust_shared_ctx) : NULL; + if (shared) { + cbm_run_rust_lsp_cross_with_registry(&result->arena, source, source_len, def_module, + shared, imp_keys, imp_vals, imp_count, + result->cached_tree, cbm_pxc_get_rust_manifest(), + &result->resolved_calls, + /*result=*/NULL); + } else { + cbm_pxc_run_one(lang, result, source, source_len, def_module, file_defs, file_def_count, + imp_keys, imp_vals, imp_count); + } + } else if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX) { + bool js; + bool jsx; + bool dts; + cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); + cbm_pxc_run_one_ts(result, source, source_len, def_module, file_defs, file_def_count, + imp_keys, imp_vals, imp_count, js, jsx, dts); + } else { + cbm_pxc_run_one(lang, result, source, source_len, def_module, file_defs, file_def_count, + imp_keys, imp_vals, imp_count); + } + free(filtered); +} + static bool pxc_build_rust_manifest(const cbm_pipeline_ctx_t *ctx, CBMArena *marena, CBMCargoManifest *out_m) { if (!ctx || !ctx->repo_path || !marena || !out_m) @@ -680,6 +821,32 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name, def_modules, &def_count); + /* Shared prepare (mirrors run_parallel_pipeline): inverted module-def + * index + per-language shared registries, built ONCE for the whole pass. + * The per-file loop below then dispatches through the SAME helper the + * parallel resolve worker uses — previously this driver handed the FULL + * def list to full per-file registry builds (O(files x defs); the + * ms-typescript sequential crawl). The registries live in the + * CALLER-OWNED ctx->seq_cross_arena: resolved_calls may borrow registry + * strings that the later calls pass still reads, so the arena must + * outlive this pass (run_sequential_pipeline destroys it after all + * passes; freeing here was a pass_calls use-after-free). */ + CBMModuleDefIndex *module_def_index = + all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL; + CBMCrossLspRegistries cross_registries = {0}; + if (all_defs) { + CBMArena *xa = &ctx->seq_cross_arena; + if (!ctx->seq_cross_arena_live) { + cbm_arena_init(xa); + ctx->seq_cross_arena_live = true; + } + cross_registries.go = cbm_go_build_cross_registry(xa, all_defs, def_count); + cross_registries.python = cbm_py_build_cross_registry(xa, all_defs, def_count); + cross_registries.c = cbm_c_build_cross_registry(xa, all_defs, def_count); + cross_registries.cs = cbm_cs_build_cross_registry(xa, all_defs, def_count); + cross_registries.ts = cbm_ts_build_cross_registry(xa, all_defs, def_count); + } + int processed = 0; int skipped_no_lsp = 0; int skipped_no_source = 0; @@ -713,15 +880,14 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * pxc_build_import_map(ctx->gbuf, ctx->project_name, files[i].rel_path, &imp_keys, &imp_vals, &imp_count); - if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX) { - bool js, jsx, dts; - cbm_pxc_ts_modes(lang, files[i].rel_path, &js, &jsx, &dts); - cbm_pxc_run_one_ts(cache[i], source, source_len, def_modules[i], all_defs, def_count, - imp_keys, imp_vals, imp_count, js, jsx, dts); - } else { - cbm_pxc_run_one(lang, cache[i], source, source_len, def_modules[i], all_defs, def_count, - imp_keys, imp_vals, imp_count); - } + /* Journal around the resolve: a hang here must be attributed to THIS + * file, not to a stale extraction marker (the innocent-quarantine + * failure mode). */ + cbm_index_mark_start(files[i].rel_path); + cbm_pxc_dispatch_file(lang, cache[i], source, source_len, files[i].rel_path, def_modules[i], + &cross_registries, module_def_index, all_defs, def_count, imp_keys, + imp_vals, imp_count, NULL, NULL); + cbm_index_mark_done(files[i].rel_path); per_lang_calls++; processed++; @@ -729,6 +895,7 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * free(source); } + cbm_pxc_free_module_def_index(module_def_index); free(all_defs); for (int i = 0; i < file_count; i++) free(def_modules[i]); diff --git a/src/pipeline/pass_lsp_cross.h b/src/pipeline/pass_lsp_cross.h index 27149807b..b63a1ce11 100644 --- a/src/pipeline/pass_lsp_cross.h +++ b/src/pipeline/pass_lsp_cross.h @@ -161,4 +161,18 @@ void cbm_pxc_run_one_ts(CBMFileResult *r, const char *source, int source_len, co const char **imp_vals, int imp_count, bool js_mode, bool jsx_mode, bool dts_mode); +/* Per-file cross-LSP dispatch shared by the parallel resolve worker AND the + * sequential driver (one path = one semantics): module-def-index filter → + * shared prebuilt registry (overlay pattern, no per-file registry build) → + * per-file fallback with FILTERED defs for languages without a shared + * variant. rust_shared_get (nullable) supplies the lazily-built shared Rust + * registry for NULL-filter rust files. */ +void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *source, + int source_len, const char *rel, const char *def_module, + const CBMCrossLspRegistries *cross_registries, + const CBMModuleDefIndex *module_def_index, CBMLSPDef *all_defs, + int all_def_count, const char **imp_keys, const char **imp_vals, + int imp_count, CBMTypeRegistry *(*rust_shared_get)(void *), + void *rust_shared_ctx); + #endif /* CBM_PIPELINE_PASS_LSP_CROSS_H */ diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index da3957424..8e0d35691 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2584,6 +2584,12 @@ static CBMTypeRegistry *pp_rust_shared_registry(resolve_ctx_t *rc) { return p; } +/* void*-typed adapter so the shared dispatch helper (pass_lsp_cross.c) can + * borrow the lazily-built shared Rust registry without knowing resolve_ctx_t. */ +static CBMTypeRegistry *pp_rust_shared_registry_get(void *ctx) { + return pp_rust_shared_registry((resolve_ctx_t *)ctx); +} + static void resolve_worker(int worker_id, void *ctx_ptr) { resolve_ctx_t *rc = ctx_ptr; resolve_worker_state_t *ws = &rc->workers[worker_id]; @@ -2733,141 +2739,18 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { uint64_t lsp_t0 = extract_now_ns(); - /* Tier 2 full fast path: pre-built per-language registry. - * When available, skip the per-file registry build entirely - * and pass the shared finalized registry. Dispatch per-lang - * to the appropriate _with_registry variant. */ - bool used_prebuilt = false; - CBMTypeRegistry *prebuilt = cbm_pxc_registry_for_lang(rc->cross_registries, lang); - if (prebuilt) { - switch (lang) { - case CBM_LANG_GO: { - /* Tier 3 (metadata-driven): the per-file LSP - * during extract ALREADY captured receiver-type - * QNs and pkg-aliased call expressions inside - * result->resolved_calls entries flagged with - * strategy="lsp_unresolved". Cross-LSP is now a - * pure lookup pass — iterate those, look up in - * the global pre-built registry, emit resolved - * entries on top. NO TREE-SITTER PARSE, NO AST - * WALK. The slow path - * (cbm_run_go_lsp_cross_with_registry) would - * just re-derive the same metadata via a second - * AST walk and arrive at the same answers — it - * is now skipped entirely for Go. */ - cbm_go_fast_resolve_qualified_calls(result, prebuilt, imp_keys, imp_vals, - imp_count); - used_prebuilt = true; - break; - } - case CBM_LANG_PYTHON: - cbm_run_py_lsp_cross_with_registry( - &result->arena, lsp_source, lsp_source_len, def_module, prebuilt, - imp_keys, imp_vals, imp_count, result->cached_tree, - &result->resolved_calls); - used_prebuilt = true; - break; - case CBM_LANG_C: - case CBM_LANG_CPP: - case CBM_LANG_CUDA: - cbm_run_c_lsp_cross_with_registry( - &result->arena, lsp_source, lsp_source_len, def_module, - (lang != CBM_LANG_C), prebuilt, imp_keys, imp_vals, imp_count, - result->cached_tree, &result->resolved_calls); - used_prebuilt = true; - break; - case CBM_LANG_CSHARP: - cbm_run_cs_lsp_cross_with_registry( - &result->arena, lsp_source, lsp_source_len, def_module, prebuilt, - imp_vals, imp_count, result->cached_tree, &result->resolved_calls); - used_prebuilt = true; - break; - case CBM_LANG_JAVASCRIPT: - case CBM_LANG_TYPESCRIPT: - case CBM_LANG_TSX: { - /* TS uses a per-file OVERLAY chained to the shared - * base (prebuilt): the file's own-module defs are - * registered into the overlay so the AST refinement - * passes can mutate them; imports/stdlib resolve via - * the shared base. Filter to own+imports so the - * overlay builder can pick out own-module defs. */ - bool js, jsx, dts; - cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); - CBMLSPDef *ts_defs = rc->all_defs; - int ts_def_count = rc->def_count; - CBMLSPDef *ts_filtered = NULL; - if (rc->module_def_index) { - int fc = 0; - ts_filtered = cbm_pxc_filter_defs_for_file( - rc->module_def_index, rc->all_defs, lang, result->namespace_name, - def_module, imp_vals, imp_count, &fc); - if (ts_filtered) { - ts_defs = ts_filtered; - ts_def_count = fc; - } - } - cbm_run_ts_lsp_cross_with_registry( - &result->arena, lsp_source, lsp_source_len, def_module, js, jsx, dts, - prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count, - result->cached_tree, &result->resolved_calls); - free(ts_filtered); - used_prebuilt = true; - break; - } - /* PHP falls through to the per-file build path below - * until its overlay variant lands. */ - default: - break; - } - } - - CBMLSPDef *filtered = NULL; - if (!used_prebuilt) { - /* Fallback: gopls per-file filter + per-file registry build. */ - CBMLSPDef *file_defs = rc->all_defs; - int file_def_count = rc->def_count; - int filtered_count = 0; - if (rc->module_def_index) { - filtered = cbm_pxc_filter_defs_for_file( - rc->module_def_index, rc->all_defs, lang, result->namespace_name, - def_module, imp_vals, imp_count, &filtered_count); - if (filtered) { - file_defs = filtered; - file_def_count = filtered_count; - } - } - if (lang == CBM_LANG_RUST && !filtered) { - /* Rust NULL-filter file (the ~all_defs amplifier) → resolve against - * the LAZILY-built shared registry instead of rebuilding all_defs - * per file. Byte-identical: a per-file all_defs build == the shared - * all_defs build (def_module_qn always set). SUBSET rust files - * (filtered != NULL) fall to the per-file build below, unchanged. - * Manifest via the getter = same value cbm_pxc_run_one reads here. */ - CBMTypeRegistry *shared = pp_rust_shared_registry(rc); - if (shared) { - cbm_run_rust_lsp_cross_with_registry( - &result->arena, lsp_source, lsp_source_len, def_module, shared, - imp_keys, imp_vals, imp_count, result->cached_tree, - cbm_pxc_get_rust_manifest(), &result->resolved_calls, - /*result=*/NULL); - } else { - cbm_pxc_run_one(lang, result, lsp_source, lsp_source_len, def_module, - file_defs, file_def_count, imp_keys, imp_vals, - imp_count); - } - } else if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || - lang == CBM_LANG_TSX) { - bool js, jsx, dts; - cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts); - cbm_pxc_run_one_ts(result, lsp_source, lsp_source_len, def_module, - file_defs, file_def_count, imp_keys, imp_vals, imp_count, - js, jsx, dts); - } else { - cbm_pxc_run_one(lang, result, lsp_source, lsp_source_len, def_module, - file_defs, file_def_count, imp_keys, imp_vals, imp_count); - } - } - free(filtered); + /* Shared per-file dispatch (pass_lsp_cross.c): module-def + * filter → shared prebuilt registry (overlay pattern) → + * filtered per-file fallback. The SAME helper drives the + * sequential pass — one path, one semantics. Journal the + * file around the resolve so a hang HERE is attributed to + * this file, not to a stale extraction marker. */ + cbm_index_mark_start(rel); + cbm_pxc_dispatch_file(lang, result, lsp_source, lsp_source_len, rel, def_module, + rc->cross_registries, rc->module_def_index, rc->all_defs, + rc->def_count, imp_keys, imp_vals, imp_count, + pp_rust_shared_registry_get, rc); + cbm_index_mark_done(rel); /* Free the on-demand re-read (no-op when source was retained). */ free_source(lsp_source_owned); /* Contract: cbm_slab_reclaim() requires the thread parser to be diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index c3215f15c..cacfd7639 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -801,6 +801,12 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, free(seq_cache); ctx->result_cache = NULL; } + /* Release the lsp_cross pass's shared registries only now: resolved_calls + * borrowed registry-owned strings that the calls pass read above. */ + if (ctx->seq_cross_arena_live) { + cbm_arena_destroy(&ctx->seq_cross_arena); + ctx->seq_cross_arena_live = false; + } return rc; } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index fbbd32f7b..a3d806558 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -99,6 +99,17 @@ typedef struct { /* Directory subtrees excluded during discovery. Borrowed from pipeline.c. */ char **excluded_dirs; int excluded_count; + + /* Sequential cross-LSP registry arena. The lsp_cross pass builds its + * shared per-language registries here; resolved_calls entries may BORROW + * strings owned by these registries, and the later calls pass still + * reads them — so the arena is OWNED and destroyed by + * run_sequential_pipeline AFTER all passes, never by the lsp_cross pass + * itself (destroying at pass end was a use-after-free in pass_calls). + * Mirrors the parallel path, where cross_lsp_arena outlives the fused + * resolve. */ + CBMArena seq_cross_arena; + bool seq_cross_arena_live; } cbm_pipeline_ctx_t; static inline int cbm_pipeline_relpath_is_excluded(const char *rel_path, char *const *excluded_dirs, diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 8a264f41c..795bb620a 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -7,6 +7,7 @@ */ #include "test_framework.h" #include "cbm.h" +#include /* wide-flat linearity bound (extract_wide_flat_file_is_linear) */ /* ── Helpers ───────────────────────────────────────────────────── */ @@ -2101,12 +2102,12 @@ TEST(python_calls) { } TEST(python_iris_classMethodValue) { - CBMFileResult *r = extract( - "import iris\n" - "iris_obj = iris.cls('%Library.ObjectScript')\n" - "def call_bfs(n):\n" - " return iris_obj.classMethodValue('Graph.KG.TraversalBFS', 'BFSFastJson', n)\n", - CBM_LANG_PYTHON, "t", "store.py"); + CBMFileResult *r = + extract("import iris\n" + "iris_obj = iris.cls('%Library.ObjectScript')\n" + "def call_bfs(n):\n" + " return iris_obj.classMethodValue('Graph.KG.TraversalBFS', 'BFSFastJson', n)\n", + CBM_LANG_PYTHON, "t", "store.py"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_call(r, "Graph.KG.TraversalBFS.BFSFastJson")); @@ -3092,7 +3093,7 @@ TEST(complexity_go_method_receiver_self_recursion) { ASSERT_FALSE(r->has_error); const CBMDefinition *d = find_def(r, "save"); ASSERT_NOT_NULL(d); - ASSERT_TRUE(d->is_recursive); /* s.save() == receiver s → self */ + ASSERT_TRUE(d->is_recursive); /* s.save() == receiver s → self */ ASSERT_FALSE(d->unguarded_recursion); /* guarded by `if n > 0` */ cbm_free_result(r); @@ -3355,15 +3356,14 @@ TEST(walk_defs_no_truncation_over_4096_issue668) { * otherwise file-path-based (cbm_is_test_file), so test fns in a regular .rs * file leak. (#855) */ TEST(extract_rust_test_attr_marks_is_test_issue855) { - CBMFileResult *r = extract( - "pub fn real_fn() {}\n" - "\n" - "#[test]\n" - "fn sync_test() {}\n" - "\n" - "#[tokio::test]\n" - "async fn async_test() {}\n", - CBM_LANG_RUST, "t", "src/lib.rs"); + CBMFileResult *r = extract("pub fn real_fn() {}\n" + "\n" + "#[test]\n" + "fn sync_test() {}\n" + "\n" + "#[tokio::test]\n" + "async fn async_test() {}\n", + CBM_LANG_RUST, "t", "src/lib.rs"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); @@ -3390,10 +3390,65 @@ TEST(extract_rust_test_attr_marks_is_test_issue855) { PASS(); } +/* Reproduce-first (ms-typescript reallyLargeFile.ts, 2026-07-07): a file + * whose root node has hundreds of thousands of FLAT SIBLINGS (580k //// + * comment lines in the 3.5 MB fourslash fixture) hung extraction for over + * 15 minutes: walk_defs pushed children via index-based ts_node_child(i), + * which is O(i) per call in tree-sitter — O(n^2) per wide node (~1.7e11 + * iterator steps on the real file; 100% of stack samples inside + * ts_node_child_iterator_next). The supervisor then killed the silent + * worker as a hang and, via the stale extraction marker, quarantined + * INNOCENT files on every retry. + * + * This fixture is an 80k-sibling flat file: quadratic child access needs + * minutes under ASan; the linear TSTreeCursor collection finishes in + * milliseconds. The 30 s bound has ~100x headroom over the fixed cost — + * RED on index-based child pushes, GREEN on the cursor walk. The def-count + * guard keeps the test honest: extraction must actually process the whole + * breadth, not skip it. */ +TEST(extract_wide_flat_file_is_linear) { + /* Mirror the monster's exact shape: its 580k wide siblings are COMMENT + * nodes, not defs, so this fixture isolates the WALK cost. A def-heavy + * fixture (400k var statements) additionally hits a separate per-def + * sibling-scan cost in extraction (O(defs x siblings), tracked as its own + * finding) and took 647s even with the walk fixed — it guarded the wrong + * thing. Sparse real defs keep the anti-vacuous breadth check. */ + const int n = 400 * 1000; /* comment siblings */ + const size_t cap = (size_t)n * 24 + (size_t)8192; /* "// wide filler 399999\n" = 22 chars */ + char *src = malloc(cap); + ASSERT_NOT_NULL(src); + size_t off = 0; + for (int i = 0; i < n; i++) { + off += (size_t)snprintf(src + off, cap - off, "// wide filler %d\n", i); + if (i % 4000 == 0) { + off += (size_t)snprintf(src + off, cap - off, "var wide_a%d = %d;\n", i, i); + } + } + time_t start = time(NULL); + CBMFileResult *r = + cbm_extract_file(src, (int)off, CBM_LANG_JAVASCRIPT, "proj", "wide.js", 0, NULL, NULL); + long elapsed_s = (long)(time(NULL) - start); + free(src); + ASSERT_NOT_NULL(r); + /* Anti-vacuous guard: the breadth was actually walked. */ + ASSERT_GTE(r->defs.count, 100); + cbm_free_result(r); + if (elapsed_s >= 30) { + char msg[128]; + snprintf(msg, sizeof(msg), + "wide-flat extract took %lds (>=30s bound) — quadratic child access", elapsed_s); + FAIL(msg); + } + PASS(); +} + SUITE(extraction) { /* Initialize extraction library */ cbm_init(); + /* Wide-flat-file linearity (ms-typescript hang) */ + RUN_TEST(extract_wide_flat_file_is_linear); + /* Perl call-graph noise (#459 follow-up) */ RUN_TEST(extract_perl_config_string_not_a_callee); RUN_TEST(extract_perl_builtin_call_is_function_not_method); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index b358cd010..ddda022ba 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -409,9 +409,9 @@ TEST(mcp_get_architecture_aspects_schema_enum_pr560) { ASSERT_TRUE(yyjson_is_arr(enum_arr)); /* The enum must be exactly the valid-token set — no more, no less. */ - static const char *expected[] = {"all", "overview", "structure", "dependencies", - "routes", "languages", "packages", "entry_points", - "hotspots", "boundaries", "layers", "file_tree", + static const char *expected[] = {"all", "overview", "structure", "dependencies", + "routes", "languages", "packages", "entry_points", + "hotspots", "boundaries", "layers", "file_tree", "clusters"}; size_t expected_count = sizeof(expected) / sizeof(expected[0]); ASSERT_EQ(yyjson_arr_size(enum_arr), expected_count); @@ -1071,18 +1071,34 @@ TEST(tool_trace_call_path_distinct_defs_not_over_unioned) { cbm_mcp_server_set_project(srv, proj); cbm_store_upsert_project(st, proj, "/tmp/ou"); /* two unrelated real definitions of "dupreal", DIFFERENT body spans */ - cbm_node_t da = {.project = proj, .label = "Function", .name = "dupreal", - .qualified_name = "ou-proj.a.dupreal", .file_path = "a.c", - .start_line = 10, .end_line = 20}; /* span 10 */ - cbm_node_t db = {.project = proj, .label = "Function", .name = "dupreal", - .qualified_name = "ou-proj.b.dupreal", .file_path = "b.c", - .start_line = 10, .end_line = 40}; /* span 30 (no tie) */ - cbm_node_t ca = {.project = proj, .label = "Function", .name = "callerA", - .qualified_name = "ou-proj.a.callerA", .file_path = "a.c", - .start_line = 30, .end_line = 40}; - cbm_node_t cb = {.project = proj, .label = "Function", .name = "callerB", - .qualified_name = "ou-proj.b.callerB", .file_path = "b.c", - .start_line = 50, .end_line = 60}; + cbm_node_t da = {.project = proj, + .label = "Function", + .name = "dupreal", + .qualified_name = "ou-proj.a.dupreal", + .file_path = "a.c", + .start_line = 10, + .end_line = 20}; /* span 10 */ + cbm_node_t db = {.project = proj, + .label = "Function", + .name = "dupreal", + .qualified_name = "ou-proj.b.dupreal", + .file_path = "b.c", + .start_line = 10, + .end_line = 40}; /* span 30 (no tie) */ + cbm_node_t ca = {.project = proj, + .label = "Function", + .name = "callerA", + .qualified_name = "ou-proj.a.callerA", + .file_path = "a.c", + .start_line = 30, + .end_line = 40}; + cbm_node_t cb = {.project = proj, + .label = "Function", + .name = "callerB", + .qualified_name = "ou-proj.b.callerB", + .file_path = "b.c", + .start_line = 50, + .end_line = 60}; int64_t id_da = cbm_store_upsert_node(st, &da); int64_t id_db = cbm_store_upsert_node(st, &db); int64_t id_ca = cbm_store_upsert_node(st, &ca); @@ -1097,9 +1113,10 @@ TEST(tool_trace_call_path_distinct_defs_not_over_unioned) { cbm_store_insert_edge(st, &eb); char *resp = cbm_mcp_server_handle( - srv, "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\"," - "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":\"dupreal\"," - "\"project\":\"ou-proj\",\"direction\":\"inbound\"}}}"); + srv, + "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":\"dupreal\"," + "\"project\":\"ou-proj\",\"direction\":\"inbound\"}}}"); ASSERT_NOT_NULL(resp); char *inner = extract_text_content(resp); ASSERT_NOT_NULL(inner); @@ -1123,18 +1140,34 @@ TEST(tool_trace_call_path_dts_stub_unions_with_impl) { const char *proj = "dts-proj"; cbm_mcp_server_set_project(srv, proj); cbm_store_upsert_project(st, proj, "/tmp/dts"); - cbm_node_t impl = {.project = proj, .label = "Function", .name = "sym546", - .qualified_name = "dts-proj.impl.sym546", .file_path = "src/sym.ts", - .start_line = 10, .end_line = 30}; /* real body */ - cbm_node_t stub = {.project = proj, .label = "Function", .name = "sym546", - .qualified_name = "dts-proj.stub.sym546", .file_path = "types/sym.d.ts", - .start_line = 5, .end_line = 5}; /* body-less ambient decl */ - cbm_node_t crel = {.project = proj, .label = "Function", .name = "callerRel", - .qualified_name = "dts-proj.callerRel", .file_path = "src/rel.ts", - .start_line = 1, .end_line = 8}; - cbm_node_t cali = {.project = proj, .label = "Function", .name = "callerAlias", - .qualified_name = "dts-proj.callerAlias", .file_path = "src/ali.ts", - .start_line = 1, .end_line = 8}; + cbm_node_t impl = {.project = proj, + .label = "Function", + .name = "sym546", + .qualified_name = "dts-proj.impl.sym546", + .file_path = "src/sym.ts", + .start_line = 10, + .end_line = 30}; /* real body */ + cbm_node_t stub = {.project = proj, + .label = "Function", + .name = "sym546", + .qualified_name = "dts-proj.stub.sym546", + .file_path = "types/sym.d.ts", + .start_line = 5, + .end_line = 5}; /* body-less ambient decl */ + cbm_node_t crel = {.project = proj, + .label = "Function", + .name = "callerRel", + .qualified_name = "dts-proj.callerRel", + .file_path = "src/rel.ts", + .start_line = 1, + .end_line = 8}; + cbm_node_t cali = {.project = proj, + .label = "Function", + .name = "callerAlias", + .qualified_name = "dts-proj.callerAlias", + .file_path = "src/ali.ts", + .start_line = 1, + .end_line = 8}; int64_t id_impl = cbm_store_upsert_node(st, &impl); int64_t id_stub = cbm_store_upsert_node(st, &stub); int64_t id_crel = cbm_store_upsert_node(st, &crel); @@ -1726,7 +1759,8 @@ TEST(search_code_scoped_path_with_spaces_issue687) { * a project with two indexed files that both contain the search pattern — * src/handler.go (inside the filter) and vendor/other.go (outside it). */ static cbm_mcp_server_t *setup_prefilter_server(char *tmp, size_t tmp_sz, char *src_path, - size_t src_sz, char *vendor_path, size_t vendor_sz) { + size_t src_sz, char *vendor_path, + size_t vendor_sz) { snprintf(tmp, tmp_sz, "/tmp/cbm_srch_pref_XXXXXX"); if (!cbm_mkdtemp(tmp)) { return NULL; @@ -2268,9 +2302,8 @@ TEST(tool_manage_adr_not_found_rich_error) { cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); ASSERT_NOT_NULL(srv); - char *resp = - cbm_mcp_handle_tool(srv, "manage_adr", - "{\"project\":\"cbm-no-such-project-zzz\",\"mode\":\"get\"}"); + char *resp = cbm_mcp_handle_tool(srv, "manage_adr", + "{\"project\":\"cbm-no-such-project-zzz\",\"mode\":\"get\"}"); ASSERT_NOT_NULL(resp); ASSERT_NOT_NULL(strstr(resp, "or not indexed")); ASSERT_NOT_NULL(strstr(resp, "hint")); @@ -4216,8 +4249,7 @@ TEST(index_supervisor_gate_requires_marked_host_issue845) { if (signalled) { printf(" child killed by signal %d (alarm => recursive spawn chain hang)\n", sig); } else if (code != IDX845_OK) { - printf(" child exit code %d (41=worker spawned, 42=no result, 43=not indexed)\n", - code); + printf(" child exit code %d (41=worker spawned, 42=no result, 43=not indexed)\n", code); } ASSERT_FALSE(signalled); ASSERT_EQ(code, IDX845_OK); @@ -4380,6 +4412,171 @@ TEST(index_bg_paths_route_through_supervisor_issue832) { #endif } +/* ══════════════════════════════════════════════════════════════════ + * Parallel-only crash recovery (ms-typescript cascade fix) + * ══════════════════════════════════════════════════════════════════ */ + +/* The old recovery loop re-ran the worker SINGLE-THREADED to keep one exact + * crash marker. At scale that fell into the sequential crawl, was killed as + * a hang mid-pass, and the stale marker quarantined FOUR innocent + * ms-typescript fixtures, one 15-minute retry at a time. The reworked loop + * re-runs PARALLEL with a marker journal; a file is quarantined only when + * it is in-flight across two consecutive failed runs. + * + * This guard proves the CONTRACT: with an injected crasher among good + * files, the supervised index must (a) never spawn a single-threaded worker + * (cbm_index_supervisor_spawn_st_count stays 0 — RED on the old loop), + * (b) quarantine exactly the crasher, (c) leave the innocents indexed and + * NOT quarantined. */ +enum { + IDXPAR_OK = 0, + IDXPAR_ST_SPAWN = 61, /* single-threaded recovery spawn happened (RED) */ + IDXPAR_NULL_RESP = 62, /* supervised entry degraded to NULL */ + IDXPAR_NOT_INDEXED = 63, /* response lacks status indexed */ + IDXPAR_NO_QUARANTINE = 64, /* crasher missing from skipped[] */ + IDXPAR_INNOCENT_HIT = 65, /* a good file was quarantined/skipped */ + IDXPAR_GOOD_MISSING = 66, /* good file's Function absent from the store */ +}; + +#ifndef _WIN32 +static int idxpar_recovery_check(const char *repo_dir) { + cbm_index_supervisor_mark_host(); + cbm_unsetenv("CBM_INDEX_SUPERVISOR"); + /* Rounds needed: fail+record, fail+quarantine, clean. Generous cap. */ + cbm_setenv("CBM_INDEX_MAX_RESTARTS", "5", 1); + cbm_setenv("CBM_INDEX_WORKER_TIMEOUT_S", "30", 1); + cbm_setenv("CBM_TEST_CRASH_ON", "idxpar_crasher", 1); + + int st_before = cbm_index_supervisor_spawn_st_count(); + char *resp = cbm_mcp_index_run_supervised_path(repo_dir); + int st_after = cbm_index_supervisor_spawn_st_count(); + cbm_unsetenv("CBM_TEST_CRASH_ON"); + + if (st_after != st_before) { + free(resp); + return IDXPAR_ST_SPAWN; /* discriminating assertion: RED on the old loop */ + } + if (!resp) { + return IDXPAR_NULL_RESP; + } + bool indexed = response_contains_json_fragment(resp, "\"status\":\"indexed\""); + bool crasher_skipped = strstr(resp, "idxpar_crasher.py") != NULL; + bool innocent_hit = + strstr(resp, "idxpar_good_a.py") != NULL || strstr(resp, "idxpar_good_b.py") != NULL; + free(resp); + if (!indexed) { + return IDXPAR_NOT_INDEXED; + } + if (!crasher_skipped) { + return IDXPAR_NO_QUARANTINE; + } + if (innocent_hit) { + return IDXPAR_INNOCENT_HIT; + } + + /* Store proof: an innocent's Function node exists. */ + char *project = cbm_project_name_from_path(repo_dir); + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + int code = IDXPAR_OK; + if (srv && project) { + char q[512]; + snprintf(q, sizeof(q), + "{\"project\":\"%s\",\"name_pattern\":\"idxpar_good_fn\",\"label\":\"Function\"}", + project); + char *sr = cbm_mcp_handle_tool(srv, "search_graph", q); + if (!sr || !strstr(sr, "idxpar_good_fn")) { + code = IDXPAR_GOOD_MISSING; + } + free(sr); + } + if (srv) { + cbm_mcp_server_free(srv); + } + free(project); + return code; +} +#endif /* !_WIN32 */ + +TEST(index_recovery_parallel_quarantines_crasher) { +#ifdef _WIN32 + SKIP("POSIX fork harness (supervised recovery covered by smoke on Windows)"); +#else + char tmp_dir[CBM_SZ_256]; + snprintf(tmp_dir, sizeof(tmp_dir), "/tmp/cbm-idxpar-XXXXXX"); + if (!cbm_mkdtemp(tmp_dir)) { + FAIL("mkdtemp failed"); + } + char cache[CBM_SZ_256]; + snprintf(cache, sizeof(cache), "/tmp/cbm-idxpar-cache-XXXXXX"); + if (!cbm_mkdtemp(cache)) { + FAIL("mkdtemp cache failed"); + } + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? cbm_strdup(saved_cache) : NULL; + cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char p1[CBM_SZ_512]; + char p2[CBM_SZ_512]; + char pc[CBM_SZ_512]; + snprintf(p1, sizeof(p1), "%s/idxpar_good_a.py", tmp_dir); + snprintf(p2, sizeof(p2), "%s/idxpar_good_b.py", tmp_dir); + snprintf(pc, sizeof(pc), "%s/idxpar_crasher.py", tmp_dir); + FILE *f = fopen(p1, "w"); + ASSERT_NOT_NULL(f); + fputs("def idxpar_good_fn():\n return 'ok'\n", f); + fclose(f); + f = fopen(p2, "w"); + ASSERT_NOT_NULL(f); + fputs("def idxpar_good_fn_b():\n return 'ok'\n", f); + fclose(f); + f = fopen(pc, "w"); + ASSERT_NOT_NULL(f); + fputs("def idxpar_crash_fn():\n return 'boom'\n", f); + fclose(f); + + int code = -1; + bool signalled = false; + int sig = 0; + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + alarm(120); /* generous: three supervised rounds + clean run */ + _exit(idxpar_recovery_check(tmp_dir)); + } + ASSERT_TRUE(pid > 0); + int status = 0; + (void)waitpid(pid, &status, 0); + if (WIFEXITED(status)) { + code = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + signalled = true; + sig = WTERMSIG(status); + } + + char *project = cbm_project_name_from_path(tmp_dir); + cleanup_project_db(cache, project); + free(project); + restore_cache_dir(saved_cache_copy); + free(saved_cache_copy); + remove(p1); + remove(p2); + remove(pc); + cbm_rmdir(cache); + cbm_rmdir(tmp_dir); + + if (signalled) { + printf(" child killed by signal %d (alarm => recovery loop hang)\n", sig); + } else if (code != IDXPAR_OK) { + printf(" child exit code %d (61=ST spawn/RED, 62=null resp, 63=not indexed, " + "64=no quarantine, 65=innocent hit, 66=good missing)\n", + code); + } + ASSERT_FALSE(signalled); + ASSERT_EQ(code, IDXPAR_OK); + PASS(); +#endif +} + /* ══════════════════════════════════════════════════════════════════ * AUTO_WATCH GATE (distilled from PR #625) * @@ -4801,6 +4998,7 @@ SUITE(mcp) { RUN_TEST(tool_index_repository_dot_uses_absolute_project_key_and_preserves_adr); RUN_TEST(index_supervisor_gate_requires_marked_host_issue845); RUN_TEST(index_bg_paths_route_through_supervisor_issue832); + RUN_TEST(index_recovery_parallel_quarantines_crasher); RUN_TEST(tool_manage_adr_not_found_rich_error); RUN_TEST(tool_manage_adr_get_accepts_abs_path); RUN_TEST(tool_manage_adr_get_accepts_symlink_path); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index fc35928a7..1b91fb227 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -6621,6 +6621,78 @@ TEST(pipeline_backpressure_futile_nap_disengages) { PASS(); } +/* TS cross-registry test hooks (ts_lsp.c) — extern to avoid pulling the + * tree-sitter-typed ts_lsp.h into this store-level test. */ +extern long cbm_ts_full_registry_builds(void); +extern void cbm_ts_full_registry_builds_reset(void); + +/* Reproduce-first (ms-typescript finding, 2026-07-07): the SEQUENTIAL + * cross-LSP driver must resolve TS files through the SHARED prebuilt + * registry, never a full per-file build. cbm_run_ts_lsp_cross registers + * stdlib + EVERY cross-file def + finalizes once PER FILE — O(files x defs). + * On an 81k-file TS corpus that ground one core for hours (74% of stack + * samples inside build_qn_index), and when the supervisor's quiet-timeout + * killed the crawl mid-pass, the stale extraction marker blamed innocent + * files, quarantining four of them one 15-minute retry at a time. + * + * The fixture stays UNDER MIN_FILES_FOR_PARALLEL (50) so the pipeline + * routes through the sequential driver — the path that lacked the + * shared-registry prepare. + * RED on the unfixed driver: full builds == TS file count (40). + * GREEN: full builds == 0 AND the cross-file TS call still resolves + * (quality guard — the shared path must not lose the edge). */ +TEST(pipeline_seq_ts_cross_uses_shared_registry) { + snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/cbm_test_XXXXXX"); + if (!cbm_mkdtemp(g_tmpdir)) { + FAIL("failed to create temp dir"); + } + char path[512]; + snprintf(path, sizeof(path), "%s/shared.ts", g_tmpdir); + FILE *f = fopen(path, "w"); + if (!f) { + FAIL("failed to create fixture file"); + } + fputs("export function sharedHelper(): number {\n return 42;\n}\n", f); + fclose(f); + for (int i = 0; i < 39; i++) { + snprintf(path, sizeof(path), "%s/caller%02d.ts", g_tmpdir, i); + f = fopen(path, "w"); + if (!f) { + FAIL("failed to create fixture file"); + } + fprintf(f, + "import { sharedHelper } from \"./shared\";\n" + "export function caller%02d(): number {\n return sharedHelper();\n}\n", + i); + fclose(f); + } + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/seqts.db", g_tmpdir); + cbm_ts_full_registry_builds_reset(); + cbm_pipeline_t *p = cbm_pipeline_new(g_tmpdir, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + long builds = cbm_ts_full_registry_builds(); + + /* Quality guard FIRST: the cross-file call must resolve either way. */ + cbm_store_t *s = cbm_store_open_path(db_path); + bool linked = false; + if (s) { + linked = + cross_file_call_exists(s, cbm_pipeline_project_name(p), "caller00", "sharedHelper"); + cbm_store_close(s); + } + cbm_pipeline_free(p); + teardown_test_repo(); + + ASSERT_EQ(rc, 0); + ASSERT_TRUE(linked); + /* The point: ZERO full per-file registry builds on the sequential path. */ + ASSERT_EQ(builds, 0); + PASS(); +} + SUITE(pipeline) { /* Index lock */ RUN_TEST(pipeline_lock_try_acquire); @@ -6636,6 +6708,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_run_null); /* Extraction back-pressure */ RUN_TEST(pipeline_backpressure_futile_nap_disengages); + /* Sequential cross-LSP shared registry (ms-typescript quadratic) */ + RUN_TEST(pipeline_seq_ts_cross_uses_shared_registry); /* File persistence */ RUN_TEST(store_file_persistence); RUN_TEST(store_bulk_persistence);