diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a0daf515..c1b557fc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.63 (2026-09-16) +- Feature: query seeding now considers a node's `body` attribute, not just its label, source path, and the curated `rationale` attribute. A non-extraction emitter (a Confluence page, a Jira issue) has nowhere else to put its content — storing it on an arbitrary attribute bought nothing on its own, since the node could never become a seed and was therefore never traversed no matter how relevant. `body` is the documented convention, matched at its own tier well below `rationale` since raw scraped prose is noisier than curated why-text (#3313, thanks @vongohren). + - Feature: Elixir `alias`/`import`/`require`/`use` targets now resolve onto the module's `defmodule` node across files, so the internal module dependency graph is no longer dropped as dangling. Only top-level modules are indexed (a nested `defmodule`, labeled with its bare inner name, cannot capture an unrelated `use ` from another file), and a same-file reference is left unresolved so it cannot clobber the structural `contains` edge (#3603, thanks @ayushcodes10). - Feature: a Rust `self.method()` call now resolves to a method defined on the same type in another file (the common split-`impl`-block layout), pooling methods across every `impl` of one type and refusing to link when two unrelated types share a bare name (#3602, thanks @ayushcodes10). - Feature: a Ruby member call `obj.foo` on a known-type receiver now resolves to a method `foo` inherited from a superclass, including across files, using the same conservative promotion as the implicit-self resolver — a single owning class, matching method kind, and one unambiguous ancestry chain, or it stays dangling (#3585, thanks @oleksii-tumanov). diff --git a/graphify/serve.py b/graphify/serve.py index 9301a4a32..946741812 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -312,6 +312,17 @@ def _query_terms(question: str) -> list[str]: # toward term coverage, so a long rationale adds recall without winning back # an exact-label tier it did not earn. _RATIONALE_MATCH_BONUS = 0.75 +# A node emitted by something other than graphify's own extraction (a +# Confluence page, a Jira issue) has nowhere to put its free text except an +# arbitrary attribute, and nothing considers those for seeding: the content +# is in the graph, the ranker is fine, the node is just never visited (#3313). +# `body` is the documented convention such an emitter can use. Weighted well +# below the curated `rationale` tier (which the extraction spec itself +# writes) rather than reusing it, since raw scraped/pasted text is noisier +# and an unweighted body match was measured to make retrieval WORSE - a few +# hundred characters of prose matches many query terms, so a page that +# merely mentions the words ties with the page that is actually about them. +_BODY_MATCH_BONUS = 0.35 def _compute_idf(G: nx.Graph, terms: list[str]) -> dict[str, float]: @@ -360,6 +371,21 @@ def _node_rationale_text(data: dict) -> str: return _strip_diacritics(str(raw)).lower() +def _node_body_text(data: dict) -> str: + """The node's `body` attribute normalized like a label (#3313). + + The documented convention for free text a non-extraction emitter (a + Confluence page, a Jira issue) has nowhere else to put. Same shape as + `_node_rationale_text`: a list is joined, missing or empty is "". + """ + raw = data.get("body") + if not raw: + return "" + if isinstance(raw, (list, tuple)): + raw = " ".join(str(part) for part in raw if part) + return _strip_diacritics(str(raw)).lower() + + def _node_search_text(data: dict, nid: str) -> str: """Concatenate every field _score_nodes / _find_node match a query against, so one trigram index over this text is a complete candidate generator for both. @@ -367,6 +393,8 @@ def _node_search_text(data: dict, nid: str) -> str: - `rationale` (normalized via `_node_rationale_text`) feeds _score_nodes' rationale tier (#2293); appended last, and only when present, so every other field position is unchanged. + - `body` (normalized via `_node_body_text`) feeds _score_nodes' body tier + (#3313); same append-only-when-present treatment as rationale. - `norm_label` and `source_file` feed _score_nodes' per-term substring tiers. - `label_tokens` (the space-joined token form) feeds _find_node's @@ -401,6 +429,9 @@ class 0 and therefore survive the combining-character filter. The field is rationale = _node_rationale_text(data) if rationale: fields += (rationale,) + body = _node_body_text(data) + if body: + fields += (body,) return "\x00".join(fields) @@ -535,6 +566,18 @@ def _score_query( trigram candidate set (needles `norm_terms + [joined]`) is a superset of each per-token `[t]` candidate set, so iterating combined candidates discovers every non-zero singleton-score node for every term. + + `label`, `source_file`, `rationale` (#2293), and `body` (#3313) are the + only attributes matched against. An emitter with nowhere else to put free + text (a Confluence page, a Jira issue) can use `body`; anything stored + under another attribute name is still invisible to seeding, so the node + can never become a seed no matter how relevant its content is - the data + is in the graph and the ranker is fine, the node is just never visited, + which makes this easy to hit and hard to diagnose. `body` is weighted + well below `rationale` rather than sharing its tier: unweighted body + matching was measured to make retrieval worse, since a few hundred + characters of prose matches many query terms and a page that merely + mentions the words ties with the page that is actually about them. """ scored: list[tuple[float, str]] = [] # Dedupe tokens, order-preserving (as _pick_seeds already does): a repeated @@ -578,6 +621,7 @@ def _score_query( label_tokens = " ".join(_search_tokens(data.get("label") or "")) source = (data.get("source_file") or "").lower() rationale = _node_rationale_text(data) + body = _node_body_text(data) # `nid_lower` is needed both by the full-query tier (`if joined`) and by # the per-token singleton tier (joined-singlet exact-match check). When # neither runs (`joined` empty AND not collecting seeds) skip the call; @@ -643,6 +687,15 @@ def _score_query( if rationale and t in rationale: rationale_value = _RATIONALE_MATCH_BONUS * w score += rationale_value + # Body tier (#3313): recall for content a non-extraction emitter + # stored on the node with nowhere else to put it. Same shape as + # the rationale tier - adds to the score, not to `matched` - at a + # lower weight, since unlike a curated rationale, raw body text is + # noisy prose where many terms coincidentally appear. + body_value = 0.0 + if body and t in body: + body_value = _BODY_MATCH_BONUS * w + score += body_value tiered += tier_value if collect_per_term_seeds and best_by_term is not None: # Singleton score for [t] on this node, mirroring @@ -661,7 +714,7 @@ def _score_query( singleton = _PREFIX_MATCH_BONUS * 10 * w else: singleton = 0.0 - singleton += tier_value + substr_value + source_value + rationale_value + singleton += tier_value + substr_value + source_value + rationale_value + body_value if singleton > 0: # Tie-break key mirrors the legacy sort+max(degree): # (-singleton, -degree, label_len, nid) — the minimum diff --git a/tests/test_serve.py b/tests/test_serve.py index 6e3f5d2d9..c8830e45b 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -1827,3 +1827,66 @@ def test_query_graph_text_seeds_the_node_whose_rationale_answers_a_why_question( ) header = text.split("\n\n", 1)[0] assert "FAB visibility rule" in header, header + + +# --- body attribute scoring (#3313) --- + +def test_score_nodes_reads_body_when_label_does_not_match(): + """The #3313 shape: a non-extraction emitter (a wiki page, a ticket) puts + its content on `body` since there is nowhere else to put it, and the + label alone shares no query term.""" + G = nx.Graph() + G.add_node( + "page", label="Masterlist/Certificate controls", source_file="confluence/page1.md", + body="The CSCA Master List defines certificate authority trust anchors.", + ) + G.add_node("ticket", label="CSCA rollout ticket", source_file="jira/TICKET-1.md") + G.add_node("other", label="Unrelated page", source_file="confluence/other.md", + body="Nothing to do with certificates at all.") + assert "page" in [nid for _, nid in _score_nodes(G, ["CSCA", "Master", "List"])] + + +def test_score_nodes_body_tier_sits_below_rationale_and_source(): + """Body is the noisiest signal (raw prose, not curated why-text), so it + must rank below both the rationale and source-path tiers, not above.""" + G = nx.Graph() + G.add_node("lbl", label="popover-anchor", source_file="ui/a.py") + G.add_node("rat", label="Sheet drag", source_file="ui/b.py", rationale="starts only once the popover is closed") + G.add_node("src", label="Thing", source_file="ui/popover/thing.py") + G.add_node("bod", label="Notes", source_file="ui/c.py", body="mentions a popover somewhere in the middle") + assert [nid for _, nid in _score_nodes(G, ["popover"])] == ["lbl", "rat", "src", "bod"] + + +def test_score_nodes_body_does_not_count_toward_term_coverage(): + """Like rationale and source, a body hit adds recall but must not restore + the coverage-scaled exact tier.""" + from graphify.serve import _EXACT_MATCH_BONUS + G = nx.Graph() + G.add_node("a", label="cache", source_file="x.py", body="invalidated on every write") + G.add_node("b", label="cache", source_file="y.py") + score = {nid: s for s, nid in _score_nodes(G, ["cache", "invalidated"])} + assert score["a"] > score["b"] + assert score["a"] - score["b"] < _EXACT_MATCH_BONUS * 0.5 + + +def test_score_nodes_tolerates_list_valued_body(): + G = nx.Graph() + G.add_node("n", label="X", source_file="x.py", body=["first paragraph", "popover second"]) + assert [nid for _, nid in _score_nodes(G, ["popover"])] == ["n"] + + +def test_node_search_text_includes_body_so_trigram_prefilter_stays_complete(): + parts = _node_search_text( + {"label": "Foo", "source_file": "a.py", "body": "Describes the Popover in detail"}, "foo" + ).split("\x00") + assert "describes the popover in detail" in parts + # No body: field layout unchanged (the #2467 positions still hold). + assert len(_node_search_text({"label": "Foo", "source_file": "a.py"}, "foo").split("\x00")) == 5 + + +def test_node_search_text_includes_both_rationale_and_body_when_both_present(): + parts = _node_search_text( + {"label": "Foo", "source_file": "a.py", "rationale": "why text", "body": "body text"}, "foo" + ).split("\x00") + assert "why text" in parts + assert "body text" in parts