Merge pull request #5 from Modsofthenation/cursor/review-accuracy-ccb4

Score tests on the impact path and prefer generated OpenAPI clients

Author: cursor[bot]

src/loadpath/architecture/rules.py

@@ -296,19 +296,40 @@ def _task_idempotency(store: GraphStore, changed_ids: set[str] | None) -> list[F
     return out
 
 
+REL_FIELD_TYPES = {
+    "ForeignKey",
+    "OneToOneField",
+    "ManyToManyField",
+    "GenericForeignKey",
+    "GenericRelation",
+}
+
+
 def _nplusone(store: GraphStore) -> list[Finding]:
     out: list[Finding] = []
+    fields = list(store.nodes([NodeType.FIELD]))
+    fields_by_name: dict[str, list[dict]] = {}
+    for field in fields:
+        fields_by_name.setdefault(field["name"], []).append(field)
     for node in store.nodes():
         hits = (node.get("extra") or {}).get("nplusone") or []
+        owner_app = (node.get("extra") or {}).get("app")
         for hit in hits:
-            accessed = ", ".join(hit.get("accessed") or []) or "related fields"
+            accessed = list(hit.get("accessed") or [])
+            related, conf = _related_accesses(accessed, fields_by_name, owner_app)
+            if not related:
+                continue
+            hit = dict(hit)
+            hit["accessed"] = related
+            hit["confidence"] = conf
+            accessed_s = ", ".join(related)
             fix = hit.get("suggested_fix") or ".select_related()"
             out.append(
                 Finding(
                     rule="queryset_nplusone",
                     severity=RuleSeverity.WARNING,
                     message=(
-                        f"{node['name']} loops `{hit.get('loop_var')}` over a queryset and touches {accessed} "
+                        f"{node['name']} loops `{hit.get('loop_var')}` over a queryset and touches {accessed_s} "
                         f"without {fix} ({node.get('file_path')}:{hit.get('line')})"
                     ),
                     node_id=node["id"],
@@ -319,6 +340,33 @@ def _nplusone(store: GraphStore) -> list[Finding]:
     return out
 
 
+def _related_accesses(
+    accessed: list[str], fields_by_name: dict[str, list[dict]], owner_app: str | None
+) -> tuple[list[str], str]:
+    related: list[str] = []
+    unknown = False
+    for name in accessed:
+        matches = fields_by_name.get(name) or []
+        if owner_app:
+            scoped = [f for f in matches if (f.get("extra") or {}).get("app") == owner_app]
+            if scoped:
+                matches = scoped
+        if not matches:
+            related.append(name)
+            unknown = True
+            continue
+        if any(_is_relation(f) for f in matches):
+            related.append(name)
+    return related, ("medium" if unknown else "high")
+
+
+def _is_relation(field: dict) -> bool:
+    extra = field.get("extra") or {}
+    if extra.get("relation"):
+        return True
+    return extra.get("field_type") in REL_FIELD_TYPES
+
+
 def _missing_index(store: GraphStore) -> list[Finding]:
     out: list[Finding] = []
     fields_by_name: dict[str, list[dict]] = {}

src/loadpath/extractors/django_boot.py

@@ -2,15 +2,136 @@
 
 from __future__ import annotations
 
+import json
 import os
+import subprocess
 import sys
 from pathlib import Path
 
 from loadpath.config import LoadpathConfig
 from loadpath.types import Edge, EdgeType, ExtractedGraph, Node, NodeType, node_id
 
+BOOT_JSON_MARKER = "__LOADPATH_BOOT_JSON__"
+
 
 def try_boot_models(repo_root: Path, config: LoadpathConfig) -> ExtractedGraph:
+    """Boot Django in a subprocess so django.setup() is not process-global."""
+    if os.environ.get("LOADPATH_BOOT_INPROCESS") == "1":
+        return _boot_inprocess(repo_root, config)
+    return _boot_subprocess(repo_root, config)
+
+
+def _boot_subprocess(repo_root: Path, config: LoadpathConfig) -> ExtractedGraph:
+    src_root = Path(__file__).resolve().parents[2]
+    env = os.environ.copy()
+    env["LOADPATH_BOOT_INPROCESS"] = "1"
+    env["PYTHONPATH"] = str(src_root) + os.pathsep + env.get("PYTHONPATH", "")
+    payload = json.dumps(
+        {
+            "repo_root": str(repo_root.resolve()),
+            "django_root": config.django_root,
+        }
+    )
+    code = (
+        "import io,json,sys\n"
+        "from contextlib import redirect_stdout\n"
+        "from pathlib import Path\n"
+        "from loadpath.config import load_config\n"
+        "from loadpath.extractors.django_boot import _boot_inprocess\n"
+        "meta=json.loads(sys.argv[1])\n"
+        "root=Path(meta['repo_root'])\n"
+        "cfg=load_config(root)\n"
+        "cfg.django_root=meta['django_root']\n"
+        "cfg.boot_django=True\n"
+        "buf=io.StringIO()\n"
+        "with redirect_stdout(buf):\n"
+        "    g=_boot_inprocess(root,cfg)\n"
+        "print(" + repr(BOOT_JSON_MARKER) + " + json.dumps("
+        "{'nodes':[n.to_row() for n in g.nodes],"
+        "'edges':[e.to_row() for e in g.edges],'residuals':g.residuals}))\n"
+    )
+    try:
+        proc = subprocess.run(
+            [sys.executable, "-c", code, payload],
+            capture_output=True,
+            text=True,
+            timeout=45,
+            env=env,
+            cwd=str(repo_root),
+        )
+    except subprocess.TimeoutExpired:
+        graph = ExtractedGraph()
+        graph.residuals.append("django.setup() skipped: boot subprocess timed out")
+        return graph
+    if proc.returncode != 0:
+        graph = ExtractedGraph()
+        err = (proc.stderr or proc.stdout or "unknown error").strip().splitlines()
+        tail = err[-1] if err else "unknown error"
+        graph.residuals.append(f"django.setup() skipped: {tail}")
+        return graph
+    data = _parse_boot_payload(proc.stdout)
+    if data is None:
+        graph = ExtractedGraph()
+        graph.residuals.append("django.setup() skipped: boot subprocess returned invalid JSON")
+        return graph
+    return _graph_from_boot_data(data)
+
+
+def _graph_from_boot_data(data: dict) -> ExtractedGraph:
+    graph = ExtractedGraph()
+    graph.residuals.extend(data.get("residuals") or [])
+    try:
+        for row in data.get("nodes") or []:
+            extra = row.get("extra") or {}
+            if isinstance(extra, str):
+                extra = json.loads(extra)
+            graph.nodes.append(
+                Node(
+                    id=row["id"],
+                    type=NodeType(row["type"]),
+                    name=row["name"],
+                    qualified_name=row["qualified_name"],
+                    file_path=row.get("file_path"),
+                    start_line=row.get("start_line"),
+                    end_line=row.get("end_line"),
+                    context=row.get("context"),
+                    extra=extra if isinstance(extra, dict) else {},
+                )
+            )
+        for row in data.get("edges") or []:
+            extra = row.get("extra") or {}
+            if isinstance(extra, str):
+                extra = json.loads(extra)
+            graph.edges.append(
+                Edge(
+                    src=row["src"],
+                    dst=row["dst"],
+                    type=EdgeType(row["type"]),
+                    confidence=float(row.get("confidence") or 1),
+                    extra=extra if isinstance(extra, dict) else {},
+                )
+            )
+    except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
+        graph = ExtractedGraph()
+        graph.residuals.append(f"django.setup() skipped: boot payload malformed ({exc})")
+    return graph
+
+
+def _parse_boot_payload(stdout: str | None) -> dict | None:
+    text = stdout or ""
+    idx = text.rfind(BOOT_JSON_MARKER)
+    blob = text[idx + len(BOOT_JSON_MARKER) :] if idx >= 0 else text
+    blob = blob.strip().splitlines()[0] if blob.strip() else ""
+    if not blob:
+        return None
+    try:
+        data = json.loads(blob)
+    except json.JSONDecodeError:
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def _boot_inprocess(repo_root: Path, config: LoadpathConfig) -> ExtractedGraph:
     graph = ExtractedGraph()
     settings_mod = _discover_settings_module(repo_root, config.django_root)
     if not settings_mod:

src/loadpath/extractors/react.py

@@ -236,12 +236,22 @@ def edge(src: str, dst: str, etype: EdgeType, confidence: float = 1.0, extra: di
             continue
         line = source[: m.start()].count("\n") + 1
         norm = normalize_url_template(url)
+        generated_file = "/generated/" in f"/{rel}/" or "openapi" in Path(rel).stem.lower()
+        qname = f"client:{rel}:{norm}"
+        if any(n.qualified_name == qname for n in graph.nodes):
+            continue
         client = add(
             NodeType.API_CLIENT,
             norm,
-            f"client:{norm}",
+            qname,
             line,
-            {"raw": url, "inferred": True, "feature": feature, "file": rel},
+            {
+                "raw": url,
+                "inferred": not generated_file,
+                "generated": generated_file,
+                "feature": feature,
+                "file": rel,
+            },
         )
         for owner in hooks or components:
             edge(owner.id, client.id, EdgeType.CALLS)
@@ -253,14 +263,22 @@ def edge(src: str, dst: str, etype: EdgeType, confidence: float = 1.0, extra: di
         url = m.group(1)
         line = source[: m.start()].count("\n") + 1
         norm = normalize_url_template(url)
-        if any(n.qualified_name == f"client:{norm}" for n in graph.nodes):
+        generated_file = "/generated/" in f"/{rel}/" or "openapi" in Path(rel).stem.lower()
+        qname = f"client:{rel}:{norm}"
+        if any(n.qualified_name == qname for n in graph.nodes):
             continue
         add(
             NodeType.API_CLIENT,
             norm,
-            f"client:{norm}",
+            qname,
             line,
-            {"raw": url, "inferred": True, "feature": feature, "file": rel},
+            {
+                "raw": url,
+                "inferred": not generated_file,
+                "generated": generated_file,
+                "feature": feature,
+                "file": rel,
+            },
         )
 
     for m in ROUTE_JSX_RE.finditer(source):

src/loadpath/index.py

@@ -14,6 +14,8 @@
 
 PY_SKIP = {"migrations"}  # still extract migrations, just not skip
 INDEX_EXTENSIONS = {".py", ".ts", ".tsx", ".js", ".jsx"}
+# Bump when extractor/stitch node identity changes so incremental indexes rebuild.
+INDEX_REVISION = "3"
 
 
 def default_db_path(repo_root: Path) -> Path:
@@ -80,6 +82,7 @@ def _sidecar_digest(repo_root: Path, config: LoadpathConfig) -> str:
             digest.update(rel.encode())
             digest.update(path.read_bytes())
     digest.update(_config_digest(repo_root).encode())
+    digest.update(INDEX_REVISION.encode())
     return digest.hexdigest()
 
 

src/loadpath/review/confidence.py

@@ -40,14 +40,17 @@ def score_confidence(
 
     tested_ids: set[str] = set()
     impact_ids = {n["id"] for n in impact_nodes}
-    all_edges = list(store.edges())
-    for e in list(impact_edges) + all_edges:
-        if e["type"] == EdgeType.TESTED_BY.value:
+    for e in impact_edges:
+        if e["type"] != EdgeType.TESTED_BY.value:
+            continue
+        if e["src"] in impact_ids and e["dst"] in impact_ids:
             tested_ids.add(e["src"])
 
-    # A sink is covered if it, or a producer within two hops (view/serializer/hook/page), is tested.
+    # A sink is covered if it, or a producer within two hops on THIS path, is tested.
     inbound: dict[str, list[str]] = {}
-    for e in all_edges:
+    for e in impact_edges:
+        if e["src"] not in impact_ids or e["dst"] not in impact_ids:
+            continue
         inbound.setdefault(e["dst"], []).append(e["src"])
         inbound.setdefault(e["src"], []).append(e["dst"])
 

src/loadpath/review/engine.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from uuid import uuid4
 
-from loadpath.architecture.rules import evaluate
+from loadpath.architecture.rules import _related_accesses, evaluate
 from loadpath.config import LoadpathConfig, load_config
 from loadpath.graph.store import GraphStore
 from loadpath.index import default_db_path, index_drift, index_repo
@@ -243,6 +243,9 @@ def collect_residuals(store: GraphStore, impact_nodes: list[dict], diff: DiffSet
     for line in stored.splitlines():
         if any(f and f in line for f in impact_files) or any(n and str(n) in line for n in impact_names):
             residuals.append(line)
+    fields_by_name: dict[str, list[dict]] = {}
+    for field in store.nodes([NodeType.FIELD]):
+        fields_by_name.setdefault(field["name"], []).append(field)
     for n in impact_nodes:
         extra = n.get("extra") or {}
         if extra.get("get_serializer_class"):
@@ -254,12 +257,28 @@ def collect_residuals(store: GraphStore, impact_nodes: list[dict], diff: DiffSet
         if extra.get("queryset_in_serializer"):
             residuals.append(f"Queryset inside serializer {n['qualified_name']}")
         for hit in extra.get("nplusone") or []:
-            accessed = ", ".join(hit.get("accessed") or []) or "related fields"
+            accessed = list(hit.get("accessed") or [])
+            related, _ = _related_accesses(accessed, fields_by_name, extra.get("app"))
+            if not related:
+                continue
             residuals.append(
-                f"N+1 {accessed} in {n.get('file_path')}:{hit.get('line')} — {hit.get('suggested_fix')}"
+                f"N+1 {', '.join(related)} in {n.get('file_path')}:{hit.get('line')} — {hit.get('suggested_fix')}"
             )
     residuals.extend(_test_field_residuals(impact_nodes, diff))
     residuals.extend(_react_path_residuals(impact_nodes, diff))
+    ids = {n["id"] for n in impact_nodes}
+    for e in store.edges():
+        if e["src"] not in ids or e["dst"] not in ids:
+            continue
+        extra = e.get("extra") or {}
+        if extra.get("overlap"):
+            residuals.append(
+                f"Inferred serializer/Zod overlap fields={extra['overlap']}"
+            )
+        if extra.get("superseded_by_generated"):
+            residuals.append(
+                f"String URL stitch {extra.get('react')} superseded by a generated OpenAPI client"
+            )
     seen = set()
     out = []
     for r in residuals:
@@ -269,6 +288,11 @@ def collect_residuals(store: GraphStore, impact_nodes: list[dict], diff: DiffSet
     return out
 
 
+def _serious_evolution_notes(notes: list[str]) -> list[str]:
+    tokens = ("hotspot", "silo", "crosses a bounded", "cross-context", "temporal coupling")
+    return [n for n in notes if any(tok in n.lower() for tok in tokens)]
+
+
 def suggested_reviewers(config: LoadpathConfig, impact_nodes: list[dict]) -> list[str]:
     owners: list[str] = []
     for n in impact_nodes:
@@ -358,10 +382,11 @@ def run_review(
     residuals = collect_residuals(store, impact_nodes, diff)
     evolution = analyze_evolution(repo_root, diff, impact_nodes, config)
     confidence = score_confidence(store, impact_nodes, impact_edges, scoped, residuals)
-    if evolution.get("notes") and confidence["level"] == "high":
+    serious = _serious_evolution_notes(evolution.get("notes") or [])
+    if serious and confidence["level"] == "high":
         confidence["level"] = "medium"
         reasons = list(confidence.get("reasons") or [])
-        reasons = [evolution["notes"][0], *reasons][:3]
+        reasons = [serious[0], *reasons][:3]
         confidence["reasons"] = reasons
     boot = store.get_meta("django_boot") or "off"
     if boot == "failed" and confidence["level"] == "high":