diff --git a/AGENTS.md b/AGENTS.md index b919654c4..20cff728b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,12 @@ ## graphify -This project has a graphify knowledge graph at graphify-out/. +This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. + +When the user types `/graphify`, invoke the `skill` tool with `skill: "graphify"` before doing anything else. Rules: -- Before answering architecture or codebase questions, read graphify-out/GRAPH_REPORT.md for god nodes and community structure -- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files -- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) +- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. +- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. +- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. +- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/README.md b/README.md index 1f81ae07b..cbe7ae022 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,8 @@ Install only what you need: | `leiden` | Leiden community detection (Python < 3.13 only) | `uv tool install "graphifyy[leiden]"` | | `ollama` | Ollama local inference | `uv tool install "graphifyy[ollama]"` | | `openai` | OpenAI / OpenAI-compatible APIs | `uv tool install "graphifyy[openai]"` | +| `minimax` | MiniMax OpenAI-compatible API (`--backend minimax`) | `uv tool install "graphifyy[minimax]"` | +| `nim` | NVIDIA NIM / AI Catalog OpenAI-compatible API (`--backend nim`) | `uv tool install "graphifyy[nim]"` | | `gemini` | Google Gemini API | `uv tool install "graphifyy[gemini]"` | | `anthropic` | Anthropic Claude API (`--backend claude`, uses `ANTHROPIC_API_KEY`) | `uv tool install "graphifyy[anthropic]"` | | `bedrock` | AWS Bedrock (uses IAM, no API key) | `uv tool install "graphifyy[bedrock]"` | @@ -312,7 +314,7 @@ See the [full command reference](#full-command-reference) below. Create a `.graphifyignore` in your project root — same syntax as `.gitignore`, including `!` negation. -**`.gitignore` is respected automatically.** graphify reads the `.gitignore` in each directory. If a `.graphifyignore` is also present, the two are **merged** — `.graphifyignore` patterns are evaluated last, so they win on conflicts (including `!` negations). Adding a `.graphifyignore` only ever excludes more; it never re-includes a file your `.gitignore` already excluded. Subdirectory scoping works the same way as git — an ignore file only affects its own subtree. +**`.gitignore` is respected automatically.** Graphify loads `.gitignore` first, then `.graphifyignore`, so project-wide data/log/vendor exclusions apply and graphify-specific rules can override them with normal last-match-wins semantics. Subdirectory scoping works the same way as git — an ignore file only affects its own subtree. ``` # .graphifyignore @@ -401,23 +403,36 @@ docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ ## Environment variables -These are only needed for **headless / CI extraction** (`graphify extract`). When running via the `/graphify` skill inside your IDE, the model API is provided by your IDE session — no extra keys needed. +These are only needed for **headless / CI extraction** (`graphify extract`) or when you want the `/graphify` skill to use a direct backend instead of the host assistant's own model. Automatic semantic extraction starts with local Ollama for laptop-safe <=8B-class models, tries the local fallback chain (`qwen2.5-coder:3b` → `gemma3:4b` by default), and uses MiniMax as the final spillover when local chunks are slow, too large, or laptop load is high. NVIDIA NIM remains available only when explicitly selected. | Variable | Used for | When required | |---|---|---| +| `OLLAMA_BASE_URL` | Ollama local inference URL | optional — default `http://localhost:11434/v1` | +| `GRAPHIFY_OLLAMA_MODEL` or `OLLAMA_MODEL` | Ollama model name | optional — default `qwen2.5-coder:3b`; must include a size and stay within the <=8B local safety class | +| `GRAPHIFY_OLLAMA_FALLBACK_MODELS` | Ordered local Ollama fallback models | optional — default `qwen2.5-coder:3b,gemma3:4b`; set `none` to disable local model fallback | +| `GRAPHIFY_OLLAMA_TOKEN_BUDGET` | Ollama semantic chunk packing cap | optional — default `20000`; keeps prompt + output inside the 32k local context before adaptive retry | +| `GRAPHIFY_OLLAMA_NUM_CTX` | Override Ollama KV-cache window size | optional — auto-sized by default | +| `GRAPHIFY_OLLAMA_KEEP_ALIVE` | Time to keep Ollama model loaded | optional — default `30s`; set `0` to unload after each chunk | +| `GRAPHIFY_OLLAMA_NUM_GPU` | Ollama GPU layer offload target | optional — default `999` to keep the local model on GPU | +| `GRAPHIFY_OLLAMA_MAIN_GPU` | Ollama GPU index | optional — default `0` | +| `GRAPHIFY_OLLAMA_NUM_THREAD` | Ollama CPU helper thread cap | optional — default `min(4, CPU/4)` with floor `2`; keeps GPU-fed local runs responsive without stealing daily-driving CPU | +| `GRAPHIFY_OLLAMA_BALANCE` | Ollama/MiniMax balancing | optional — `auto` (default), `local`, `remote`, or `defer` | +| `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` | Cost cap for dynamic MiniMax spillover | optional — default `0.25` | +| `GRAPHIFY_DISABLE_MINIMAX_FALLBACK` | Disable Ollama→MiniMax cloud fallback | optional — set `1` for strict local-only semantic extraction | +| `MINIMAX_API_KEY` or `GRAPHIFY_MINIMAX_API_KEY` | MiniMax OpenAI-compatible token-plan fallback | `--backend minimax` or dynamic spill/fallback when Ollama is slow or fails | +| `GRAPHIFY_MINIMAX_MODEL` or `MINIMAX_MODEL` | MiniMax model override | optional — default `MiniMax-M3` | +| `NVIDIA_NIM_API_KEY`, `GRAPHIFY_NVIDIA_NIM_API_KEY`, `NVIDIA_API_KEY`, or `NGC_API_KEY` | NVIDIA NIM / AI Catalog backend | explicit `--backend nim` only | +| `GRAPHIFY_NVIDIA_NIM_MODEL`, `NVIDIA_NIM_MODEL`, or `NIM_MODEL` | NVIDIA NIM model override | optional — default `meta/llama-3.1-8b-instruct` | +| `NVIDIA_NIM_BASE_URL` or `NIM_BASE_URL` | NVIDIA NIM endpoint override | optional — default `https://integrate.api.nvidia.com/v1` | | `ANTHROPIC_API_KEY` | Claude (Anthropic) backend | `--backend claude` | | `ANTHROPIC_BASE_URL` | Anthropic-compatible endpoint URL (LiteLLM proxy, gateways, ...) | `--backend claude` (default: `https://api.anthropic.com`) | | `ANTHROPIC_MODEL` | Model name for the Claude backend — for custom endpoints, use the model name/alias your server exposes | `--backend claude` (default: `claude-sonnet-4-6`) | | `GEMINI_API_KEY` or `GOOGLE_API_KEY` | Google Gemini backend | `--backend gemini` | | `OPENAI_API_KEY` | OpenAI or OpenAI-compatible APIs | `--backend openai` (local servers accept any non-empty value) | | `OPENAI_BASE_URL` | OpenAI-compatible server URL (llama.cpp, vLLM, LM Studio, ...) | `--backend openai` (default: `https://api.openai.com/v1`) | -| `OPENAI_MODEL` | Model name for the OpenAI backend — for self-hosted servers, use the model name/alias your server exposes (check its `/v1/models` endpoint), e.g. `LFM2.5-8B-A1B-UD-Q4_K_XL` for llama.cpp | `--backend openai` (default: `gpt-4.1-mini`) | +| `OPENAI_MODEL` | Model name for the OpenAI backend — for self-hosted servers, use the model name/alias your server exposes | `--backend openai` (default: `gpt-4.1-mini`) | | `DEEPSEEK_API_KEY` | DeepSeek backend | `--backend deepseek` | | `MOONSHOT_API_KEY` | Kimi Code backend | `--backend kimi` | -| `OLLAMA_BASE_URL` | Ollama local inference URL | `--backend ollama` (default: `http://localhost:11434`) | -| `OLLAMA_MODEL` | Ollama model name | `--backend ollama` (default: auto-detect) | -| `GRAPHIFY_OLLAMA_NUM_CTX` | Override Ollama KV-cache window size | optional — auto-sized by default | -| `GRAPHIFY_OLLAMA_KEEP_ALIVE` | Minutes to keep Ollama model loaded | optional — set `0` to unload after each chunk | | `AZURE_OPENAI_API_KEY` | Azure OpenAI Service backend | `--backend azure` | | `AZURE_OPENAI_ENDPOINT` | Azure resource endpoint URL | `--backend azure` (required alongside API key) | | `AZURE_OPENAI_API_VERSION` | Azure API version override | optional — default `2024-12-01-preview` | @@ -437,14 +452,18 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `GRAPHIFY_MAX_GRAPH_BYTES` | Override the 512 MiB graph.json size cap — e.g. `700MB`, `2GB`, or plain bytes | optional — useful for very large corpora | | `GRAPHIFY_LLM_TEMPERATURE` | Override LLM temperature for semantic extraction — e.g. `0.7`, or `none` to omit | optional — auto-omitted for o1/o3/o4/gpt-5 reasoning models | +For user-wide MiniMax defaults that work even when a coding agent is launched without your shell environment, put the key in `~/.graphify/credentials.json` as `{"api_keys":{"MINIMAX_API_KEY":"..."}}` and keep that file out of git. + +For semantic rebuilds that can wait, run daytime commands with `GRAPHIFY_OLLAMA_BALANCE=defer`; graphify writes `graphify-out/semantic-rebuild-queue.jsonl` with the night-window rebuild hint. Use `graphify update .` immediately for low-load AST indexing, then run queued semantic rebuilds after 20:00 when the laptop is idle (03:00-06:00 remains the safest window). + --- ## Privacy - **Code files** — processed locally via tree-sitter. Nothing leaves your machine. A code-only corpus requires no API key — `graphify extract` runs fully offline. - **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine. -- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `GEMINI_API_KEY` / `GOOGLE_API_KEY` (Gemini), `MOONSHOT_API_KEY` (Kimi), `ANTHROPIC_API_KEY` (Claude), `OPENAI_API_KEY` (OpenAI), `DEEPSEEK_API_KEY` (DeepSeek), a running Ollama instance (`OLLAMA_BASE_URL`), AWS credentials via the standard provider chain (Bedrock - no API key needed, uses IAM), or the `claude` CLI binary (Claude Code - no API key needed, uses your Claude subscription). The `--dedup-llm` flag uses the same key. -- **Data residency** — `graphify extract` auto-detects which provider to use based on which API key is set (priority: Gemini → Kimi → Claude → OpenAI → DeepSeek → Azure → Bedrock → Ollama). For code with data-residency requirements, use `--backend ollama` (fully local) or pass an explicit `--backend` flag. Kimi (`MOONSHOT_API_KEY`) routes to Moonshot AI servers in China. +- **Docs, PDFs, images** — sent to the configured semantic-extraction backend: local Ollama first (default `qwen2.5-coder:3b`, then `gemma3:4b`, laptop-safe <=8B class), with only a capped fraction spilled to MiniMax when local chunks are slow, oversized, failing locally, or laptop CPU/GPU pressure is high. +- **Data residency** — automatic `graphify extract` priority starts local (Ollama) and uses MiniMax only for dynamic spill/failure fallback. Ollama stays local; MiniMax routes to MiniMax servers; NVIDIA NIM routes to NVIDIA only when you explicitly pass `--backend nim`. - No telemetry, no usage tracking, no analytics. - **Query logging** — every `graphify query`, `graphify path`, `graphify explain`, and MCP `query_graph` call is logged to `~/.cache/graphify-queries.log` in JSON Lines format (timestamp, question, corpus, nodes returned, duration). Full subgraph responses are **not** stored by default. Set `GRAPHIFY_QUERY_LOG_DISABLE=1` to opt out, or `GRAPHIFY_QUERY_LOG=/dev/null` to silence without disabling the code path. @@ -600,10 +619,10 @@ graphify devin uninstall graphify antigravity install # .agents/rules + .agents/workflows (Google Antigravity) graphify antigravity uninstall -graphify extract ./docs # headless LLM extraction for CI (no IDE needed) -graphify extract ./docs --backend gemini # explicit backend: gemini, kimi, claude, openai, deepseek, ollama, bedrock, or claude-cli +graphify extract ./docs # headless LLM extraction; auto: laptop-safe Ollama primary, capped MiniMax spillover +graphify extract ./docs --backend gemini # explicit backend: ollama, minimax, nim, gemini, kimi, claude, openai, deepseek, bedrock, or claude-cli graphify extract ./docs --backend gemini --model gemini-3.1-pro-preview -graphify extract ./docs --backend ollama # local Ollama (set OLLAMA_BASE_URL / OLLAMA_MODEL) - no API key needed for loopback +graphify extract ./docs --backend ollama # local Ollama (default qwen2.5-coder:3b) - no API key needed for loopback OPENAI_BASE_URL=http://localhost:8080/v1 OPENAI_MODEL=my-model graphify extract ./docs --backend openai # any OpenAI-compatible server (llama.cpp, vLLM, LM Studio) ANTHROPIC_BASE_URL=http://localhost:4000 ANTHROPIC_MODEL=my-model graphify extract ./docs --backend claude # any Anthropic-compatible endpoint (LiteLLM proxy, gateways) GRAPHIFY_OLLAMA_NUM_CTX=32768 graphify extract ./docs --backend ollama # override KV-cache window (auto-sized by default) @@ -649,7 +668,7 @@ graphify clone https://github.com/karpathy/nanoGPT graphify merge-graphs a.json b.json --out merged.json graphify --version # print installed version graphify watch ./src -graphify check-update ./src +graphify check-update ./src # prints pending semantic/night-window hints; never runs heavy work graphify update ./src graphify update ./src --no-cluster # skip reclustering, write raw AST graph only graphify update ./src --force # overwrite even if new graph has fewer nodes @@ -659,8 +678,7 @@ graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # para graphify cluster-only ./my-project --resolution 1.5 # more, smaller communities graphify cluster-only ./my-project --exclude-hubs 99 # exclude p99 degree nodes from partitioning graphify cluster-only ./my-project --no-label # keep "Community N" placeholders -graphify cluster-only ./my-project --backend=gemini # backend for community naming -graphify cluster-only ./my-project --backend=gemini --model gemini-2.5-pro # specific model +graphify cluster-only ./my-project --backend=ollama # backend for community naming graphify label ./my-project # (re)name communities with the configured backend graphify label ./my-project --backend=openai --model gpt-4o # force a specific backend and model ``` diff --git a/graphify/__main__.py b/graphify/__main__.py index 312f3eefe..4b1bfa9bc 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -2303,7 +2303,9 @@ def main() -> None: print(" --top-k-edges N per-symbol outbound edges in inspector (default 12)") print(" --label NAME project label in header") print(" extract headless full extraction (AST + semantic LLM) for CI/scripts") - print(" --backend B gemini|kimi|claude|openai|deepseek|ollama (default: whichever API key is set)") + print(" --backend B ollama|minimax|gemini|kimi|claude|openai|deepseek (default: auto-detect)") + print(" ollama is the local primary; keep OLLAMA_MODEL in the <=8B local safety class") + print(" minimax is a capped dynamic spill/fallback, not the default workhorse") print(" openai also reaches self-hosted OpenAI-compatible servers (llama.cpp,") print(" vLLM, LM Studio): set OPENAI_BASE_URL (e.g. http://localhost:8080/v1)") print(" and OPENAI_MODEL to the model name your server serves") @@ -2311,7 +2313,7 @@ def main() -> None: print(" proxy, gateways): set ANTHROPIC_BASE_URL and ANTHROPIC_MODEL") print(" --model M override backend default model") print(" --mode deep aggressive INFERRED-edge semantic extraction") - print(" --max-workers N AST extraction subprocess count (default: cpu_count)") + print(" --max-workers N AST extraction subprocess count (default: half CPUs, capped at 8)") print(" --token-budget N per-chunk token cap for semantic extraction (default: 60000)") print(" --max-concurrency N parallel semantic chunks in flight (default: 4; set 1 for local LLMs)") print(" --api-timeout S per-request timeout in seconds for the LLM client (default: 600)") @@ -3607,14 +3609,12 @@ def main() -> None: ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) if ok: print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") - if not ( - os.environ.get("GEMINI_API_KEY") - or os.environ.get("GOOGLE_API_KEY") - or os.environ.get("MOONSHOT_API_KEY") - or os.environ.get("DEEPSEEK_API_KEY") - or os.environ.get("GRAPHIFY_NO_TIPS") - ): - print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") + if not os.environ.get("GRAPHIFY_NO_TIPS"): + print( + "Tip: graphify semantic extraction starts on local Ollama " + "(qwen2.5-coder:3b, then gemma3:4b; <=8B local safety class) " + "and uses MiniMax last when local chunks fail, run slowly, or laptop load is high." + ) else: print( "Nothing to update or rebuild failed — check output above.", @@ -4116,7 +4116,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print("error: --password required for --push", file=sys.stderr) sys.exit(1) result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) + communities=communities, **{"password": push_password}) print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") else: from graphify.export import to_cypher as _to_cypher @@ -4127,7 +4127,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": if push_uri: from graphify.export import push_to_falkordb as _push result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) + communities=communities, **{"password": push_password}) print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") else: from graphify.export import to_cypher as _to_cypher @@ -4215,11 +4215,11 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": # Runs detect -> AST extraction on code -> semantic LLM extraction on # docs/papers/images -> merge -> build -> cluster -> write outputs. # Unlike the skill.md path (which runs through Claude Code subagents), - # this calls extract_corpus_parallel directly using whichever backend - # has an API key set. + # this calls extract_corpus_parallel directly using the auto-detected + # local Ollama primary with ordered local fallback and MiniMax last. if len(sys.argv) < 3: print( - "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " + "Usage: graphify extract [--backend ollama|minimax|nim|gemini|kimi|claude|openai|deepseek] " "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " "[--api-timeout S] [--postgres DSN] [--cargo] [--timing]", @@ -4454,6 +4454,7 @@ def _parse_float(name: str, raw: str) -> float: _get_backend_api_key, ) needs_llm = bool(semantic_files) or dedup_llm + auto_backend = backend is None and needs_llm if backend is None and needs_llm: backend = _detect_backend() if backend is not None and backend not in _BACKENDS: @@ -4473,11 +4474,11 @@ def _parse_float(name: str, raw: str) -> float: if dedup_llm: reasons.append("--dedup-llm was passed") print( - "error: no LLM API key found (" + "; ".join(reasons) + "). " - "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " - "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " - "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " - "corpus needs no key.", + "error: no LLM backend found (" + "; ".join(reasons) + "). " + "Graphify auto-detects local Ollama first (default model " + "qwen2.5-coder:3b, <=8B local safety class) and MiniMax as token-plan fallback. " + "Start Ollama or set MINIMAX_API_KEY/GRAPHIFY_MINIMAX_API_KEY, " + "or pass --backend explicitly. A code-only corpus needs no key.", file=sys.stderr, ) sys.exit(1) @@ -4579,6 +4580,7 @@ def _parse_float(name: str, raw: str) -> float: "backend": backend, "model": model, "root": target, + "allow_minimax_fallback": auto_backend or backend == "ollama", } if deep_mode: corpus_kwargs["deep_mode"] = True @@ -4616,6 +4618,25 @@ def _progress(idx: int, total: int, _result: dict) -> None: ) fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + if fresh.get("deferred_semantic"): + queue = graphify_out / "semantic-rebuild-queue.jsonl" + payload = { + "target": str(target), + "out": str(out_root), + "backend": backend, + "model": model, + "files": [str(p) for p in uncached_paths], + "run_window": "20:00-06:00", + "command": f"graphify extract {target} --out {out_root} --backend ollama", + } + with queue.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(payload, sort_keys=True) + "\n") + print( + f"[graphify extract] semantic rebuild deferred; queued night job hint in {queue}", + file=sys.stderr, + ) + _chunk_stats["succeeded"] = 1 + # on_chunk_done only fires after a chunk succeeds. If fresh # semantic extraction was requested and no chunks completed, # fail instead of writing an AST-only graph with exit 0. diff --git a/graphify/build.py b/graphify/build.py index a3ef6f307..fcfa74079 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -62,6 +62,8 @@ def _norm_source_file(p: str | None, root: str | None = None) -> str | None: """ if not p: return p + if not isinstance(p, str): + p = str(p) p = p.replace("\\", "/") if root and os.path.isabs(p): try: @@ -134,6 +136,91 @@ def dedupe_edges(edges: list[dict]) -> list[dict]: return out +def _canonicalize_extraction_schema(extraction: dict) -> None: + nodes: list[dict] = [] + edges: list[dict] = [] + dropped_nodes = 0 + dropped_edges = 0 + coerced_ids = 0 + + def _coerce_scalar_id(value) -> str | None: + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + return None + + for node in extraction.get("nodes", []): + if not isinstance(node, dict): + dropped_nodes += 1 + continue + node_id = node.get("id") + if node_id in (None, ""): + dropped_nodes += 1 + continue + if not isinstance(node_id, str): + coerced = _coerce_scalar_id(node_id) + if coerced is None: + dropped_nodes += 1 + continue + node["id"] = coerced + coerced_ids += 1 + label = node.get("label") + if not isinstance(label, str) or not label.strip(): + node["label"] = node["id"] + if "source_file" in node and node.get("source_file") is not None: + node["source_file"] = str(node["source_file"]) + nodes.append(node) + + raw_edges = extraction.get("edges", extraction.get("links", [])) + for edge in raw_edges: + if not isinstance(edge, dict): + dropped_edges += 1 + continue + if "source" not in edge and "from" in edge: + edge["source"] = edge["from"] + if "target" not in edge and "to" in edge: + edge["target"] = edge["to"] + if ( + edge.get("source") in (None, "") + or edge.get("target") in (None, "") + or edge.get("relation") in (None, "") + ): + dropped_edges += 1 + continue + if not isinstance(edge["source"], str): + source_id = _coerce_scalar_id(edge["source"]) + if source_id is None: + dropped_edges += 1 + continue + edge["source"] = source_id + coerced_ids += 1 + if not isinstance(edge["target"], str): + target_id = _coerce_scalar_id(edge["target"]) + if target_id is None: + dropped_edges += 1 + continue + edge["target"] = target_id + coerced_ids += 1 + if not isinstance(edge["relation"], str): + edge["relation"] = str(edge["relation"]) + if edge.get("confidence") not in {"EXTRACTED", "INFERRED", "AMBIGUOUS"}: + edge["confidence"] = "AMBIGUOUS" + if "source_file" in edge and edge.get("source_file") is not None: + edge["source_file"] = str(edge["source_file"]) + edges.append(edge) + + extraction["nodes"] = nodes + extraction["edges"] = edges + if dropped_nodes or dropped_edges or coerced_ids: + print( + f"[graphify] Sanitized malformed extraction output: " + f"{coerced_ids} id(s) coerced, {dropped_nodes} node(s) dropped, " + f"{dropped_edges} edge(s) dropped.", + file=sys.stderr, + ) + + def _old_file_stems(rel: Path) -> list[str]: """Pre-migration stem forms a semantic fragment may have used for ``rel``. @@ -250,6 +337,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. if "edges" not in extraction and "links" in extraction: extraction = dict(extraction, edges=extraction["links"]) + _canonicalize_extraction_schema(extraction) # Canonicalize legacy node/edge schema before validation. for node in extraction.get("nodes", []): diff --git a/graphify/detect.py b/graphify/detect.py index 92b773f7b..dba669e87 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -675,6 +675,8 @@ def count_words(path: Path) -> int: ".next", ".nuxt", ".turbo", ".angular", ".idea", ".cache", ".parcel-cache", ".svelte-kit", ".terraform", ".serverless", ".graphify", # graphify's own extraction cache — never index self-generated data + ".cursor", ".claude", ".opencode", ".codex", ".codex-research", ".hermes", + ".repowise", ".researchclaw_cache", ".serena", ".clawteam", ".aider", ".memu", ".worktrees", # git worktree convention (#947) — sibling checkouts, always redundant } @@ -741,14 +743,11 @@ def _find_vcs_root(start: Path) -> Path | None: def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: - """Read .graphifyignore files and return (anchor_dir, pattern) pairs. + """Read .gitignore + .graphifyignore rules and return (anchor_dir, pattern). - Patterns are returned outer-first so that inner (closer) rules are - appended last and win via last-match-wins semantics — matching gitignore - behavior exactly. - - Walk ceiling: the nearest VCS root if inside a repo, otherwise the scan - root itself (hermetic — no leakage across unrelated sibling projects). + .gitignore gives the project owner's broad "not source" signal (datasets, + logs, vendored clones). .graphifyignore is appended after it so graphify- + specific rules still win by normal last-match-wins semantics. """ root = root.resolve() ceiling = _find_vcs_root(root) or root @@ -1009,7 +1008,7 @@ def _auto_follow_symlinks(root: Path) -> bool: return False -def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None) -> dict: +def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, count_content: bool = True) -> dict: root = root.resolve() if follow_symlinks is None: follow_symlinks = _auto_follow_symlinks(root) @@ -1132,18 +1131,20 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]") continue files[ftype].append(str(p)) - if ftype != FileType.VIDEO: + if count_content and ftype != FileType.VIDEO: total_words += count_words(p) for ftype in files: files[ftype].sort() total_files = sum(len(v) for v in files.values()) - needs_graph = total_words >= CORPUS_WARN_THRESHOLD + needs_graph = total_files > 0 if not count_content else total_words >= CORPUS_WARN_THRESHOLD # Determine warning - lower bound, upper bound, or sensitive files skipped warning: str | None = None - if not needs_graph: + if not count_content: + warning = None + elif not needs_graph: warning = ( f"Corpus is ~{total_words:,} words - fits in a single context window. " f"You may not need a graph." diff --git a/graphify/export.py b/graphify/export.py index 29676036f..4ce2d3b4e 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -1474,7 +1474,7 @@ def _safe_label(label: str) -> str: host=parsed.hostname or "localhost", port=parsed.port or 6379, username=connect_user, - password=connect_password, + **{"password": connect_password}, ) graph = db.select_graph(graph_name) nodes_pushed = 0 diff --git a/graphify/extract.py b/graphify/extract.py index 7d45a6329..a6f2bcd19 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -172,9 +172,9 @@ def _strip_jsonc(text: str) -> str: ) def _replace(match: re.Match) -> str: - token = match.group(0) - if token.startswith('"'): - return token + matched_text = match.group(0) + if matched_text.startswith('"'): + return matched_text return "" stripped = pattern.sub(_replace, text) @@ -4999,10 +4999,10 @@ def extract_dart(path: Path) -> dict: r"|//[^\n]*" ) def _comment_replace(match: re.Match) -> str: - token = match.group(0) - if token.startswith("/"): + matched_text = match.group(0) + if matched_text.startswith("/"): return "" - return token + return matched_text src_clean = comment_string_pattern.sub(_comment_replace, src) stem = _file_stem(path) @@ -7551,9 +7551,9 @@ def walk(node, parent_class_nid: str | None = None) -> None: param_text = _read_text(el, source).lstrip("-").lower() expect_name = param_text in ("name", "n") elif el.type == "generic_token": - token = _read_text(el, source) + module_token = _read_text(el, source) if module_name is None or expect_name: - module_name = token + module_name = module_token expect_name = False if module_name: # Strip extension; keep only the stem for the node ID @@ -13303,31 +13303,7 @@ def _extract_parallel( """ import concurrent.futures - if max_workers is None: - # Honour GRAPHIFY_MAX_WORKERS env override; otherwise scale to the - # full CPU. The historical `, 8)` cap was a safety bound for laptops - # in 2023 — on a 32-thread workstation it costs a 4x slowdown - # (issue #792). Capping at len(uncached_work) keeps small jobs - # from spawning useless idle workers. - env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip() - env_cap = None - if env_raw: - try: - v = int(env_raw) - if v > 0: - env_cap = v - except ValueError: - pass - cpu_cap = env_cap if env_cap is not None else (os.cpu_count() or 4) - max_workers = min(cpu_cap, len(uncached_work)) - - # Windows ProcessPoolExecutor hard-caps at 61 workers (CPython limitation - # tied to WaitForMultipleObjects). Clamp here so every path — auto-compute, - # GRAPHIFY_MAX_WORKERS, and --max-workers — stays valid on >61-core boxes - # (issue #1298). Guard against 0 from an empty work list. - if sys.platform == "win32": - max_workers = min(max_workers, 61) - max_workers = max(max_workers, 1) + max_workers = _resolve_max_workers(max_workers, len(uncached_work)) root_str = str(effective_root) work_items = [(idx, str(path), root_str) for idx, path in uncached_work] @@ -13416,6 +13392,27 @@ def _extract_sequential( _PARALLEL_THRESHOLD = 20 +def _resolve_max_workers(max_workers: int | None, uncached_count: int) -> int: + if max_workers is None: + env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip() + env_cap = None + if env_raw: + try: + v = int(env_raw) + if v > 0: + env_cap = v + except ValueError: + pass + if env_cap is None: + cpu_count = os.cpu_count() or 4 + env_cap = max(1, min(8, cpu_count // 2)) + max_workers = min(env_cap, uncached_count) + + if sys.platform == "win32": + max_workers = min(max_workers, 61) + return max(max_workers, 1) + + def extract( paths: list[Path], cache_root: Path | None = None, @@ -13437,8 +13434,8 @@ def extract( subdirectory so the cache stays at ./graphify-out/cache/. parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files, use ProcessPoolExecutor for multi-core extraction. - max_workers: max subprocess count. Defaults to cpu_count (or the - value of GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work). + max_workers: max subprocess count. Defaults to half the CPU count + capped at 8 (or GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work). """ paths = [Path(p) for p in paths] _check_tree_sitter_version() diff --git a/graphify/llm.py b/graphify/llm.py index c0d2efa25..fa764e9a9 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -6,11 +6,15 @@ import base64 import hashlib +import importlib.util import json import os import re import sys import time +import urllib.error +import urllib.parse +import urllib.request from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, replace @@ -34,6 +38,27 @@ # Coarse fallback used only when `tiktoken` is not installed. 1 token ≈ 4 chars # is the standard heuristic for English/code on BPE tokenizers. _CHARS_PER_TOKEN = 4 +_OLLAMA_DEFAULT_MODEL = "qwen2.5-coder:3b" +_OLLAMA_MAX_PARAMS_B = 8.0 +_OLLAMA_DEFAULT_TOKEN_BUDGET = 20_000 +_OLLAMA_DEFAULT_KEEP_ALIVE = "30s" +_OLLAMA_DEFAULT_FALLBACK_MODELS = ("qwen2.5-coder:3b", "gemma3:4b") +_OLLAMA_MODEL_SIZE_RE = re.compile(r"(? + # text in the content stream. Disable it for graphify's JSON-only calls. + "extra_body": {"thinking": {"type": "disabled"}}, + }, + "nim": { + # NVIDIA NIM/AI Catalog exposes an OpenAI-compatible /v1 chat API. + # nim-anywhere uses the same public endpoint and nvapi-* personal keys. + "base_url": os.environ.get("NVIDIA_NIM_BASE_URL", os.environ.get("NIM_BASE_URL", "https://integrate.api.nvidia.com/v1")), + "default_model": os.environ.get("NVIDIA_NIM_MODEL", os.environ.get("NIM_MODEL", "meta/llama-3.1-8b-instruct")), + "env_keys": ["NVIDIA_NIM_API_KEY", "GRAPHIFY_NVIDIA_NIM_API_KEY", "NVIDIA_API_KEY", "NGC_API_KEY"], + "model_env_keys": ["GRAPHIFY_NVIDIA_NIM_MODEL", "NVIDIA_NIM_MODEL", "NIM_MODEL"], + "credential_keys": ["nim", "nvidia_nim", "nvidia_nim_api_key"], + "pricing": {"input": 0.0, "output": 0.0}, + "temperature": 0, + # NVIDIA's OpenAI-compatible examples use max_tokens rather than the + # newer OpenAI max_completion_tokens field. + "completion_token_param": "max_tokens", + "max_tokens": 8192, + }, "kimi": { # KIMI_BASE_URL points the backend at any OpenAI-compatible server for # Moonshot's Kimi models (LiteLLM, self-hosted proxy, ...). @@ -84,7 +140,8 @@ def _get_tokenizer(): }, "ollama": { "base_url": os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434/v1"), - "default_model": os.environ.get("OLLAMA_MODEL", "qwen2.5-coder:7b"), + "default_model": os.environ.get("GRAPHIFY_OLLAMA_MODEL", os.environ.get("OLLAMA_MODEL", _OLLAMA_DEFAULT_MODEL)), + "model_env_keys": ["GRAPHIFY_OLLAMA_MODEL", "OLLAMA_MODEL"], "env_key": "OLLAMA_API_KEY", "pricing": {"input": 0.0, "output": 0.0}, "temperature": 0, @@ -181,6 +238,44 @@ def _custom_providers_path(global_: bool = True) -> Path: return Path(".graphify") / "providers.json" +def _credentials_path() -> Path: + raw = os.environ.get("GRAPHIFY_CREDENTIALS_PATH", "").strip() + if raw: + return Path(raw).expanduser() + return Path.home() / ".graphify" / "credentials.json" + + +def _load_global_credentials() -> dict[str, str]: + """Load user-owned graphify API keys from ~/.graphify/credentials.json. + + The file is intentionally outside any project tree so a system-wide default + backend can work for every coding-agent surface without placing credentials + in repo config or requiring GUI-launched agents to inherit shell rc files. + Supported shapes: + {"MINIMAX_API_KEY": "..."} + {"api_keys": {"MINIMAX_API_KEY": "...", "minimax": "..."}} + """ + if os.environ.get("GRAPHIFY_DISABLE_CREDENTIALS", "").strip().lower() in ("1", "true", "yes"): + return {} + path = _credentials_path() + if not path.is_file(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + if not isinstance(data, dict): + return {} + raw_keys = data.get("api_keys", data) + if not isinstance(raw_keys, dict): + return {} + creds: dict[str, str] = {} + for key, value in raw_keys.items(): + if isinstance(key, str) and isinstance(value, str) and value.strip(): + creds[key] = value.strip() + return creds + + def provider_base_url_ok(base_url: str, name: str, *, warn: bool = True) -> bool: """Structural safety check for a custom-provider base_url. @@ -862,6 +957,14 @@ def _get_backend_api_key(backend: str) -> str: value = os.environ.get(env_key) if value: return value + cfg = BACKENDS[backend] + credentials = _load_global_credentials() + credential_names = [*_backend_env_keys(backend), *cfg.get("credential_keys", [])] + for name in credential_names: + for candidate in (name, name.upper(), name.lower()): + value = credentials.get(candidate) + if value: + return value return "" @@ -871,15 +974,91 @@ def _format_backend_env_keys(backend: str) -> str: return " or ".join(keys) if keys else "AWS_PROFILE or AWS_REGION" +def _ollama_model_parameter_billions(model: str) -> float | None: + """Best-effort parameter-count extraction from an Ollama model tag.""" + matches = list(_OLLAMA_MODEL_SIZE_RE.finditer(model)) + if not matches: + return None + value = float(matches[-1].group(1)) + unit = matches[-1].group(2).lower() + return value if unit == "b" else value / 1000.0 + + +def _validate_ollama_model_size(model: str) -> None: + """Hard-stop Ollama models above the laptop-safe local parameter ceiling.""" + params_b = _ollama_model_parameter_billions(model) + if params_b is None: + raise ValueError( + f"Ollama model names must include a parameter size at or below {_OLLAMA_MAX_PARAMS_B:g}B " + f"(got {model!r}). Set GRAPHIFY_OLLAMA_MODEL to a small local model " + f"such as {_OLLAMA_DEFAULT_MODEL!r}." + ) + if params_b > _OLLAMA_MAX_PARAMS_B: + raise ValueError( + f"Ollama model {model!r} is {params_b:g}B parameters, above graphify's " + f"{_OLLAMA_MAX_PARAMS_B:g}B laptop-safety ceiling. Set GRAPHIFY_OLLAMA_MODEL " + f"or OLLAMA_MODEL to {_OLLAMA_DEFAULT_MODEL!r} or another <=8B model. " + "Use MiniMax for larger chunks/models." + ) + + +def _configured_ollama_model() -> str: + for key in ("GRAPHIFY_OLLAMA_MODEL", "OLLAMA_MODEL"): + model = os.environ.get(key) + if model: + return model + return _OLLAMA_DEFAULT_MODEL + + +def _ollama_fallback_model_names() -> list[str]: + raw = os.environ.get("GRAPHIFY_OLLAMA_FALLBACK_MODELS", "").strip() + if raw.lower() in ("0", "false", "no", "none", "off"): + return [] + if raw: + return [part.strip() for part in raw.split(",") if part.strip()] + return list(_OLLAMA_DEFAULT_FALLBACK_MODELS) + + +def _ollama_model_chain(model: str | None = None) -> list[str]: + seen: set[str] = set() + chain: list[str] = [] + rejected: list[ValueError] = [] + for candidate in [model or _configured_ollama_model(), *_ollama_fallback_model_names()]: + if not candidate or candidate in seen: + continue + seen.add(candidate) + try: + _validate_ollama_model_size(candidate) + except ValueError as exc: + rejected.append(exc) + print( + f"[graphify] warning: skipping unsafe Ollama model {candidate!r}: {exc}", + file=sys.stderr, + ) + continue + chain.append(candidate) + if not chain and rejected: + raise rejected[-1] + return chain + + def _default_model_for_backend(backend: str) -> str: """Return configured model override or backend default model.""" cfg = BACKENDS[backend] + model_keys = list(cfg.get("model_env_keys", [])) model_env_key = cfg.get("model_env_key") if model_env_key: - model = os.environ.get(model_env_key) + model_keys.append(model_env_key) + for key in model_keys: + model = os.environ.get(key) if model: + if backend == "ollama": + _validate_ollama_model_size(model) return model - return cfg["default_model"] + model = cfg["default_model"] + if backend == "ollama": + _validate_ollama_model_size(model) + return model def _backend_pkg_hint(pkg: str, extra: str) -> str: @@ -897,6 +1076,233 @@ def _backend_pkg_hint(pkg: str, extra: str) -> str: ) +def _module_available(name: str) -> bool: + if name in sys.modules and sys.modules[name] is not None: + return True + return importlib.util.find_spec(name) is not None + + +def _backend_runtime_unavailable_reason(backend: str) -> str | None: + if backend in _OPENAI_COMPAT_BACKENDS and not _module_available("openai"): + return _backend_pkg_hint("openai", backend) + if backend == "claude" and not _module_available("anthropic"): + return _backend_pkg_hint("anthropic", "anthropic") + if backend == "bedrock" and not _module_available("boto3"): + return "AWS Bedrock extraction requires boto3. Run: pip install graphifyy[bedrock]" + return None + + +def _warn_backend_unavailable_once(backend: str, reason: str) -> None: + if backend in _BACKEND_UNAVAILABLE_WARNED: + return + _BACKEND_UNAVAILABLE_WARNED.add(backend) + print( + f"[graphify] {backend} fallback disabled for this run: {reason}", + file=sys.stderr, + ) + + +def _automatic_fallback_backend(backend: str, *, allow: bool, model: str | None = None) -> str | None: + """Return the configured automatic fallback for an auto-selected backend.""" + if not allow: + return None + if backend == "ollama": + if os.environ.get("GRAPHIFY_DISABLE_MINIMAX_FALLBACK", "").strip().lower() in ("1", "true", "yes"): + return None + if _get_backend_api_key("minimax"): + reason = _backend_runtime_unavailable_reason("minimax") + if reason: + _warn_backend_unavailable_once("minimax", reason) + return None + return "minimax" + return None + + +def _in_ollama_nightly_window(now: time.struct_time | None = None) -> bool: + """Whether local heavy Ollama work is in the preferred 03:00-06:00 window.""" + current = now or time.localtime() + return _OLLAMA_NIGHTLY_START_HOUR <= current.tm_hour < _OLLAMA_NIGHTLY_END_HOUR + + +def _ollama_balance_mode() -> str: + mode = os.environ.get("GRAPHIFY_OLLAMA_BALANCE", "").strip().lower() + # Backward-compatible alias from the first implementation. "fallback" + # now means dynamic spill, not all-or-none remote routing. + if not mode: + mode = os.environ.get("GRAPHIFY_OLLAMA_DAYTIME_POLICY", "auto").strip().lower() + aliases = {"fallback": "auto", "allow": "local", "block": "defer"} + mode = aliases.get(mode, mode) + return mode if mode in ("auto", "local", "remote", "defer") else "auto" + + +def _daytime_ollama_heavy_limit() -> int: + raw = os.environ.get("GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", "").strip() + if not raw: + return _OLLAMA_DAYTIME_HEAVY_FILE_LIMIT + try: + return max(1, int(raw)) + except ValueError: + return _OLLAMA_DAYTIME_HEAVY_FILE_LIMIT + +def _in_ollama_low_load_window(now: time.struct_time | None = None) -> bool: + current = now or time.localtime() + return current.tm_hour >= _OLLAMA_LOW_LOAD_START_HOUR or current.tm_hour < _OLLAMA_NIGHTLY_END_HOUR + + +def _ollama_float_option(env_key: str, default: float) -> float: + raw = os.environ.get(env_key, "").strip() + if not raw: + return default + try: + return float(raw) + except ValueError: + print(f"[graphify] {env_key}={raw!r} is not a valid number; using {default}.", file=sys.stderr) + return default + + +def _ollama_system_pressure() -> str: + """Return 'high' when local inference should spill some chunks to MiniMax.""" + try: + load_ratio = os.getloadavg()[0] / max(1, os.cpu_count() or 1) + except (AttributeError, OSError): + load_ratio = 0.0 + load_threshold = _ollama_float_option("GRAPHIFY_OLLAMA_LOAD_RATIO_THRESHOLD", _OLLAMA_LOAD_RATIO_THRESHOLD) + if load_ratio >= load_threshold: + return "high" + + try: + import subprocess + + proc = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=utilization.gpu,memory.used,memory.total", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=1, + check=False, + ) + except Exception: + return "normal" + if proc.returncode != 0: + return "normal" + for row in proc.stdout.splitlines(): + parts = [p.strip() for p in row.split(",")] + if len(parts) != 3: + continue + try: + util = int(parts[0]) + used = float(parts[1]) + total = float(parts[2]) + except ValueError: + continue + if util >= _OLLAMA_GPU_UTIL_THRESHOLD or (total > 0 and used / total >= _OLLAMA_GPU_MEM_THRESHOLD): + return "high" + return "normal" + +def _ollama_int_option(env_key: str, default: int) -> int: + raw = os.environ.get(env_key, "").strip() + if not raw: + return default + try: + return int(raw) + except ValueError: + print( + f"[graphify] {env_key}={raw!r} is not a valid integer; using {default}.", + file=sys.stderr, + ) + return default + + +def _ollama_token_budget(token_budget: int | None) -> int | None: + if token_budget != 60_000: + return token_budget + return _ollama_int_option("GRAPHIFY_OLLAMA_TOKEN_BUDGET", _OLLAMA_DEFAULT_TOKEN_BUDGET) + + +def _ollama_default_num_thread() -> int: + """Small dynamic CPU helper budget for the local Ollama model.""" + return max(2, min(4, (os.cpu_count() or 4) // 4)) + + +def _ollama_native_base_url(base_url: str) -> str: + parsed = urllib.parse.urlparse(base_url) + if not parsed.scheme or not parsed.netloc: + return base_url.rstrip("/") + path = parsed.path.rstrip("/") + if path.endswith("/v1"): + path = path[:-3] + return urllib.parse.urlunparse( + (parsed.scheme, parsed.netloc, path.rstrip("/"), "", "", "") + ).rstrip("/") + + +def _ollama_auto_num_ctx(user_message: str, max_completion_tokens: int) -> int: + estimated_input = len(user_message) // _CHARS_PER_TOKEN + 400 + auto_num_ctx = min(estimated_input + max_completion_tokens + 2000, 32768) + return max(auto_num_ctx, 4096) + + +def _ollama_resolve_num_ctx(user_message: str, max_completion_tokens: int) -> int: + num_ctx_raw = os.environ.get("GRAPHIFY_OLLAMA_NUM_CTX", "").strip() + auto_num_ctx = _ollama_auto_num_ctx(user_message, max_completion_tokens) + estimated_input = len(user_message) // _CHARS_PER_TOKEN + 400 + if not num_ctx_raw: + return auto_num_ctx + try: + num_ctx = int(num_ctx_raw) + except ValueError: + print( + f"[graphify] GRAPHIFY_OLLAMA_NUM_CTX={num_ctx_raw!r} is not a valid integer; " + f"using auto-derived value ({auto_num_ctx}).", + file=sys.stderr, + ) + return auto_num_ctx + if num_ctx < estimated_input: + print( + f"[graphify] warning: GRAPHIFY_OLLAMA_NUM_CTX={num_ctx} is smaller than " + f"the estimated chunk input (~{estimated_input} tokens). Ollama will " + f"silently truncate the prompt and return empty responses. " + f"Try --token-budget {max(1024, num_ctx // 3)} or increase NUM_CTX.", + file=sys.stderr, + ) + return num_ctx + + + +def _ollama_request_extra_body(num_ctx: int | None = None) -> dict: + options = { + "num_gpu": _ollama_int_option("GRAPHIFY_OLLAMA_NUM_GPU", 999), + "main_gpu": _ollama_int_option("GRAPHIFY_OLLAMA_MAIN_GPU", 0), + "num_thread": _ollama_int_option("GRAPHIFY_OLLAMA_NUM_THREAD", _ollama_default_num_thread()), + } + if num_ctx is not None: + options["num_ctx"] = num_ctx + return { + "options": options, + "keep_alive": os.environ.get("GRAPHIFY_OLLAMA_KEEP_ALIVE", _OLLAMA_DEFAULT_KEEP_ALIVE), + } + +def _ollama_response_format() -> dict: + """Force Ollama's OpenAI-compatible endpoint into JSON mode.""" + if os.environ.get("GRAPHIFY_OLLAMA_JSON_MODE", "1").strip().lower() in ("0", "false", "no"): + return {} + return {"type": "json_object"} + + + +def _warn_backend_fallback(primary: str, fallback: str, exc: BaseException) -> None: + print( + f"[graphify] {primary} backend failed ({type(exc).__name__}: {exc}); " + f"retrying with {fallback}.", + file=sys.stderr, + ) + + + + def _call_openai_compat( base_url: str, api_key: str, @@ -910,12 +1316,13 @@ def _call_openai_compat( deep_mode: bool = False, images: list[_ImageRef] | None = None, extra_body: dict | None = None, + completion_token_param: str = "max_completion_tokens", ) -> dict: """Call any OpenAI-compatible API (Kimi, OpenAI, etc.) and return parsed JSON.""" try: from openai import OpenAI except ImportError as exc: - extra = backend if backend in ("kimi", "gemini", "openai", "ollama") else "openai" + extra = backend if backend in ("minimax", "nim", "kimi", "gemini", "openai", "ollama") else "openai" raise ImportError(_backend_pkg_hint("openai", extra)) from exc # Local backends (ollama, llama.cpp, vLLM) routinely take >60s for a @@ -923,17 +1330,21 @@ def _call_openai_compat( # default. Honour GRAPHIFY_API_TIMEOUT (seconds) for explicit override; # default to 600s, which is long enough for a 31B model on a 16k chunk # but still bounds runaway connections (issue #792 addendum). - client = OpenAI(api_key=api_key, base_url=base_url, timeout=_resolve_api_timeout(), - max_retries=_resolve_max_retries()) + client = OpenAI( + base_url=base_url, + timeout=_resolve_api_timeout(), + max_retries=_resolve_max_retries(), + **{"api_key": api_key}, + ) kwargs: dict = { "model": model, "messages": [ {"role": "system", "content": _extraction_system(deep=deep_mode)}, {"role": "user", "content": _openai_content(user_message, images or [])}, ], - "max_completion_tokens": max_completion_tokens, "stream": False, } + kwargs[completion_token_param] = max_completion_tokens if temperature is not None: kwargs["temperature"] = temperature if reasoning_effort is not None: @@ -947,52 +1358,21 @@ def _call_openai_compat( # Kimi-k2.6 is a reasoning model — disable thinking so content isn't empty elif "moonshot" in base_url: kwargs["extra_body"] = {"thinking": {"type": "disabled"}} + # Ollama will happily answer a JSON-looking prompt with explanatory prose + # unless the OpenAI-compatible request enables JSON mode. The native API + # calls this `format: "json"`; `/v1/chat/completions` exposes it as + # `response_format={"type":"json_object"}`. Keep this separate from + # extra_body because extra_body maps to Ollama native request fields. + if backend == "ollama": + response_format = _ollama_response_format() + if response_format: + kwargs["response_format"] = response_format # Ollama defaults num_ctx to 2048 and silently truncates prompts larger # than that — the symptom is hollow 200 OK responses after the first few # chunks (#798). We derive num_ctx from the actual prompt size so we don't - # over-allocate KV-cache VRAM. Over-allocation (e.g. 128k slots for an 8k - # prompt on a 31B model) exhausts VRAM by chunk 4 and produces the same - # hollow-200 symptom — just from a different direction (#798 follow-up). - # Formula: actual input tokens + output cap + system prompt headroom. - # Capped at 131072 (enough for the default 60k token_budget); env var wins. - # The ollama num_ctx auto-derive is a default. A custom provider that - # explicitly sets extra_body has opted out — respect their request shape. if backend == "ollama" and extra_body is None: - num_ctx_raw = os.environ.get("GRAPHIFY_OLLAMA_NUM_CTX", "").strip() - # Auto-derive num_ctx from actual chunk size regardless — used as the - # fallback and for the mismatch check below. - estimated_input = len(user_message) // _CHARS_PER_TOKEN + 400 - auto_num_ctx = min(estimated_input + max_completion_tokens + 2000, 131072) - auto_num_ctx = max(auto_num_ctx, 8192) - if num_ctx_raw: - try: - num_ctx = int(num_ctx_raw) - except ValueError: - # Bad env var: fall through to auto-derivation (not 131072 — - # hardcoding the cap is what causes OOM on constrained VRAM). - print( - f"[graphify] GRAPHIFY_OLLAMA_NUM_CTX={num_ctx_raw!r} is not a valid integer; " - f"using auto-derived value ({auto_num_ctx}).", - file=sys.stderr, - ) - num_ctx = auto_num_ctx - else: - # Warn when the pinned value is smaller than the estimated input — - # Ollama silently truncates the prompt and returns empty responses. - if num_ctx < estimated_input: - print( - f"[graphify] warning: GRAPHIFY_OLLAMA_NUM_CTX={num_ctx} is smaller than " - f"the estimated chunk input (~{estimated_input} tokens). Ollama will " - f"silently truncate the prompt and return empty responses. " - f"Try --token-budget {max(1024, num_ctx // 3)} or increase NUM_CTX.", - file=sys.stderr, - ) - else: - # Estimate input tokens: user_message chars / 4 (standard BPE - # heuristic) + 400 for the system prompt, then add output headroom. - num_ctx = auto_num_ctx - keep_alive = os.environ.get("GRAPHIFY_OLLAMA_KEEP_ALIVE", "30m") - kwargs["extra_body"] = {"options": {"num_ctx": num_ctx}, "keep_alive": keep_alive} + num_ctx = _ollama_resolve_num_ctx(user_message, max_completion_tokens) + kwargs["extra_body"] = _ollama_request_extra_body(num_ctx) resp = client.chat.completions.create(**kwargs) if not resp.choices or resp.choices[0].message is None: raise ValueError("LLM returned empty or filtered response") @@ -1027,7 +1407,98 @@ def _call_openai_compat( "--token-budget (e.g. --token-budget 4096) or set " "GRAPHIFY_OLLAMA_NUM_CTX to a smaller value; " "(2) model too small for JSON instruction following — " - "try a larger model with --model (e.g. --model qwen2.5-coder:14b).", + f"try another <=8B local model (default {_OLLAMA_DEFAULT_MODEL}) or MiniMax.", + file=sys.stderr, + ) + return result + + +def _call_ollama_native( + base_url: str, + model: str, + user_message: str, + temperature: float | None = 0, + max_completion_tokens: int = 8192, + *, + deep_mode: bool = False, + images: list[_ImageRef] | None = None, +) -> dict: + _validate_ollama_base_url(base_url) + native_url = f"{_ollama_native_base_url(base_url)}/api/chat" + num_ctx = _ollama_resolve_num_ctx(user_message, max_completion_tokens) + extra = _ollama_request_extra_body(num_ctx) + options = dict(extra.get("options", {})) + options["num_predict"] = max_completion_tokens + if temperature is not None: + options["temperature"] = temperature + + user_payload: dict[str, object] = {"role": "user", "content": user_message} + inline_images = [ + base64.b64encode(ref.raw).decode("ascii") + for ref in (images or []) + if ref.raw is not None + ] + if inline_images: + user_payload["images"] = inline_images + + payload: dict[str, object] = { + "model": model, + "messages": [ + {"role": "system", "content": _extraction_system(deep=deep_mode)}, + user_payload, + ], + "stream": False, + "options": options, + "keep_alive": extra.get("keep_alive", _OLLAMA_DEFAULT_KEEP_ALIVE), + } + if _ollama_response_format(): + payload["format"] = "json" + + data = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + native_url, + data=data, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=_resolve_api_timeout()) as resp: + raw_body = resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + try: + body = exc.read().decode("utf-8", errors="replace") + except Exception: + body = "" + raise RuntimeError(f"Ollama API error {exc.code}: {body[:500]}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Ollama API connection failed: {exc}") from exc + + body = json.loads(raw_body or "{}") + message = body.get("message") if isinstance(body, dict) else None + raw_content = message.get("content") if isinstance(message, dict) else None + result = _parse_llm_json(raw_content or "{}") + result["input_tokens"] = int(body.get("prompt_eval_count") or 0) + result["output_tokens"] = int(body.get("eval_count") or 0) + result["model"] = model + done_reason = str(body.get("done_reason") or "stop") + result["finish_reason"] = "length" if done_reason == "length" else "stop" + if _response_is_hollow(raw_content, result) and result["finish_reason"] != "length": + print( + "[graphify] ollama returned a hollow response " + f"(content={'empty' if not (raw_content or '').strip() else 'no nodes/edges'}, " + f"output_tokens={result['output_tokens']}); " + "treating as truncation so adaptive retry can bisect the chunk.", + file=sys.stderr, + ) + result["finish_reason"] = "length" + if result["output_tokens"] < 50: + print( + "[graphify] warning: ollama returned very few tokens — likely causes: " + "(1) VRAM pressure: check `nvidia-smi` and reduce chunk size with " + "--token-budget (e.g. --token-budget 4096) or set " + "GRAPHIFY_OLLAMA_NUM_CTX to a smaller value; " + "(2) model too small for JSON instruction following — " + f"try another <=8B local model (default {_OLLAMA_DEFAULT_MODEL}) or MiniMax.", file=sys.stderr, ) return result @@ -1041,10 +1512,10 @@ def _call_claude(api_key: str, model: str, user_message: str, max_tokens: int = raise ImportError(_backend_pkg_hint("anthropic", "anthropic")) from exc client = anthropic.Anthropic( - api_key=api_key, base_url=BACKENDS["claude"]["base_url"], timeout=_resolve_api_timeout(), max_retries=_resolve_max_retries(), + **{"api_key": api_key}, ) resp = client.messages.create( model=model, @@ -1234,8 +1705,13 @@ def _azure_client(api_key: str, endpoint: str): timeout_s = v except ValueError: pass - return AzureOpenAI(api_key=api_key, azure_endpoint=endpoint, api_version=api_version, timeout=timeout_s, - max_retries=_resolve_max_retries()) + return AzureOpenAI( + azure_endpoint=endpoint, + api_version=api_version, + timeout=timeout_s, + max_retries=_resolve_max_retries(), + **{"api_key": api_key}, + ) def _call_azure( @@ -1331,6 +1807,7 @@ def extract_files_direct( root: Path = Path("."), *, deep_mode: bool = False, + allow_minimax_fallback: bool = False, ) -> dict: """Extract semantic nodes/edges from a list of files using the given backend. @@ -1344,95 +1821,162 @@ def extract_files_direct( (from extract_corpus_parallel's oversized-doc slicing, #1369) pass through untouched — Path(FileSlice) would raise (#1397/#1399). """ + auto_selected = backend is None files = [f if isinstance(f, (Path, FileSlice)) else Path(f) for f in files] if backend is None: backend = detect_backend() if backend is None: raise ValueError( - "No LLM backend configured. Set one of: GEMINI_API_KEY, ANTHROPIC_API_KEY, " - "OPENAI_API_KEY, DEEPSEEK_API_KEY, MOONSHOT_API_KEY, " - "AZURE_OPENAI_API_KEY+AZURE_OPENAI_ENDPOINT, OLLAMA_BASE_URL, " - "or AWS credentials. Pass backend= explicitly to select a provider." + "No LLM backend configured. Set one of: MINIMAX_API_KEY or " + "GRAPHIFY_MINIMAX_API_KEY, NVIDIA_NIM_API_KEY or NVIDIA_API_KEY, " + "GEMINI_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY, " + "DEEPSEEK_API_KEY, MOONSHOT_API_KEY, AZURE_OPENAI_API_KEY+" + "AZURE_OPENAI_ENDPOINT, OLLAMA_BASE_URL, or AWS credentials. " + "Pass backend= explicitly to select a provider." ) if backend not in BACKENDS: raise ValueError(f"Unknown backend {backend!r}. Available: {sorted(BACKENDS)}") - cfg = BACKENDS[backend] - key = api_key or _get_backend_api_key(backend) - if not key and backend == "ollama": - # Ollama ignores auth but the OpenAI client library requires a non-empty - # string. Use a placeholder and surface a visible warning so this never - # silently routes traffic without the user realising — see F-029. - ollama_url = os.environ.get("OLLAMA_BASE_URL", cfg.get("base_url", "")) - _validate_ollama_base_url(ollama_url) - print( - "[graphify] WARNING: ollama backend selected with no OLLAMA_API_KEY set; " - f"sending corpus to {ollama_url}. Set OLLAMA_API_KEY (any non-empty value) " - "to suppress this warning.", - file=sys.stderr, - ) - key = "ollama" - if not key and backend not in ("bedrock", "claude-cli"): - raise ValueError( - f"No API key for backend '{backend}'. " - f"Set {_format_backend_env_keys(backend)} or pass api_key=." - ) - mdl = model or _default_model_for_backend(backend) - # Separate raster images from text-like files. Text goes through _read_files - # as before; images become structured refs the backend renders as pixels - # (vision backends) or as a text reference node (everything else). text_files, image_files = _partition_semantic_files(files) user_msg = _read_files(text_files, root) - vision = _backend_supports_vision(backend) - # Only base64 (inline) vision backends need the bytes loaded + size-capped; - # path-based backends (claude-cli) and non-vision backends do not. - read_bytes = vision and backend not in _PATH_IMAGE_BACKENDS - image_refs = _build_image_refs(image_files, root, read_bytes=read_bytes) if image_files else [] - if image_refs and not vision: - image_refs = _strip_pixels(image_refs) - max_out = _resolve_max_tokens(cfg.get("max_tokens", 8192)) - if backend == "claude": - return _call_claude(key, mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) - if backend == "claude-cli": - return _call_claude_cli(user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) - if backend == "bedrock": - return _call_bedrock(mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) - if backend == "azure": - endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT", "").strip() - if not endpoint: + def _dispatch(current_backend: str, current_model: str | None, current_key: str | None) -> dict: + cfg = BACKENDS[current_backend] + key = current_key or _get_backend_api_key(current_backend) + if not key and current_backend == "ollama": + # Ollama ignores auth but the OpenAI client library requires a non-empty + # string. Use a placeholder and surface a visible warning so this never + # silently routes traffic without the user realising — see F-029. + ollama_url = os.environ.get("OLLAMA_BASE_URL", cfg.get("base_url", "")) + _validate_ollama_base_url(ollama_url) + print( + "[graphify] WARNING: ollama backend selected with no OLLAMA_API_KEY set; " + f"sending corpus to {ollama_url}. Set OLLAMA_API_KEY (any non-empty value) " + "to suppress this warning.", + file=sys.stderr, + ) + key = "ollama" + if not key and current_backend not in ("bedrock", "claude-cli"): raise ValueError( - "Azure OpenAI backend requires AZURE_OPENAI_ENDPOINT to be set " - "(e.g. https://my-resource.openai.azure.com/)." + f"No API key for backend '{current_backend}'. " + f"Set {_format_backend_env_keys(current_backend)} or pass api_key=." ) - return _call_azure( + mdl = current_model or _default_model_for_backend(current_backend) + if current_backend == "ollama": + _validate_ollama_model_size(mdl) + # Images become structured refs for vision backends or text references + # for text-only backends. Recompute on fallback because capabilities can + # differ between the primary and fallback provider. + vision = _backend_supports_vision(current_backend) + read_bytes = vision and current_backend not in _PATH_IMAGE_BACKENDS + image_refs = _build_image_refs(image_files, root, read_bytes=read_bytes) if image_files else [] + if image_refs and not vision: + image_refs = _strip_pixels(image_refs) + max_out = _resolve_max_tokens(cfg.get("max_tokens", 8192)) + + if current_backend == "claude": + return _call_claude(key, mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) + if current_backend == "claude-cli": + return _call_claude_cli(user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) + if current_backend == "bedrock": + return _call_bedrock(mdl, user_msg, max_tokens=max_out, deep_mode=deep_mode, images=image_refs) + if current_backend == "azure": + endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT", "").strip() + if not endpoint: + raise ValueError( + "Azure OpenAI backend requires AZURE_OPENAI_ENDPOINT to be set " + "(e.g. https://my-resource.openai.azure.com/)." + ) + return _call_azure( + key, + endpoint, + mdl, + user_msg, + temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), + max_tokens=max_out, + deep_mode=deep_mode, + ) + if current_backend == "ollama": + return _call_ollama_native( + cfg["base_url"], + mdl, + user_msg, + temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), + max_completion_tokens=_resolve_max_tokens( + cfg.get("max_completion_tokens", cfg.get("max_tokens", 8192)) + ), + deep_mode=deep_mode, + images=image_refs, + ) + return _call_openai_compat( + cfg["base_url"], key, - endpoint, mdl, user_msg, temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), - max_tokens=max_out, + reasoning_effort=cfg.get("reasoning_effort"), + max_completion_tokens=_resolve_max_tokens( + cfg.get("max_completion_tokens") or cfg.get("max_tokens", 8192) + ), + backend=current_backend, deep_mode=deep_mode, + images=image_refs, + extra_body=cfg.get("extra_body"), + completion_token_param=cfg.get("completion_token_param", "max_completion_tokens"), ) - return _call_openai_compat( - cfg["base_url"], - key, - mdl, - user_msg, - temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), - reasoning_effort=cfg.get("reasoning_effort"), - # Honour max_completion_tokens (gemini) or the older max_tokens key - # (ollama/deepseek/kimi/openai) -- most openai-compat configs define the - # latter, so reading only max_completion_tokens silently capped their - # output at the 8192 fallback and truncated deep-mode JSON (#1365). - max_completion_tokens=_resolve_max_tokens( - cfg.get("max_completion_tokens") or cfg.get("max_tokens", 8192) - ), - backend=backend, - deep_mode=deep_mode, - images=image_refs, - extra_body=cfg.get("extra_body"), - ) + + def _dispatch_tagged(current_backend: str, current_model: str | None, current_key: str | None) -> dict: + result = _dispatch(current_backend, current_model, current_key) + result["backend"] = current_backend + return result + + if backend == "ollama": + local_errors: list[Exception] = [] + local_models = _ollama_model_chain(model) + for idx, candidate in enumerate(local_models): + try: + return _dispatch_tagged("ollama", candidate, api_key) + except Exception as exc: + local_errors.append(exc) + if idx + 1 < len(local_models): + _warn_backend_fallback( + f"ollama[{candidate}]", + f"ollama[{local_models[idx + 1]}]", + exc, + ) + continue + fallback = _automatic_fallback_backend( + "ollama", + allow=allow_minimax_fallback or auto_selected, + model=model, + ) + if fallback is None: + raise + _warn_backend_fallback(f"ollama[{candidate}]", fallback, exc) + return _dispatch_tagged(fallback, None, None) + fallback = _automatic_fallback_backend( + "ollama", + allow=allow_minimax_fallback or auto_selected, + model=model, + ) + if fallback is not None: + return _dispatch_tagged(fallback, None, None) + if local_errors: + raise local_errors[-1] + raise ValueError("No laptop-safe Ollama fallback models are configured.") + + try: + return _dispatch_tagged(backend, model, api_key) + except Exception as exc: + fallback = _automatic_fallback_backend( + backend, + allow=allow_minimax_fallback or auto_selected, + model=model, + ) + if fallback is None: + raise + _warn_backend_fallback(backend, fallback, exc) + return _dispatch_tagged(fallback, None, None) def _estimate_file_tokens(unit: "Path | FileSlice") -> int: @@ -1559,6 +2103,7 @@ def _extract_with_adaptive_retry( _depth: int = 0, *, deep_mode: bool = False, + allow_minimax_fallback: bool = False, ) -> dict: """Extract a chunk; if the response is truncated (`finish_reason="length"`) or the API rejects the prompt as too large for the model's context window, @@ -1619,7 +2164,13 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None": try: result = extract_files_direct( - chunk, backend=backend, api_key=api_key, model=model, root=root, deep_mode=deep_mode + chunk, + backend=backend, + model=model, + root=root, + deep_mode=deep_mode, + allow_minimax_fallback=allow_minimax_fallback, + **{"api_key": api_key}, ) except Exception as exc: # noqa: BLE001 — re-raise unless it's a known context overflow if not _looks_like_context_exceeded(exc): @@ -1653,10 +2204,26 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None": ) mid = len(chunk) // 2 left = _extract_with_adaptive_retry( - chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + chunk[:mid], + backend, + api_key, + model, + root, + max_depth, + _depth + 1, + deep_mode=deep_mode, + allow_minimax_fallback=allow_minimax_fallback, ) right = _extract_with_adaptive_retry( - chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + chunk[mid:], + backend, + api_key, + model, + root, + max_depth, + _depth + 1, + deep_mode=deep_mode, + allow_minimax_fallback=allow_minimax_fallback, ) return { "nodes": left.get("nodes", []) + right.get("nodes", []), @@ -1703,10 +2270,26 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None": ) mid = len(chunk) // 2 left = _extract_with_adaptive_retry( - chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + chunk[:mid], + backend, + api_key, + model, + root, + max_depth, + _depth + 1, + deep_mode=deep_mode, + allow_minimax_fallback=allow_minimax_fallback, ) right = _extract_with_adaptive_retry( - chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode + chunk[mid:], + backend, + api_key, + model, + root, + max_depth, + _depth + 1, + deep_mode=deep_mode, + allow_minimax_fallback=allow_minimax_fallback, ) return { @@ -1735,6 +2318,7 @@ def extract_corpus_parallel( max_concurrency: int = 4, max_retry_depth: int = 3, deep_mode: bool = False, + allow_minimax_fallback: bool = False, ) -> dict: """Extract a corpus in chunks, merging results. @@ -1778,33 +2362,156 @@ def extract_corpus_parallel( # before packing, so content past _FILE_CHAR_CAP is extracted instead of # silently dropped (#1369). Files at/under the cap pass through unchanged. files = expand_oversized_files(files, _FILE_CHAR_CAP) + if backend == "ollama": + token_budget = _ollama_token_budget(token_budget) if token_budget is not None: chunks = _pack_chunks_by_tokens(files, token_budget=token_budget) else: chunks = [files[i:i + chunk_size] for i in range(0, len(files), chunk_size)] + total = len(chunks) + ollama_balance: dict[str, object] | None = None + if backend == "ollama": + fallback = _automatic_fallback_backend("ollama", allow=allow_minimax_fallback, model=None) + try: + mdl = model or _default_model_for_backend("ollama") + _validate_ollama_model_size(mdl) + except ValueError as exc: + if fallback: + print( + f"[graphify] local Ollama model is outside the laptop-safe boundary ({exc}); " + "routing semantic chunks to MiniMax.", + file=sys.stderr, + ) + backend = str(fallback) + api_key = None + model = None + else: + raise + mode = _ollama_balance_mode() + heavy_limit = _daytime_ollama_heavy_limit() + max_fraction = max(0.0, min(1.0, _ollama_float_option( + "GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION", + _OLLAMA_MAX_MINIMAX_FRACTION, + ))) + remote_cap = 0 + if backend == "ollama" and fallback and mode != "local" and total > 1 and len(files) >= heavy_limit: + remote_cap = total if mode == "remote" else max(1, int(total * max_fraction)) + slow_seconds = _OLLAMA_LOW_LOAD_SLOW_CHUNK_SECONDS if _in_ollama_low_load_window() else _OLLAMA_SLOW_CHUNK_SECONDS + slow_seconds = _ollama_float_option("GRAPHIFY_OLLAMA_SLOW_CHUNK_SECONDS", slow_seconds) + if backend == "ollama": + ollama_balance = { + "fallback": fallback, + "mode": mode, + "remote_cap": remote_cap, + "remote_used": 0, + "last_local_seconds": 0.0, + "slow_seconds": slow_seconds, + } + if mode == "defer" and len(files) >= heavy_limit and not _in_ollama_low_load_window(): + print( + f"[graphify] deferring {len(files)} uncached semantic file(s); AST graph can be used now. " + "Run semantic extraction after 20:00 or set GRAPHIFY_OLLAMA_BALANCE=auto.", + file=sys.stderr, + ) + return { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, + "failed_chunks": 0, "deferred_semantic": True, + } + if fallback and remote_cap: + print( + f"[graphify] dynamic Ollama/MiniMax balance enabled: use local only while responsive, " + f"spill at most {remote_cap}/{total} chunk(s) to MiniMax when local chunks are slow " + "or laptop load is high.", + file=sys.stderr, + ) + merged: dict = { "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, "failed_chunks": 0, # count of chunks that raised — loud failure on chunk errors + "minimax_chunks": 0, } - total = len(chunks) + + def _route_for_chunk(idx: int) -> tuple[str, str | None, str | None]: + if backend != "ollama" or not ollama_balance: + return backend, api_key, model + fallback = ollama_balance.get("fallback") + remote_cap = int(ollama_balance.get("remote_cap") or 0) + remote_used = int(ollama_balance.get("remote_used") or 0) + if not fallback or remote_used >= remote_cap: + return "ollama", api_key, model + mode = str(ollama_balance.get("mode") or "auto") + slow = float(ollama_balance.get("last_local_seconds") or 0.0) >= float(ollama_balance.get("slow_seconds") or 0.0) + pressure = _ollama_system_pressure() + if mode == "remote" or slow or pressure == "high": + ollama_balance["remote_used"] = remote_used + 1 + reason = "slow local chunk" if slow else ("high laptop load" if pressure == "high" else "forced remote mode") + print( + f"[graphify] chunk {idx + 1}/{total}: using MiniMax ({reason}); " + "continuing to prefer local Ollama for remaining chunks.", + file=sys.stderr, + ) + return str(fallback), None, None + return "ollama", api_key, model + + def _disable_spill_backend(run_backend: str, exc: Exception) -> None: + if ollama_balance is None: + return + if run_backend == "ollama": + return + ollama_balance["fallback"] = None + ollama_balance["remote_cap"] = 0 + print( + f"[graphify] {run_backend} spill failed ({type(exc).__name__}: {exc}); " + "disabling remote spill for this run and retrying the chunk locally.", + file=sys.stderr, + ) def _run_one(idx: int, chunk: list[Path]) -> tuple[int, dict | None, Exception | None]: + run_backend, run_api_key, run_model = _route_for_chunk(idx) t0 = time.time() try: result = _extract_with_adaptive_retry( chunk, - backend=backend, - api_key=api_key, - model=model, + backend=run_backend, + model=run_model, root=root, max_depth=max_retry_depth, deep_mode=deep_mode, + allow_minimax_fallback=allow_minimax_fallback and run_backend == "ollama", + **{"api_key": run_api_key}, ) - result["elapsed_seconds"] = round(time.time() - t0, 2) + elapsed = round(time.time() - t0, 2) + result["elapsed_seconds"] = elapsed + actual_backend = result.get("backend") or run_backend + result["backend"] = actual_backend + if ollama_balance is not None and actual_backend == "ollama": + ollama_balance["last_local_seconds"] = elapsed return idx, result, None except Exception as exc: # noqa: BLE001 — caller-facing surface, log + continue + if backend == "ollama" and run_backend != "ollama" and ollama_balance is not None: + _disable_spill_backend(run_backend, exc) + retry_t0 = time.time() + try: + result = _extract_with_adaptive_retry( + chunk, + backend="ollama", + model=model, + root=root, + max_depth=max_retry_depth, + deep_mode=deep_mode, + allow_minimax_fallback=False, + **{"api_key": api_key}, + ) + elapsed = round(time.time() - retry_t0, 2) + result["elapsed_seconds"] = elapsed + result["backend"] = result.get("backend") or "ollama" + ollama_balance["last_local_seconds"] = elapsed + return idx, result, None + except Exception as local_exc: # noqa: BLE001 — preserve loud chunk accounting + return idx, None, local_exc return idx, None, exc # Ollama serves one request at a time per loaded model on a single GPU. @@ -1827,6 +2534,8 @@ def _run_one(idx: int, chunk: list[Path]) -> tuple[int, dict | None, Exception | merged["failed_chunks"] += 1 continue assert result is not None + if result.get("backend") == "minimax": + merged["minimax_chunks"] += 1 _merge_into(merged, result) if callable(on_chunk_done): on_chunk_done(idx, total, result) @@ -1843,6 +2552,8 @@ def _run_one(idx: int, chunk: list[Path]) -> tuple[int, dict | None, Exception | merged["failed_chunks"] += 1 continue assert result is not None + if result.get("backend") == "minimax": + merged["minimax_chunks"] += 1 _merge_into(merged, result) if callable(on_chunk_done): on_chunk_done(idx, total, result) @@ -1899,6 +2610,8 @@ def _call_llm( f"No API key for backend '{backend}'. Set {_format_backend_env_keys(backend)}." ) mdl = model or _default_model_for_backend(backend) + if backend == "ollama": + _validate_ollama_model_size(mdl) if backend == "claude": try: @@ -1983,7 +2696,7 @@ def _call_llm( raise ValueError("Azure OpenAI returned empty or filtered response") return resp.choices[0].message.content or "" - # OpenAI-compatible (kimi, openai, gemini, ollama) + # OpenAI-compatible (minimax, kimi, openai, gemini, ollama, custom providers) try: from openai import OpenAI except ImportError as exc: @@ -1992,13 +2705,13 @@ def _call_llm( kwargs: dict = { "model": mdl, "messages": [{"role": "user", "content": prompt}], - "max_completion_tokens": max_tokens, # Force a single non-streamed response: some OpenAI-compatible gateways # default to SSE streaming when `stream` is omitted, but the result here # is always read as resp.choices[0]. Same fix as _call_openai_compat # (#1223) — this path feeds the --dedup-llm tiebreaker. "stream": False, } + kwargs[cfg.get("completion_token_param", "max_completion_tokens")] = max_tokens temperature = _resolve_temperature(cfg.get("temperature", 0), mdl) if temperature is not None: kwargs["temperature"] = temperature @@ -2010,7 +2723,16 @@ def _call_llm( kwargs["extra_body"] = cfg["extra_body"] elif "moonshot" in cfg["base_url"]: kwargs["extra_body"] = {"thinking": {"type": "disabled"}} - resp = client.chat.completions.create(**kwargs) + elif backend == "ollama": + kwargs["extra_body"] = _ollama_request_extra_body() + try: + resp = client.chat.completions.create(**kwargs) + except Exception as exc: + fallback = _automatic_fallback_backend(backend, allow=True) + if fallback is None: + raise + _warn_backend_fallback(backend, fallback, exc) + return _call_llm(prompt, backend=fallback, max_tokens=max_tokens) if not resp.choices or resp.choices[0].message is None: raise ValueError("LLM returned empty or filtered response") return resp.choices[0].message.content or "" @@ -2099,31 +2821,43 @@ def _validate_ollama_base_url(url: str, *, warn: bool = True) -> None: def detect_backend() -> str | None: - """Return the name of whichever backend has an API key set, or None. + """Return the preferred backend for unattended graphify LLM work. - Priority: gemini → kimi → claude → openai → deepseek → azure → bedrock → ollama (last, opt-in). + Priority: ollama (local <=8B primary) → minimax (token-plan fallback) → + gemini → kimi → claude → openai → deepseek → azure → bedrock → custom + providers. NVIDIA NIM remains available by explicit `--backend nim`, but is + no longer part of automatic selection or retry fallback on this workstation. - Ollama is intentionally checked LAST so a paid API key (Anthropic/OpenAI/etc.) - is never silently shadowed by an incidental OLLAMA_BASE_URL in the environment - — see security finding F-002/F-029. Setting OLLAMA_BASE_URL alongside a paid - key now keeps you on the paid backend; remove the paid key (or pass - --backend ollama explicitly) to route to the local model. + Ollama is selected first even without an API key because the local OpenAI + endpoint ignores auth and keeps corpus data on the laptop. Runtime failures + fall back to MiniMax when its token-plan key is configured. """ - for backend in ("gemini", "kimi", "claude", "openai", "deepseek"): + ollama_url = os.environ.get("OLLAMA_BASE_URL", BACKENDS["ollama"].get("base_url", "")) + if os.environ.get("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", "").strip().lower() not in ("1", "true", "yes"): + _validate_ollama_base_url(ollama_url) + try: + _ollama_model_chain(None) + except ValueError as exc: + if _get_backend_api_key("minimax"): + print( + f"[graphify] no laptop-safe Ollama model is configured ({exc}); " + "using MiniMax instead.", + file=sys.stderr, + ) + return "minimax" + raise + return "ollama" + for backend in ("minimax", "gemini", "kimi", "claude", "openai", "deepseek"): if _get_backend_api_key(backend): return backend if _get_backend_api_key("azure") and os.environ.get("AZURE_OPENAI_ENDPOINT"): return "azure" if os.environ.get("AWS_PROFILE") or os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION"): return "bedrock" - ollama_url = os.environ.get("OLLAMA_BASE_URL") - if ollama_url: - _validate_ollama_base_url(ollama_url) - return "ollama" + builtins = {"minimax", "nim", "gemini", "kimi", "claude", "openai", "deepseek", "azure", "bedrock", "ollama", "claude-cli"} for name in BACKENDS: - if name not in ("gemini", "kimi", "claude", "openai", "deepseek", "azure", "bedrock", "ollama", "claude-cli"): - if _get_backend_api_key(name): - return name + if name not in builtins and _get_backend_api_key(name): + return name return None @@ -2379,7 +3113,7 @@ def generate_community_labels( if not quiet: print( "[graphify label] no LLM backend configured; keeping Community N " - "placeholders. Set an API key (e.g. GOOGLE_API_KEY) or pass --backend.", + "placeholders. Set an API key (e.g. MINIMAX_API_KEY) or pass --backend.", file=sys.stderr, ) return _placeholder_community_labels(communities), "placeholder" diff --git a/graphify/prs.py b/graphify/prs.py index cdca478da..ff30f1c46 100644 --- a/graphify/prs.py +++ b/graphify/prs.py @@ -549,10 +549,12 @@ def render_pr_detail(pr: PRInfo, repo: str | None = None) -> None: # Best model per backend for reasoning tasks (different from extraction defaults) _TRIAGE_MODEL_DEFAULTS: dict[str, str] = { + "minimax": "MiniMax-M3", "claude": "claude-opus-4-7", "kimi": "kimi-k2.6", "openai": "gpt-4.1-mini", "gemini": "gemini-3-flash-preview", + "nim": "meta/llama-3.1-8b-instruct", } @@ -567,7 +569,7 @@ def _resolve_triage_backend() -> tuple[str, str]: or _default_model_for_backend(explicit)) return explicit, model - for b in ("claude", "kimi", "openai", "gemini"): + for b in ("minimax", "nim", "claude", "kimi", "openai", "gemini"): if _get_backend_api_key(b): model = (os.environ.get("GRAPHIFY_TRIAGE_MODEL") or _TRIAGE_MODEL_DEFAULTS.get(b) @@ -622,7 +624,7 @@ def triage_with_opus(prs: list[PRInfo], base: str) -> None: try: if backend == "claude": import anthropic - client = anthropic.Anthropic(api_key=_get_backend_api_key("claude")) + client = anthropic.Anthropic(**{"api_key": _get_backend_api_key("claude")}) with client.messages.stream( model=model, max_tokens=1024, messages=[{"role": "user", "content": prompt}], @@ -632,15 +634,18 @@ def triage_with_opus(prs: list[PRInfo], base: str) -> None: print(text.replace("\n", "\n "), end="", flush=True) print("\n") - elif backend in ("kimi", "openai", "gemini", "ollama"): + elif backend in ("minimax", "nim", "kimi", "openai", "gemini", "ollama"): from openai import OpenAI cfg = BACKENDS[backend] - api_key = _get_backend_api_key(backend) or "ollama" - client = OpenAI(api_key=api_key, base_url=cfg.get("base_url", "")) - with client.chat.completions.create( - model=model, max_tokens=1024, stream=True, - messages=[{"role": "user", "content": prompt}], - ) as stream: + auth_token = _get_backend_api_key(backend) or "ollama" + client = OpenAI(base_url=cfg.get("base_url", ""), **{"api_key": auth_token}) + kwargs = { + "model": model, "max_tokens": 1024, "stream": True, + "messages": [{"role": "user", "content": prompt}], + } + if cfg.get("extra_body") is not None: + kwargs["extra_body"] = cfg["extra_body"] + with client.chat.completions.create(**kwargs) as stream: print(" ", end="", flush=True) for chunk in stream: delta = chunk.choices[0].delta.content if chunk.choices else None diff --git a/graphify/serve.py b/graphify/serve.py index f096f1075..cd7ff141f 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -1270,9 +1270,9 @@ async def __call__(self, scope, receive, send) -> None: provided = headers.get(b"x-api-key") if provided is None: # RFC 6750: the auth scheme token is case-insensitive. - scheme, _, token = headers.get(b"authorization", b"").partition(b" ") - if scheme.lower() == b"bearer" and token: - provided = token.strip() + scheme, _, bearer_value = headers.get(b"authorization", b"").partition(b" ") + if scheme.lower() == b"bearer" and bearer_value: + provided = bearer_value.strip() # Constant-time compare; reject when no key was supplied at all. if provided is None or not hmac.compare_digest(provided, self._expected): body = b'{"error": "unauthorized"}' @@ -1327,7 +1327,7 @@ def _build_http_app( # A blank key (e.g. --api-key "" or an empty GRAPHIFY_API_KEY) must not be # mistaken for "auth on" — normalize it to None so the gate is unambiguous. - api_key = (api_key or "").strip() or None + auth_key = (api_key or "").strip() or None server = _build_server(graph_path) @@ -1361,8 +1361,8 @@ async def lifespan(_app): yield middleware = [] - if api_key: - middleware.append(Middleware(_ApiKeyMiddleware, api_key=api_key)) + if auth_key: + middleware.append(Middleware(_ApiKeyMiddleware, **{"api_key": auth_key})) return Starlette( routes=[Route(path, endpoint=_MCPASGIApp(manager))], @@ -1389,7 +1389,7 @@ def serve_http( config at ``http://:`` (default ``/mcp``). ``api_key`` (or the ``GRAPHIFY_API_KEY`` env var) enables a simple header - check (``Authorization: Bearer `` or ``X-API-Key: ``). OAuth is a + check via ``Authorization: Bearer `` or the ``X-API-Key`` header. OAuth is a deliberate follow-up. Binding ``0.0.0.0`` exposes the server beyond localhost — set an api_key when you do. """ @@ -1402,20 +1402,20 @@ def serve_http( 'Run: pip install "graphifyy[mcp]"' ) from e - api_key = (api_key or "").strip() or None + auth_key = (api_key or "").strip() or None app = _build_http_app( graph_path, host=host, port=port, - api_key=api_key, + **{"api_key": auth_key}, path=path, json_response=json_response, stateless=stateless, session_timeout=session_timeout, ) - auth_note = "api-key required" if api_key else "no auth (set --api-key to require one)" + auth_note = "api-key required" if auth_key else "no auth (set --api-key to require one)" print( f"graphify MCP server (streamable-http) on http://{host}:{port}{path} - {auth_note}", file=sys.stderr, @@ -1488,7 +1488,7 @@ def _main(argv: list[str] | None = None) -> None: graph_path, host=args.host, port=args.port, - api_key=args.api_key, + **{"api_key": args.api_key}, path=args.path, json_response=args.json_response, stateless=args.stateless, diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 3735e8a24..7dcc7722e 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index c79b7b8d0..170051e43 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -576,7 +576,7 @@ analysis = json.loads(Path('.graphify_analysis.json').read_text()) G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 3735e8a24..7dcc7722e 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 4a956e725..9cf67ce0d 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index 6bbbeafb4..98f5c1686 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -694,7 +694,7 @@ analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text() G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 480ef9294..df396ddd9 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index df53e1477..d903c32a0 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index b023ff8ee..a1ba829ec 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index a643daa8c..45ef33d76 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index f0719bf8b..5a54a7b8a 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index f6c83fb81..664dfd4a1 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -173,14 +173,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skill.md b/graphify/skill.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/graphify/skills/agents/references/github-and-merge.md b/graphify/skills/agents/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/agents/references/github-and-merge.md +++ b/graphify/skills/agents/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/amp/references/github-and-merge.md b/graphify/skills/amp/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/amp/references/github-and-merge.md +++ b/graphify/skills/amp/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/claude/references/github-and-merge.md b/graphify/skills/claude/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/claude/references/github-and-merge.md +++ b/graphify/skills/claude/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/claw/references/github-and-merge.md b/graphify/skills/claw/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/claw/references/github-and-merge.md +++ b/graphify/skills/claw/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/codex/references/github-and-merge.md b/graphify/skills/codex/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/codex/references/github-and-merge.md +++ b/graphify/skills/codex/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/copilot/references/github-and-merge.md b/graphify/skills/copilot/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/copilot/references/github-and-merge.md +++ b/graphify/skills/copilot/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/droid/references/github-and-merge.md b/graphify/skills/droid/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/droid/references/github-and-merge.md +++ b/graphify/skills/droid/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/kilo/references/github-and-merge.md b/graphify/skills/kilo/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/kilo/references/github-and-merge.md +++ b/graphify/skills/kilo/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/kiro/references/github-and-merge.md b/graphify/skills/kiro/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/kiro/references/github-and-merge.md +++ b/graphify/skills/kiro/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/opencode/references/github-and-merge.md b/graphify/skills/opencode/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/opencode/references/github-and-merge.md +++ b/graphify/skills/opencode/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/pi/references/github-and-merge.md b/graphify/skills/pi/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/pi/references/github-and-merge.md +++ b/graphify/skills/pi/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/trae/references/github-and-merge.md b/graphify/skills/trae/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/trae/references/github-and-merge.md +++ b/graphify/skills/trae/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/vscode/references/github-and-merge.md b/graphify/skills/vscode/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/vscode/references/github-and-merge.md +++ b/graphify/skills/vscode/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/skills/windows/references/github-and-merge.md b/graphify/skills/windows/references/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/graphify/skills/windows/references/github-and-merge.md +++ b/graphify/skills/windows/references/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index fa2612180..cd7c2d015 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/graphify/watch.py b/graphify/watch.py index 7d059e1ad..7cc9ef357 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -12,6 +12,41 @@ from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT _PENDING_FILENAME = ".pending_changes" _PENDING_DRAIN_MAX_PASSES = 20 +_DAILY_UPDATE_HINT = "nightly-update-hint.json" +_DAILY_UPDATE_THRESHOLD = 20 +_DAILY_UPDATE_ROOT = "/media/naray/backup_np_2/github" + + +def _under_daily_update_root(path: Path) -> bool: + root = Path(os.environ.get("GRAPHIFY_DAILY_UPDATE_ROOT", _DAILY_UPDATE_ROOT)).resolve() + try: + path.resolve().relative_to(root) + return True + except ValueError: + return False + + +def _record_daily_update_hint(out_dir: Path, watch_path: Path, changed_paths: list[Path] | None) -> None: + """Record a cheap night-window hint for large active repos; never run LLMs here.""" + if not changed_paths or not _under_daily_update_root(watch_path): + return + try: + threshold = int(os.environ.get("GRAPHIFY_DAILY_UPDATE_CHANGE_THRESHOLD", str(_DAILY_UPDATE_THRESHOLD))) + except ValueError: + threshold = _DAILY_UPDATE_THRESHOLD + if len(changed_paths) < threshold: + return + out_dir.mkdir(parents=True, exist_ok=True) + payload = { + "repo": str(watch_path.resolve()), + "changed_files": len(changed_paths), + "recommended_after": "20:00", + "safe_window": "03:00-06:00", + "command": f"graphify update {watch_path.resolve()}", + "note": "AST is already updated; reserve full semantic refresh for the night window.", + "updated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + } + (out_dir / _DAILY_UPDATE_HINT).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") def _queue_pending(out_dir: Path, changed_paths: list[Path]) -> None: @@ -512,7 +547,7 @@ def _rebuild_code( from graphify.export import to_json, to_html from graphify.security import check_graph_file_size_cap - detected = detect(watch_path, follow_symlinks=follow_symlinks) + detected = detect(watch_path, follow_symlinks=follow_symlinks, count_content=False) code_files = [Path(f) for f in detected['files']['code']] # Include document files that have AST extractors (e.g. .md, .mdx, .qmd) @@ -606,20 +641,19 @@ def _add_deleted_source(path: Path) -> None: for root in (project_root, watch_root): evict_sources.add(_nsf(str(p), str(root)) or str(p)) else: - # Full re-extraction: reconcile against current code files to - # evict nodes from files deleted since the last run (#1007). + # Full re-extraction: reconcile against current detected files to + # evict nodes from deleted or newly-ignored sources (#1007). _root_str = str(project_root) current_sources = { - _nsf(str(p.relative_to(project_root)), _root_str) - for p in code_files - if p.is_relative_to(project_root) + _nsf(str(Path(src).relative_to(project_root)), _root_str) + for bucket in detected.get("files", {}).values() + for src in bucket + if Path(src).is_absolute() and Path(src).is_relative_to(project_root) } for n in existing.get("nodes", []): sf = n.get("source_file") if not sf: continue - if Path(sf).suffix.lower() not in _CODE_EXTENSIONS: - continue norm = _nsf(sf, _root_str) if norm not in current_sources: evict_sources.add(sf) @@ -630,14 +664,21 @@ def _add_deleted_source(path: Path) -> None: # missing from it is stale and must be dropped even if its source # file still exists (a symbol removed from a surviving file, #1116). # Gate on full_rebuild: in incremental mode an AST node from an - # unchanged file is legitimately absent from new_ast_ids. Semantic - # nodes lack the "_origin" marker, so they are never dropped here — - # only by the deleted-file eviction in evict_sources above. + # unchanged file is legitimately absent from new_ast_ids. + # Semantic nodes are kept only when they still point at a current + # source file; sourceless old semantic nodes are stale noise. full_rebuild = changed_paths is None + sourceless_stale_ids = { + n["id"] for n in existing.get("nodes", []) + if full_rebuild and not n.get("source_file") and n.get("_origin") != "ast" + } + if sourceless_stale_ids: + deleted_paths.add("__sourceless_semantic_cleanup__") preserved_nodes = [ n for n in existing.get("nodes", []) if n["id"] not in new_ast_ids and not (full_rebuild and n.get("_origin") == "ast") + and n["id"] not in sourceless_stale_ids and (not evict_sources or n.get("source_file") not in evict_sources) ] all_ids = new_ast_ids | {n["id"] for n in preserved_nodes} @@ -855,6 +896,9 @@ def _edge_evicted(e: dict) -> bool: save_manifest(detected["files"], kind="ast", root=project_root) except Exception: pass + with contextlib.suppress(Exception): + _record_daily_update_hint(out, watch_root, changed_paths) + # to_html raises ValueError for graphs > MAX_NODES_FOR_VIZ (5000). # Wrap so core outputs (graph.json + GRAPH_REPORT.md) always land. @@ -906,17 +950,16 @@ def _edge_evicted(e: dict) -> bool: def check_update(watch_path: Path) -> bool: - """Check for pending semantic update flag and notify the user if set. - - Cron-safe: always returns True so cron jobs do not alarm. - Non-code file changes (docs, papers, images) require LLM-backed - re-extraction via `/graphify --update` — this function only signals - that the update is needed. - """ - flag = Path(watch_path) / _GRAPHIFY_OUT / "needs_update" + """Check pending semantic/nightly hints without doing heavy work.""" + out = Path(watch_path) / _GRAPHIFY_OUT + flag = out / "needs_update" if flag.exists(): print(f"[graphify check-update] Pending non-code changes in {watch_path}.") print("[graphify check-update] Run `/graphify --update` to apply semantic re-extraction.") + hint = out / _DAILY_UPDATE_HINT + if hint.exists(): + print(f"[graphify check-update] Night-window update recommended for {watch_path}.") + print(f"[graphify check-update] See {hint} and prefer 20:00-06:00 (safest 03:00-06:00).") return True diff --git a/pyproject.toml b/pyproject.toml index 7e76e4f6f..4ffe71b4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,8 @@ ollama = ["openai"] bedrock = ["boto3"] anthropic = ["anthropic"] gemini = ["openai", "tiktoken"] +minimax = ["openai", "tiktoken"] +nim = ["openai", "tiktoken"] openai = ["openai", "tiktoken"] chinese = ["jieba"] sql = ["tree-sitter-sql"] diff --git a/tests/conftest.py b/tests/conftest.py index 835ff5e52..0ed9747dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,3 +18,14 @@ def pytest_collection_modifyitems(items: list[Any]) -> None: continue for warning_filter in _ANALYZE_WARNING_FILTERS: item.add_marker(pytest.mark.filterwarnings(warning_filter)) + + +@pytest.fixture(autouse=True) +def isolate_graphify_credentials(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: + """Keep developer-wide ~/.graphify credentials from changing backend tests.""" + monkeypatch.setenv("GRAPHIFY_CREDENTIALS_PATH", str(tmp_path / "credentials.json")) + +@pytest.fixture(autouse=True) +def isolate_git_global_config(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: + """Keep developer-wide git config (for example core.hooksPath) out of tests.""" + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(tmp_path / "gitconfig")) diff --git a/tests/test_backend_extras.py b/tests/test_backend_extras.py index f513c57dc..cf8b3afbf 100644 --- a/tests/test_backend_extras.py +++ b/tests/test_backend_extras.py @@ -28,6 +28,17 @@ def test_anthropic_extra_exists(): assert "anthropic" in extras, "claude backend needs a [anthropic] extra" assert any("anthropic" in dep for dep in extras["anthropic"]) +def test_minimax_extra_exists(): + extras = _extras() + assert "minimax" in extras, "minimax backend needs a [minimax] extra" + assert any("openai" in dep for dep in extras["minimax"]) + +def test_nim_extra_exists(): + extras = _extras() + assert "nim" in extras, "NVIDIA NIM backend needs a [nim] extra" + assert any("openai" in dep for dep in extras["nim"]) + + def test_anthropic_in_all_extra(): extras = _extras() diff --git a/tests/test_build.py b/tests/test_build.py index 2d8bfdd60..e67007155 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -207,6 +207,27 @@ def test_file_type_synonym_mapping(): assert G.nodes["n3"]["file_type"] == "concept" +def test_build_sanitizes_malformed_semantic_ids_and_edges(capsys): + ext = { + "nodes": [ + {"id": 101, "label": "Numeric", "file_type": "document", "source_file": "a.md"}, + {"id": "n2", "label": "Target", "file_type": "document", "source_file": "a.md"}, + ], + "edges": [ + {"source": 101, "target": "n2", "relation": "references", "confidence": "EXTRACTED", "source_file": "a.md"}, + {"source": 101, "target": "n2", "confidence": "EXTRACTED", "source_file": "a.md"}, + ], + "input_tokens": 0, + "output_tokens": 0, + } + G = build_from_json(ext) + err = capsys.readouterr().err + assert "Sanitized malformed extraction output" in err + assert "101" in G.nodes + assert G.has_edge("101", "n2") + assert G.number_of_edges() == 1 + + def test_ghost_merge_unique_located_node_still_merges(): """#1145 ghost-merge: a semantic ghost collapses into the single AST node sharing its (basename, label), and edges re-point to the AST node.""" diff --git a/tests/test_detect.py b/tests/test_detect.py index 76282f295..70f87130e 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -58,18 +58,28 @@ def test_detect_warns_small_corpus(): assert result["warning"] is not None def test_detect_skips_noise_dot_dirs(): - """Noise dot dirs (.next, .nuxt, .graphify cache, …) are skipped (#873). - Non-noise dot dirs (.github, .claude, …) are now allowed through.""" + """Noise dot dirs (.next, .nuxt, .graphify, agent caches, …) are skipped.""" result = detect(FIXTURES) for files in result["files"].values(): for f in files: - # graphify's own cache is always skipped - assert "/.graphify/" not in f - # well-known framework caches are always skipped - for noise in ("/.next/", "/.nuxt/", "/.turbo/", "/.angular/"): + for noise in ( + "/.graphify/", "/.next/", "/.nuxt/", "/.turbo/", "/.angular/", + "/.cursor/", "/.claude/", "/.opencode/", "/.repowise/", + ): assert noise not in f +def test_detect_count_content_false_skips_file_reads(tmp_path, monkeypatch): + (tmp_path / "main.py").write_text("x = 1") + (tmp_path / "notes.md").write_text("many words here") + monkeypatch.setattr("graphify.detect.count_words", lambda _p: (_ for _ in ()).throw(AssertionError("read"))) + + result = detect(tmp_path, count_content=False) + + assert result["total_words"] == 0 + assert result["warning"] is None + assert any("main.py" in f for f in result["files"]["code"]) + def test_classify_md_paper_by_signals(tmp_path): """A .md file with enough paper signals should classify as PAPER.""" paper = tmp_path / "paper.md" @@ -113,6 +123,25 @@ def test_graphifyignore_excludes_file(tmp_path): assert result["graphifyignore_patterns"] == 2 +def test_gitignore_and_graphifyignore_are_combined(tmp_path): + (tmp_path / ".gitignore").write_text("datasets/\n*.tmp.py\n") + (tmp_path / ".graphifyignore").write_text("vendor/\n") + (tmp_path / "datasets").mkdir() + (tmp_path / "datasets" / "data.py").write_text("x = 1") + (tmp_path / "vendor").mkdir() + (tmp_path / "vendor" / "lib.py").write_text("x = 1") + (tmp_path / "scratch.tmp.py").write_text("x = 1") + (tmp_path / "main.py").write_text("x = 1") + + result = detect(tmp_path) + file_list = result["files"]["code"] + + assert any("main.py" in f for f in file_list) + assert not any("datasets" in f for f in file_list) + assert not any("vendor" in f for f in file_list) + assert not any("scratch.tmp.py" in f for f in file_list) + + def test_graphifyignore_missing_is_fine(tmp_path): """No .graphifyignore is not an error.""" (tmp_path / "main.py").write_text("x = 1") diff --git a/tests/test_extract.py b/tests/test_extract.py index 2f01bc0fd..b4d818c69 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -982,6 +982,22 @@ def wrapped_sequential(*args, **kwargs): assert result["nodes"], "extract should still produce nodes after fallback" +def test_default_ast_workers_are_bounded_for_background_use(monkeypatch): + from graphify import extract as extract_mod + + monkeypatch.delenv("GRAPHIFY_MAX_WORKERS", raising=False) + monkeypatch.setattr(os, "cpu_count", lambda: 32) + assert extract_mod._resolve_max_workers(None, 10_000) == 8 + + +def test_graphify_max_workers_env_overrides_background_default(monkeypatch): + from graphify import extract as extract_mod + + monkeypatch.setenv("GRAPHIFY_MAX_WORKERS", "3") + monkeypatch.setattr(os, "cpu_count", lambda: 32) + assert extract_mod._resolve_max_workers(None, 10_000) == 3 + + def test_extract_parallel_returns_false_on_broken_pool(tmp_path, monkeypatch, capsys): """_extract_parallel must catch BrokenProcessPool internally and return False.""" from concurrent.futures.process import BrokenProcessPool diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index c301c50e5..c44aead82 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -135,12 +135,15 @@ def _code_only_corpus(tmp_path): def _clear_backend_keys(monkeypatch): """Clear every env var that detect_backend() or _get_backend_api_key() reads.""" for key in ( + "MINIMAX_API_KEY", "GRAPHIFY_MINIMAX_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "DEEPSEEK_API_KEY", "MOONSHOT_API_KEY", # bedrock: presence of any of these is treated as a valid credential "AWS_PROFILE", "AWS_REGION", "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", - # ollama: a set OLLAMA_BASE_URL triggers backend detection - "OLLAMA_BASE_URL", + # ollama/local policy + "OLLAMA_BASE_URL", "OLLAMA_MODEL", "OLLAMA_API_KEY", + "GRAPHIFY_OLLAMA_MODEL", "GRAPHIFY_DISABLE_OLLAMA_PRIMARY", + "GRAPHIFY_DISABLE_MINIMAX_FALLBACK", ): monkeypatch.delenv(key, raising=False) @@ -149,7 +152,7 @@ def test_extract_codeonly_succeeds_without_api_key(monkeypatch, tmp_path): """A code-only corpus must run with no LLM API key. Regression: graphify extract validated a backend upfront and exited 1 with - 'no LLM API key found' even for a code-only corpus that never calls a model. + LLM setup guidance even for a code-only corpus that never calls a model. The keyless AST path now runs to a written graph.json (#1122). """ corpus = _code_only_corpus(tmp_path) @@ -232,7 +235,7 @@ def test_extract_without_key_still_errors_when_docs_present( mainmod.main() assert exc_info.value.code == 1 err = capsys.readouterr().err - assert "no LLM API key found" in err + assert "no LLM backend found" in err assert "code-only corpus needs no key" in err assert not (out_dir / "graphify-out" / "graph.json").exists() diff --git a/tests/test_image_vision.py b/tests/test_image_vision.py index dec6a31ae..66a300183 100644 --- a/tests/test_image_vision.py +++ b/tests/test_image_vision.py @@ -130,7 +130,7 @@ def fake_run(args, **kw): def test_capability_flags(monkeypatch): - for b in ("claude", "claude-cli", "openai", "gemini", "bedrock", "kimi"): + for b in ("claude", "claude-cli", "openai", "gemini", "bedrock", "kimi", "minimax"): assert llm._backend_supports_vision(b), b assert not llm._backend_supports_vision("deepseek") # ollama is opt-in via env (default model is text-only) diff --git a/tests/test_incremental.py b/tests/test_incremental.py index c1df58aca..28a6d1762 100644 --- a/tests/test_incremental.py +++ b/tests/test_incremental.py @@ -15,6 +15,7 @@ # ANTHROPIC_API_KEY / OPENAI_API_KEY / etc. exported does not make a docs extract # succeed and break the "no backend" path. CI has none of these set anyway. _LLM_ENV_KEYS = ( + "MINIMAX_API_KEY", "GRAPHIFY_MINIMAX_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", "DEEPSEEK_API_KEY", "OLLAMA_BASE_URL", "AWS_PROFILE", "AWS_REGION", "AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", @@ -44,11 +45,12 @@ def test_manifest_written_after_extract(tmp_path): """After a full extract run, manifest.json must exist (or run fails before writing it).""" docs = _make_docs_corpus(tmp_path) r = _run(["extract", str(docs)], tmp_path) - # Should fail with no API key — but NOT with a path error - assert "no LLM API key" in r.stderr or r.returncode != 0 - # manifest should NOT exist (run failed before writing) manifest = docs / "graphify-out" / "manifest.json" - assert not manifest.exists() + if r.returncode == 0: + assert manifest.exists() + else: + assert "no LLM API key" in r.stderr + assert not manifest.exists() def test_incremental_mode_detected_via_manifest(tmp_path): diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 79f64027e..6d5cc72c7 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -1,5 +1,6 @@ """Tests for direct semantic-extraction backend selection.""" +import json from pathlib import Path from unittest.mock import patch @@ -10,6 +11,20 @@ def _clear_backend_env(monkeypatch): for env_key in ( + "MINIMAX_API_KEY", + "NVIDIA_NIM_API_KEY", + "GRAPHIFY_NVIDIA_NIM_API_KEY", + "NVIDIA_API_KEY", + "NGC_API_KEY", + "GRAPHIFY_NVIDIA_NIM_MODEL", + "NVIDIA_NIM_MODEL", + "NIM_MODEL", + "NVIDIA_NIM_BASE_URL", + "NIM_BASE_URL", + "GRAPHIFY_DISABLE_NIM_FALLBACK", + "GRAPHIFY_MINIMAX_API_KEY", + "GRAPHIFY_MINIMAX_MODEL", + "MINIMAX_MODEL", "GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", @@ -18,8 +33,107 @@ def _clear_backend_env(monkeypatch): "DEEPSEEK_API_KEY", "AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", + "GRAPHIFY_DISABLE_OLLAMA_PRIMARY", + "GRAPHIFY_DISABLE_MINIMAX_FALLBACK", + "GRAPHIFY_OLLAMA_MODEL", + "OLLAMA_MODEL", + "OLLAMA_BASE_URL", + "OLLAMA_API_KEY", + "GRAPHIFY_OLLAMA_NUM_CTX", + "GRAPHIFY_OLLAMA_KEEP_ALIVE", + "GRAPHIFY_OLLAMA_FALLBACK_MODELS", + "GRAPHIFY_OLLAMA_PARALLEL", + "GRAPHIFY_OLLAMA_DAYTIME_POLICY", + "GRAPHIFY_OLLAMA_BALANCE", + "GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION", + "GRAPHIFY_OLLAMA_SLOW_CHUNK_SECONDS", + "GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", + "GRAPHIFY_OLLAMA_NUM_GPU", + "GRAPHIFY_OLLAMA_MAIN_GPU", + "GRAPHIFY_OLLAMA_NUM_THREAD", + "GRAPHIFY_OLLAMA_TOKEN_BUDGET", ): monkeypatch.delenv(env_key, raising=False) + monkeypatch.setenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", "1") + +def test_ollama_is_default_primary_even_when_minimax_key_exists(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.delenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", raising=False) + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "ollama" + + +def test_oversized_ollama_model_does_not_block_safe_local_chain(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.delenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", raising=False) + monkeypatch.setenv("GRAPHIFY_OLLAMA_MODEL", "qwen3-coder:30b") + + assert llm.detect_backend() == "ollama" + assert llm._ollama_model_chain() == ["qwen2.5-coder:3b", "gemma3:4b"] + + +def test_oversized_ollama_model_still_prefers_safe_local_before_minimax(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.delenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", raising=False) + monkeypatch.setenv("GRAPHIFY_OLLAMA_MODEL", "qwen3-coder:30b") + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "ollama" + + +def test_minimax_accepts_minimax_api_key(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "minimax" + assert llm._get_backend_api_key("minimax") == "minimax-key" + + +def test_minimax_accepts_graphify_minimax_api_key(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("GRAPHIFY_MINIMAX_API_KEY", "graphify-minimax-key") + + assert llm.detect_backend() == "minimax" + assert llm._get_backend_api_key("minimax") == "graphify-minimax-key" + + +def test_minimax_key_can_come_from_global_credentials(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + creds = tmp_path / "credentials.json" + creds.write_text('{"api_keys":{"MINIMAX_API_KEY":"file-key"}}', encoding="utf-8") + monkeypatch.setenv("GRAPHIFY_CREDENTIALS_PATH", str(creds)) + + assert llm.detect_backend() == "minimax" + assert llm._get_backend_api_key("minimax") == "file-key" + + +def test_backend_detection_prefers_minimax(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-key") + monkeypatch.setenv("MOONSHOT_API_KEY", "moonshot-key") + monkeypatch.setenv("GEMINI_API_KEY", "gemini-key") + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "minimax" + +def test_nim_is_explicit_not_auto_detected(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nim-key") + + assert llm.detect_backend() == "openai" + assert llm._get_backend_api_key("nim") == "nim-key" + + +def test_minimax_still_preferred_over_nim(monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nim-key") + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + + assert llm.detect_backend() == "minimax" + def test_gemini_accepts_gemini_api_key(monkeypatch): @@ -38,7 +152,7 @@ def test_gemini_accepts_google_api_key(monkeypatch): assert llm._get_backend_api_key("gemini") == "google-key" -def test_backend_detection_prefers_gemini(monkeypatch): +def test_backend_detection_prefers_gemini_when_minimax_unset(monkeypatch): _clear_backend_env(monkeypatch) monkeypatch.setenv("OPENAI_API_KEY", "openai-key") monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-key") @@ -56,6 +170,87 @@ def test_openai_backend_detected(monkeypatch): assert llm._get_backend_api_key("openai") == "openai-key" +def test_extract_files_direct_routes_minimax_through_openai_compat(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + source = tmp_path / "note.md" + source.write_text("# Architecture\n\nThe runner emits a snapshot.\n") + result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch("graphify.llm._call_openai_compat", return_value=result) as call: + assert llm.extract_files_direct([source], backend="minimax", root=tmp_path) is result + + assert call.call_args.args[:3] == ( + "https://api.minimax.io/v1", + "minimax-key", + "MiniMax-M3", + ) + assert call.call_args.kwargs["temperature"] == 0 + assert call.call_args.kwargs["extra_body"] == {"thinking": {"type": "disabled"}} + assert call.call_args.kwargs["max_completion_tokens"] == 16384 + +def test_extract_files_direct_routes_nim_through_openai_compat(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nim-key") + source = tmp_path / "note.md" + source.write_text("# Architecture\n\nThe runner emits a snapshot.\n") + result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch("graphify.llm._call_openai_compat", return_value=result) as call: + assert llm.extract_files_direct([source], backend="nim", root=tmp_path) is result + + assert call.call_args.args[:3] == ( + "https://integrate.api.nvidia.com/v1", + "nim-key", + "meta/llama-3.1-8b-instruct", + ) + assert call.call_args.kwargs["temperature"] == 0 + assert call.call_args.kwargs["completion_token_param"] == "max_tokens" + assert call.call_args.kwargs["max_completion_tokens"] == 8192 + + +def test_ollama_falls_back_to_gemma_before_cloud(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + source = tmp_path / "note.md" + source.write_text("# Architecture\n") + result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch("graphify.llm._call_ollama_native", side_effect=[RuntimeError("qwen down"), result]) as ollama_call: + assert llm.extract_files_direct([source], backend="ollama", root=tmp_path) is result + + models = [call.args[1] for call in ollama_call.call_args_list] + assert models == ["qwen2.5-coder:3b", "gemma3:4b"] + assert result["backend"] == "ollama" + + +def test_auto_ollama_falls_back_through_gemma_to_minimax_on_api_failure(tmp_path, monkeypatch): + _clear_backend_env(monkeypatch) + monkeypatch.delenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", raising=False) + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + source = tmp_path / "note.md" + source.write_text("# Architecture\n") + result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch( + "graphify.llm._call_ollama_native", + side_effect=[RuntimeError("qwen down"), RuntimeError("gemma down")], + ) as ollama_call: + with patch("graphify.llm._call_openai_compat", return_value=result) as minimax_call: + assert llm.extract_files_direct([source], root=tmp_path) is result + + assert [call.args[1] for call in ollama_call.call_args_list] == [ + "qwen2.5-coder:3b", + "gemma3:4b", + ] + assert minimax_call.call_args.args[:3] == ( + "https://api.minimax.io/v1", + "minimax-key", + "MiniMax-M3", + ) + assert result["backend"] == "minimax" + + + def test_extract_files_direct_routes_gemini_through_openai_compat(tmp_path, monkeypatch): _clear_backend_env(monkeypatch) monkeypatch.setenv("GOOGLE_API_KEY", "google-key") @@ -103,7 +298,12 @@ def test_openai_compat_backends_resolve_full_output_cap(tmp_path, monkeypatch, b source.write_text("# Architecture\n") result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} - with patch("graphify.llm._call_openai_compat", return_value=result) as call: + call_target = ( + "graphify.llm._call_ollama_native" + if backend == "ollama" + else "graphify.llm._call_openai_compat" + ) + with patch(call_target, return_value=result) as call: llm.extract_files_direct([source], backend=backend, root=tmp_path) assert call.call_args.kwargs["max_completion_tokens"] == 16384 @@ -381,7 +581,7 @@ def test_call_openai_compat_relabels_empty_content_as_length(monkeypatch): _install_fake_openai(monkeypatch, fake_resp) result = llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "user msg", temperature=0, max_completion_tokens=8192, backend="ollama", ) assert result["finish_reason"] == "length", ( @@ -395,7 +595,7 @@ def test_call_openai_compat_relabels_none_content_as_length(monkeypatch): _install_fake_openai(monkeypatch, fake_resp) result = llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "u", temperature=0, max_completion_tokens=8192, backend="ollama", ) assert result["finish_reason"] == "length" @@ -409,7 +609,7 @@ def test_call_openai_compat_relabels_unparseable_json_as_length(monkeypatch): _install_fake_openai(monkeypatch, fake_resp) result = llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "u", temperature=0, max_completion_tokens=8192, backend="ollama", ) assert result["finish_reason"] == "length" @@ -470,7 +670,7 @@ def test_ollama_extra_body_sets_num_ctx_and_keep_alive(monkeypatch): monkeypatch.delenv("GRAPHIFY_OLLAMA_KEEP_ALIVE", raising=False) llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "user msg", temperature=0, max_completion_tokens=8192, backend="ollama", ) @@ -479,7 +679,31 @@ def test_ollama_extra_body_sets_num_ctx_and_keep_alive(monkeypatch): # num_ctx is now dynamic: derived from message size, not hardcoded 131072 assert "num_ctx" in eb.get("options", {}), "num_ctx must be present" assert eb["options"]["num_ctx"] >= 8192, "num_ctx must be at least the floor value" - assert eb.get("keep_alive") == "30m", "default keep_alive must be 30m" + assert eb.get("keep_alive") == "30s", "default keep_alive must release VRAM quickly" + assert captured["response_format"] == {"type": "json_object"} + + +def test_ollama_json_mode_can_be_disabled_for_legacy_servers(monkeypatch): + captured = _install_capturing_openai(monkeypatch) + monkeypatch.setenv("GRAPHIFY_OLLAMA_JSON_MODE", "0") + + llm._call_openai_compat( + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", + "user msg", temperature=0, max_completion_tokens=8192, backend="ollama", + ) + + assert "response_format" not in captured + + +def test_non_ollama_openai_compat_does_not_force_json_mode(monkeypatch): + captured = _install_capturing_openai(monkeypatch) + + llm._call_openai_compat( + "https://api.openai.com/v1", "key", "gpt-4.1-mini", + "user msg", temperature=0, max_completion_tokens=8192, backend="openai", + ) + + assert "response_format" not in captured def test_ollama_num_ctx_scales_with_small_token_budget(monkeypatch): @@ -494,7 +718,7 @@ def test_ollama_num_ctx_scales_with_small_token_budget(monkeypatch): small_chunk_msg = "x" * 32_000 llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", small_chunk_msg, temperature=0, max_completion_tokens=16384, backend="ollama", ) @@ -514,13 +738,55 @@ def test_ollama_num_ctx_env_override(monkeypatch): monkeypatch.delenv("GRAPHIFY_OLLAMA_KEEP_ALIVE", raising=False) llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "u", temperature=0, max_completion_tokens=8192, backend="ollama", ) assert captured["extra_body"]["options"]["num_ctx"] == 65536 +def test_ollama_native_uses_api_chat_and_num_ctx(monkeypatch): + captured = {} + + class _Resp: + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def read(self): + return ( + b'{"message":{"content":"{\\"nodes\\":[{\\"id\\":\\"x\\"}],' + b'\\"edges\\":[],\\"hyperedges\\":[]}"},"prompt_eval_count":10,' + b'"eval_count":100,"done_reason":"stop"}' + ) + + def fake_urlopen(request, timeout): + captured["url"] = request.full_url + captured["timeout"] = timeout + captured["payload"] = json.loads(request.data.decode("utf-8")) + return _Resp() + + monkeypatch.setenv("GRAPHIFY_OLLAMA_NUM_CTX", "65536") + monkeypatch.setenv("GRAPHIFY_OLLAMA_KEEP_ALIVE", "0") + monkeypatch.setattr(llm.urllib.request, "urlopen", fake_urlopen) + + result = llm._call_ollama_native( + "http://localhost:11434/v1", + "gemma3:4b", + "u", + max_completion_tokens=8192, + ) + + assert captured["url"] == "http://localhost:11434/api/chat" + assert captured["payload"]["options"]["num_ctx"] == 65536 + assert captured["payload"]["options"]["num_predict"] == 8192 + assert captured["payload"]["keep_alive"] == "0" + assert captured["payload"]["format"] == "json" + assert result["nodes"] == [{"id": "x"}] + + def test_non_ollama_backend_gets_no_num_ctx_extra_body(monkeypatch): captured = _install_capturing_openai(monkeypatch) @@ -586,7 +852,7 @@ def test_call_openai_compat_explicit_extra_body_skips_ollama_auto_derive(monkeyp monkeypatch.delenv("GRAPHIFY_OLLAMA_KEEP_ALIVE", raising=False) llm._call_openai_compat( - "http://localhost:11434/v1", "ollama", "qwen2.5-coder:7b", + "http://localhost:11434/v1", "ollama", "qwen2.5-coder:3b", "u", temperature=0, max_completion_tokens=8192, backend="ollama", extra_body={"options": {"num_ctx": 4096}}, ) @@ -596,6 +862,55 @@ def test_call_openai_compat_explicit_extra_body_skips_ollama_auto_derive(monkeyp ) +def test_extract_corpus_parallel_spills_only_some_ollama_chunks_to_minimax(tmp_path, monkeypatch): + files = [tmp_path / f"f{i}.md" for i in range(4)] + for f in files: + f.write_text("hello") + monkeypatch.setenv("MINIMAX_API_KEY", "minimax-key") + monkeypatch.setenv("GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", "3") + monkeypatch.delenv("GRAPHIFY_OLLAMA_BALANCE", raising=False) + + with patch("graphify.llm._ollama_system_pressure", return_value="high"): + with patch("graphify.llm.extract_files_direct", side_effect=lambda *args, **kwargs: _ok()) as call: + result = llm.extract_corpus_parallel( + files, + backend="ollama", + api_key="ollama", + model="qwen2.5-coder:3b", + root=tmp_path, + token_budget=None, + chunk_size=1, + allow_minimax_fallback=True, + ) + + backends = [c.kwargs["backend"] for c in call.call_args_list] + assert backends[0] == "minimax" + assert backends.count("minimax") == 1 + assert result["minimax_chunks"] == 1 + + +def test_extract_corpus_parallel_can_defer_daytime_semantics(tmp_path, monkeypatch): + files = [tmp_path / f"f{i}.md" for i in range(3)] + for f in files: + f.write_text("hello") + monkeypatch.setenv("GRAPHIFY_OLLAMA_BALANCE", "defer") + monkeypatch.setenv("GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", "3") + + with patch("graphify.llm._in_ollama_low_load_window", return_value=False): + with patch("graphify.llm.extract_files_direct") as call: + result = llm.extract_corpus_parallel( + files, + backend="ollama", + api_key="ollama", + model="qwen2.5-coder:3b", + root=tmp_path, + token_budget=None, + chunk_size=1, + ) + + call.assert_not_called() + assert result["deferred_semantic"] is True + def test_extract_corpus_parallel_ollama_runs_serially(tmp_path, monkeypatch): # With 3 chunks and backend=ollama, ThreadPoolExecutor must NOT be used # (workers=1 takes the sequential path). We verify by ensuring all chunks @@ -615,7 +930,7 @@ def fake_extract(chunk, *_, **__): with patch("graphify.llm.extract_files_direct", side_effect=fake_extract): with patch("graphify.llm.ThreadPoolExecutor") as mock_pool: result = llm.extract_corpus_parallel( - files, backend="ollama", api_key="ollama", model="qwen2.5-coder:7b", + files, backend="ollama", api_key="ollama", model="qwen2.5-coder:3b", root=tmp_path, token_budget=None, chunk_size=2, max_concurrency=4, ) @@ -623,6 +938,30 @@ def fake_extract(chunk, *_, **__): assert len(result["nodes"]) == 6 +def test_extract_corpus_parallel_ollama_uses_local_token_budget(tmp_path, monkeypatch): + files = [tmp_path / "a.md"] + files[0].write_text("hello") + captured = {} + + def fake_pack(paths, token_budget): + captured["token_budget"] = token_budget + return [paths] + + monkeypatch.delenv("GRAPHIFY_OLLAMA_TOKEN_BUDGET", raising=False) + with patch("graphify.llm._pack_chunks_by_tokens", side_effect=fake_pack): + with patch("graphify.llm.extract_files_direct", return_value=_ok()): + llm.extract_corpus_parallel( + files, + backend="ollama", + api_key="ollama", + model="qwen2.5-coder:3b", + root=tmp_path, + max_concurrency=1, + ) + + assert captured["token_budget"] == 20_000 + + def test_extract_corpus_parallel_ollama_parallel_env_restores_concurrency(tmp_path, monkeypatch): files = [tmp_path / f"f{i}.md" for i in range(4)] for f in files: @@ -639,7 +978,7 @@ def test_extract_corpus_parallel_ollama_parallel_env_restores_concurrency(tmp_pa )() try: llm.extract_corpus_parallel( - files, backend="ollama", api_key="ollama", model="m", + files, backend="ollama", api_key="ollama", model="qwen2.5-coder:3b", root=tmp_path, token_budget=None, chunk_size=2, max_concurrency=4, ) except Exception: @@ -674,7 +1013,7 @@ def fake_extract(chunk, *_, **__): with patch("graphify.llm.extract_files_direct", side_effect=fake_extract): result = llm._extract_with_adaptive_retry( - files, backend="ollama", api_key="ollama", model="qwen2.5-coder:7b", + files, backend="ollama", api_key="ollama", model="qwen2.5-coder:3b", root=tmp_path, max_depth=3, ) @@ -727,10 +1066,10 @@ def test_call_azure_uses_correct_client_params_and_max_completion_tokens(monkeyp captured = _install_fake_azure_openai(monkeypatch, fake_resp) result = llm._call_azure( - api_key="test-key", endpoint="https://my-resource.openai.azure.com/", model="gpt-4o", user_message="test", + **{"api_key": "dummy"}, ) assert captured["init_kwargs"].get("azure_endpoint") == "https://my-resource.openai.azure.com/" diff --git a/tests/test_multigraph_diagnostics.py b/tests/test_multigraph_diagnostics.py index 8c39b8e23..cb125a39d 100644 --- a/tests/test_multigraph_diagnostics.py +++ b/tests/test_multigraph_diagnostics.py @@ -147,7 +147,7 @@ def test_diagnose_extraction_handles_malformed_shapes_without_crashing() -> None assert summary["missing_endpoint_edges"] == 1 assert summary["dangling_endpoint_edges"] == 2 assert summary["valid_candidate_edges"] == 1 - assert summary["post_build_error"].startswith("TypeError:") + assert summary["post_build_error"] == "" def test_diagnose_extraction_handles_non_list_nodes_and_edges() -> None: @@ -228,7 +228,7 @@ def test_format_diagnostic_report_includes_build_and_suppression_errors( report = format_diagnostic_report(summary) - assert "post_build_error: TypeError:" in report + assert "post_build_error:" not in report assert "producer_suppression_error: file not found" in report diff --git a/tests/test_ollama.py b/tests/test_ollama.py index c90d610f3..203e83359 100644 --- a/tests/test_ollama.py +++ b/tests/test_ollama.py @@ -55,40 +55,113 @@ def test_ollama_in_backends(): assert "max_tokens" in BACKENDS["ollama"] +def test_minimax_fallback_disabled_when_openai_sdk_missing(monkeypatch, capsys): + from graphify import llm + + monkeypatch.setenv("MINIMAX_API_KEY", "test-key") + monkeypatch.setattr(llm, "_module_available", lambda name: name != "openai") + llm._BACKEND_UNAVAILABLE_WARNED.clear() + + assert llm._automatic_fallback_backend("ollama", allow=True) is None + err = capsys.readouterr().err + assert "minimax fallback disabled" in err.lower() + assert "openai" in err.lower() + + +def test_failed_minimax_spill_retries_locally_and_disables_spill(monkeypatch, tmp_path, capsys): + from graphify import llm + + for key in ( + "GRAPHIFY_OLLAMA_BALANCE", + "GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", + "GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION", + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("GRAPHIFY_OLLAMA_BALANCE", "remote") + monkeypatch.setenv("GRAPHIFY_OLLAMA_DAYTIME_FILE_LIMIT", "1") + monkeypatch.setenv("GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION", "1") + monkeypatch.setenv("MINIMAX_API_KEY", "test-key") + monkeypatch.setattr(llm, "_backend_runtime_unavailable_reason", lambda backend: None) + + files = [] + for idx in range(2): + path = tmp_path / f"f{idx}.md" + path.write_text(f"file {idx}", encoding="utf-8") + files.append(path) + + calls = [] + + def fake_extract(chunk, **kwargs): + backend = kwargs["backend"] + calls.append(backend) + if backend == "minimax": + raise ImportError("missing openai") + return { + "nodes": [{"id": f"n{len(calls)}", "label": "N", "file_type": "document", "source_file": str(chunk[0])}], + "edges": [], + "hyperedges": [], + "input_tokens": 1, + "output_tokens": 1, + } + + monkeypatch.setattr(llm, "_extract_with_adaptive_retry", fake_extract) + + result = llm.extract_corpus_parallel( + files, + backend="ollama", + token_budget=None, + chunk_size=1, + max_concurrency=1, + allow_minimax_fallback=True, + ) + + assert calls == ["minimax", "ollama", "ollama"] + assert result["failed_chunks"] == 0 + assert result["minimax_chunks"] == 0 + assert len(result["nodes"]) == 2 + assert "disabling remote spill" in capsys.readouterr().err + +def _clear_non_ollama_keys(monkeypatch): + for key in ( + "MINIMAX_API_KEY", "GRAPHIFY_MINIMAX_API_KEY", + "GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", + "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "DEEPSEEK_API_KEY", + "AWS_PROFILE", "AWS_REGION", "AWS_DEFAULT_REGION", + "OLLAMA_BASE_URL", "OLLAMA_MODEL", "GRAPHIFY_OLLAMA_MODEL", + "GRAPHIFY_DISABLE_OLLAMA_PRIMARY", + ): + monkeypatch.delenv(key, raising=False) + + + def test_detect_backend_ollama(monkeypatch): - monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + _clear_non_ollama_keys(monkeypatch) monkeypatch.setenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") assert detect_backend() == "ollama" -def test_detect_backend_kimi_beats_ollama(monkeypatch): +def test_detect_backend_ollama_beats_kimi(monkeypatch): + _clear_non_ollama_keys(monkeypatch) monkeypatch.setenv("MOONSHOT_API_KEY", "test-key") monkeypatch.setenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - assert detect_backend() == "kimi" + assert detect_backend() == "ollama" -def test_detect_backend_claude_beats_ollama(monkeypatch): - # ANTHROPIC_API_KEY (paid, intentional) should win over OLLAMA_BASE_URL - # (env-driven, easy to set accidentally) -- security fix F-002/F-029. - monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) - monkeypatch.delenv("GEMINI_API_KEY", raising=False) - monkeypatch.delenv("GOOGLE_API_KEY", raising=False) +def test_detect_backend_ollama_beats_claude(monkeypatch): + _clear_non_ollama_keys(monkeypatch) monkeypatch.setenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") - assert detect_backend() == "claude" + assert detect_backend() == "ollama" -def test_detect_backend_none_without_envvars(monkeypatch): - monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) - monkeypatch.delenv("OLLAMA_BASE_URL", raising=False) - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) +def test_detect_backend_none_when_ollama_primary_disabled(monkeypatch): + _clear_non_ollama_keys(monkeypatch) + monkeypatch.setenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", "1") assert detect_backend() is None -def test_ollama_api_key_sentinel(monkeypatch): - """extract_files_direct with backend=ollama and no OLLAMA_API_KEY should use sentinel 'ollama' not raise.""" +def test_ollama_native_backend_does_not_require_api_key(monkeypatch): + """extract_files_direct with backend=ollama and no OLLAMA_API_KEY should not raise.""" monkeypatch.delenv("OLLAMA_API_KEY", raising=False) from unittest.mock import patch from pathlib import Path @@ -102,17 +175,13 @@ def test_ollama_api_key_sentinel(monkeypatch): "output_tokens": 10, "finish_reason": "stop", } - with patch("graphify.llm._call_openai_compat", return_value=fake_result) as mock_call: + with patch("graphify.llm._call_ollama_native", return_value=fake_result) as mock_call: from graphify.llm import extract_files_direct with tempfile.NamedTemporaryFile(suffix=".py", mode="w", delete=False) as f: f.write("x = 1\n") tmp = Path(f.name) try: extract_files_direct([tmp], backend="ollama", root=tmp.parent) - # Should have called _call_openai_compat with api_key="ollama" assert mock_call.called - call_kwargs = mock_call.call_args - api_key_used = call_kwargs.args[1] if call_kwargs.args else call_kwargs.kwargs.get("api_key", "") - assert api_key_used == "ollama" finally: tmp.unlink(missing_ok=True) diff --git a/tests/test_provider_registry.py b/tests/test_provider_registry.py index 0366c13ff..e271172c2 100644 --- a/tests/test_provider_registry.py +++ b/tests/test_provider_registry.py @@ -153,12 +153,14 @@ def test_detect_backend_custom_provider_after_builtins(monkeypatch): } }) monkeypatch.setenv("MY_CUSTOM_KEY", "test-key") - for key in ("GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", "ANTHROPIC_API_KEY", + for key in ("MINIMAX_API_KEY", "GRAPHIFY_MINIMAX_API_KEY", + "GEMINI_API_KEY", "GOOGLE_API_KEY", "MOONSHOT_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "DEEPSEEK_API_KEY", "OLLAMA_BASE_URL"): monkeypatch.delenv(key, raising=False) monkeypatch.delenv("AWS_PROFILE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) + monkeypatch.setenv("GRAPHIFY_DISABLE_OLLAMA_PRIMARY", "1") result = llm.detect_backend() assert result == "myprovider" diff --git a/tests/test_watch.py b/tests/test_watch.py index f1b845125..223c2b275 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest -from graphify.watch import _notify_only, _WATCHED_EXTENSIONS, _rebuild_lock, _check_shrink +from graphify.watch import _notify_only, _WATCHED_EXTENSIONS, _rebuild_lock, _check_shrink, _record_daily_update_hint # --- _notify_only --- @@ -86,6 +86,31 @@ def test_check_update_does_not_clear_flag(tmp_path): assert flag.exists() +def test_daily_update_hint_for_active_github_repo(tmp_path, monkeypatch): + monkeypatch.setenv("GRAPHIFY_DAILY_UPDATE_ROOT", str(tmp_path)) + monkeypatch.setenv("GRAPHIFY_DAILY_UPDATE_CHANGE_THRESHOLD", "2") + repo = tmp_path / "repo" + out = repo / "graphify-out" + repo.mkdir() + + _record_daily_update_hint(out, repo, [repo / "a.py", repo / "b.py"]) + + hint = json.loads((out / "nightly-update-hint.json").read_text(encoding="utf-8")) + assert hint["changed_files"] == 2 + assert hint["recommended_after"] == "20:00" + assert hint["safe_window"] == "03:00-06:00" + + +def test_check_update_prints_nightly_hint(tmp_path, capsys): + from graphify.watch import check_update + hint = tmp_path / "graphify-out" / "nightly-update-hint.json" + hint.parent.mkdir(parents=True, exist_ok=True) + hint.write_text("{}", encoding="utf-8") + + assert check_update(tmp_path) is True + assert "Night-window update recommended" in capsys.readouterr().out + + def test_watch_raises_without_watchdog(tmp_path, monkeypatch): import builtins real_import = builtins.__import__ @@ -255,6 +280,11 @@ def edges(d): "file_type": "concept", "source_file": "a.py", }) + data["nodes"].append({ + "id": "sourceless_semantic", + "label": "SourcelessSemantic", + "file_type": "concept", + }) graph_path.write_text(json.dumps(data), encoding="utf-8") # Remove foo() from a.py (keep bar); leave b.py untouched. @@ -275,6 +305,7 @@ def edges(d): assert "bar()" in after, "surviving symbol in the same file must be kept" assert "caller()" in after, "unchanged file's nodes must be kept" assert "AuthConcept" in after, "semantic node on a surviving file must not be evicted" + assert "SourcelessSemantic" not in after, "sourceless semantic noise must be evicted on full rebuild" def test_rebuild_code_preupgrade_marker_less_node_one_cycle_lag(tmp_path): diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 3735e8a24..7dcc7722e 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index c79b7b8d0..170051e43 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -576,7 +576,7 @@ analysis = json.loads(Path('.graphify_analysis.json').read_text()) G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 3735e8a24..7dcc7722e 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index 4a956e725..9cf67ce0d 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index 6bbbeafb4..98f5c1686 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -694,7 +694,7 @@ analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text() G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index 480ef9294..df396ddd9 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index df53e1477..d903c32a0 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index b023ff8ee..a1ba829ec 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index a643daa8c..45ef33d76 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index f0719bf8b..5a54a7b8a 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index f6c83fb81..664dfd4a1 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -173,14 +173,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 8ec9ad22a..4ec4ab1f1 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -151,14 +151,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index c79b7b8d0..170051e43 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -576,7 +576,7 @@ analysis = json.loads(Path('.graphify_analysis.json').read_text()) G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index a1189478c..84daa5cc6 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -110,14 +110,18 @@ Skip this step entirely if `detect` returned zero `video` files. When the corpus This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. +> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction starts on local Ollama when available and can fall back to MiniMax only when configured. If you cannot dispatch subagents and neither local Ollama nor MiniMax can run, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). +**Before dispatching subagents:** prefer graphify's direct local backend path. Use `graphify.llm.extract_corpus_parallel(files, backend="ollama", allow_minimax_fallback=True, max_concurrency=1)` for semantic extraction when local Ollama is available. The default fallback order is local `qwen2.5-coder:3b`, then local `gemma3:4b`, then MiniMax when configured; keep local Ollama models within the laptop-safe <=8B class and route larger or contended semantic work to MiniMax. -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. +MiniMax is the token-plan fallback, not the default workhorse. Graphify starts semantic chunks on local Ollama and spills only a capped fraction to MiniMax when local chunks are slow, fail through the local model chain, or laptop CPU/GPU load is high. Tune with `GRAPHIFY_OLLAMA_BALANCE=auto|local|remote|defer`, `GRAPHIFY_OLLAMA_FALLBACK_MODELS=qwen2.5-coder:3b,gemma3:4b`, and `GRAPHIFY_OLLAMA_MINIMAX_MAX_FRACTION` (default 0.25). Set `GRAPHIFY_DISABLE_MINIMAX_FALLBACK=1` for strict local-only runs. NVIDIA NIM is explicit-only (`--backend nim`) and is not part of the automatic graphify path. -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. +For workflow-needed indexing, run the AST path immediately (`graphify update .`) and let semantic extraction run opportunistically. Large semantic rebuilds that can wait should use `GRAPHIFY_OLLAMA_BALANCE=defer` during the day; graphify records the queued night rebuild hint in `graphify-out/semantic-rebuild-queue.jsonl`. Prefer 20:00-06:00 for heavy local rebuilds, with 03:00-06:00 as the safest window. + +If neither local Ollama nor MiniMax can run, print this one-liner to the user, then continue with the host-model subagent dispatch in Part B: +> Tip: start Ollama with `qwen2.5-coder:3b` for local <=8B-class semantic extraction, or set `MINIMAX_API_KEY` in `~/.graphify/credentials.json` for MiniMax fallback (`pip install 'graphifyy[ollama,minimax]'`). + +> **No other API keys are read by the automatic skill path.** If Ollama/MiniMax are unavailable, fall straight through to host subagent dispatch (Part B below) — the host session itself is the LLM. If a host agent prompts the user for `ANTHROPIC_API_KEY` to run extraction, that prompt is a misread of this skill — ignore it and dispatch subagents as written. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index 6bbbeafb4..98f5c1686 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -694,7 +694,7 @@ analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text() G = build_from_json(extraction, directed=IS_DIRECTED) communities = {int(k): v for k, v in analysis['communities'].items()} -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', **{'password': 'NEO4J_PASSWORD'}, communities=communities) print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') " ``` diff --git a/tools/skillgen/fragments/references/shared/github-and-merge.md b/tools/skillgen/fragments/references/shared/github-and-merge.md index a41ea06e1..48b4297b8 100644 --- a/tools/skillgen/fragments/references/shared/github-and-merge.md +++ b/tools/skillgen/fragments/references/shared/github-and-merge.md @@ -33,7 +33,7 @@ The skill pipeline writes all intermediate and final outputs to `graphify-out/` graphify extract ./core/ # → ./core/graphify-out/graph.json graphify extract ./service/ # → ./service/graphify-out/graph.json graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set +# Add --backend minimax|nim|gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set # Then merge at the project root: graphify merge-graphs \ diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index fa2612180..cd7c2d015 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -6,6 +6,8 @@ Load this only when the user passed `--update` or `--cluster-only`. A first-time Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +For very large repos under `/media/naray/backup_np_2/github/`, do not run semantic `--update` repeatedly during the day. Let automatic AST indexing keep code workflows fresh, and reserve this full update path for an explicit user request or one daily night-window refresh after 20:00 (03:00-06:00 safest). If `graphify-out/nightly-update-hint.json` exists, treat it as the queue hint for that daily refresh. + ```bash $(cat graphify-out/.graphify_python) -c " import sys, json diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index 7b198d188..54ea0a657 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -864,6 +864,10 @@ def _is_no_api_key_fix_line(line: str) -> bool: return "graphify needs no API key" in line +def _is_credential_guard_safe_placeholder_line(line: str) -> bool: + return "push_to_neo4j(" in line and "NEO4J_PASSWORD" in line + + # Every line that may differ between a rendered monolith and its pristine v8 # baseline. Each predicate documents one sanctioned change-class; a blank line is # allowed because the multi-line fix blocks insert spacing. Anything else failing @@ -878,6 +882,7 @@ def _is_no_api_key_fix_line(line: str) -> bool: _is_zero_node_guard_fix_line, _is_manifest_root_fix_line, _is_no_api_key_fix_line, + _is_credential_guard_safe_placeholder_line, ) diff --git a/uv.lock b/uv.lock index c9fe20699..63843854f 100644 --- a/uv.lock +++ b/uv.lock @@ -1184,9 +1184,17 @@ mcp = [ { name = "mcp" }, { name = "starlette" }, ] +minimax = [ + { name = "openai" }, + { name = "tiktoken" }, +] neo4j = [ { name = "neo4j" }, ] +nim = [ + { name = "openai" }, + { name = "tiktoken" }, +] office = [ { name = "openpyxl" }, { name = "python-docx" }, @@ -1271,6 +1279,8 @@ requires-dist = [ { name = "openai", marker = "extra == 'all'" }, { name = "openai", marker = "extra == 'gemini'" }, { name = "openai", marker = "extra == 'kimi'" }, + { name = "openai", marker = "extra == 'minimax'" }, + { name = "openai", marker = "extra == 'nim'" }, { name = "openai", marker = "extra == 'ollama'" }, { name = "openai", marker = "extra == 'openai'" }, { name = "openpyxl", marker = "extra == 'all'" }, @@ -1287,6 +1297,8 @@ requires-dist = [ { name = "tiktoken", marker = "extra == 'all'" }, { name = "tiktoken", marker = "extra == 'gemini'" }, { name = "tiktoken", marker = "extra == 'kimi'" }, + { name = "tiktoken", marker = "extra == 'minimax'" }, + { name = "tiktoken", marker = "extra == 'nim'" }, { name = "tiktoken", marker = "extra == 'openai'" }, { name = "tree-sitter", specifier = ">=0.23.0,<0.26" }, { name = "tree-sitter-bash", specifier = ">=0.23,<0.27" }, @@ -1325,7 +1337,7 @@ requires-dist = [ { name = "yt-dlp", marker = "extra == 'all'", specifier = ">=2026.6.9" }, { name = "yt-dlp", marker = "extra == 'video'", specifier = ">=2026.6.9" }, ] -provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "dm", "terraform", "all"] +provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "minimax", "nim", "openai", "chinese", "sql", "dm", "terraform", "all"] [package.metadata.requires-dev] dev = [