diff --git a/.github/workflows/build_main_documentation.yml b/.github/workflows/build_main_documentation.yml index 39a2b1c8..5de99c24 100644 --- a/.github/workflows/build_main_documentation.yml +++ b/.github/workflows/build_main_documentation.yml @@ -49,6 +49,13 @@ on: python_version: type: string description: "Python version for the build venv (e.g. '3.11'). Defaults to the runner's system Python." + translated_languages: + type: string + description: > + Languages whose pages are machine-translated, separated by spaces. Their source is + pulled from the hf-doc-build/doc-translate bucket instead of from the repo. Anything + listed here must also appear in `languages`. Leave it empty (the default) and every + language comes from the repo as usual. secrets: hf_token: required: true @@ -186,6 +193,48 @@ jobs: doc-builder notebook-to-mdx ${{ env.doc_folder }} --open_notebook_prefix https://colab.research.google.com/github/${{ inputs.repo_owner }}/${{ inputs.package }}/blob/$branch$remaining_part + # Machine-translated pages are not kept in the repo. A separate nightly job writes them to + # a bucket, and this step fetches them so the build has something to work from. It does + # nothing at all unless a workflow asks for it, since every HF library shares this file. + # + # The bucket path uses the repo name, not the Python package name. The job that writes + # those files names them after the repo it cloned, so both sides have to agree. + - name: Sync machine translations + if: inputs.translated_languages != '' + env: + HF_TOKEN: ${{ secrets.hf_token }} + BUCKET: hf://buckets/hf-doc-build/doc-translate/translations/${{ inputs.package }} + # This is handed over as an environment variable rather than written straight into + # the script below. GitHub pastes a ${{ }} value into the script text before bash + # reads any of it, so quoting it there would not stop someone slipping in extra + # commands -- and this job is holding a token that can write to our buckets. + TRANSLATED_LANGUAGES: ${{ inputs.translated_languages }} + run: | + for lang in $TRANSLATED_LANGUAGES; do + target="${{ env.doc_folder }}/$lang" + # Download next to where the files are going, rather than into /tmp. Same disk, so + # putting them in place is a rename instead of copying 700 files. Cleared out first + # in case an earlier run was interrupted halfway through. + staged="$target.incoming" + rm -rf "$staged" + uvx --from huggingface_hub hf sync "$BUCKET/$lang" "$staged" + + # Download everything first, then swap it in. If we wrote straight into place and + # the download died halfway, the build would carry on quite happily with pages + # missing and nobody would notice. The sidebar file is the last thing we expect to + # see, so if it is there the download finished. + if [ ! -f "$staged/_toctree.yml" ]; then + echo "::error::no _toctree.yml in $BUCKET/$lang - refusing to build a partial tree" + exit 1 + fi + + # Replace the folder outright rather than merging into it. That also clears out + # pages whose English original has since been deleted -- transformers has 20 of + # those sitting in its Japanese docs today. + rm -rf "$target" && mv "$staged" "$target" + echo "$lang: $(find "$target" -name '*.md' | wc -l) page(s) from $BUCKET/$lang" + done + - name: Make documentation shell: bash env: diff --git a/pyproject.toml b/pyproject.toml index a52c275a..164a1bc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,16 @@ Repository = "https://github.com/huggingface/doc-builder" [project.optional-dependencies] transformers = ["transformers[dev]"] +# `doc-builder translate` runs a real model, so these cannot be mocked away like the heavy +# deps in `mock_deps/`. Kept an extra so the doc-build path never installs them. +# accelerate is required by `device_map="cuda"` in translate/pipeline.py -- without it +# from_pretrained raises before the model is even downloaded. +# +# kernels lets `flash_attention_2` fall back to the kernels-community/flash-attn2 kernel on +# the Hub when the compiled flash-attn package isn't installed. Without it the fallback can't +# happen and from_pretrained raises instead. Compiling flash-attn inside the job would be slow +# and fragile, so the Hub kernel is the path we want. +translate = ["torch", "transformers", "accelerate", "kernels>=0.11.0"] testing = [ "pytest", "pytest-xdist", @@ -92,6 +102,9 @@ build-backend = "setuptools.build_meta" [tool.setuptools] package-dir = {"" = "src"} +[tool.setuptools.package-data] +doc_builder = ["mock_deps/*.txt", "glossaries/*.yml"] + [tool.setuptools.packages.find] where = ["src"] diff --git a/setup.py b/setup.py index 61a62151..1b240b1f 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ extras = {} extras["transformers"] = ["transformers[dev]"] +extras["translate"] = ["torch", "transformers", "accelerate", "kernels>=0.11.0"] extras["testing"] = [ "pytest", "pytest-xdist", @@ -49,7 +50,7 @@ keywords="doc documentation doc-builder huggingface hugging face", url="https://github.com/huggingface/doc-builder", package_dir={"": "src"}, - package_data={"doc_builder": ["mock_deps/*.txt"]}, + package_data={"doc_builder": ["mock_deps/*.txt", "glossaries/*.yml"]}, include_package_data=True, packages=find_packages("src"), extras_require=extras, diff --git a/src/doc_builder/commands/doc_builder_cli.py b/src/doc_builder/commands/doc_builder_cli.py index 43a0959b..8cec8a57 100644 --- a/src/doc_builder/commands/doc_builder_cli.py +++ b/src/doc_builder/commands/doc_builder_cli.py @@ -24,6 +24,7 @@ from doc_builder.commands.preview import preview_command_parser from doc_builder.commands.push import push_command_parser from doc_builder.commands.style import style_command_parser +from doc_builder.commands.translate import translate_command_parser def main(): @@ -40,6 +41,7 @@ def main(): style_command_parser(subparsers=subparsers) preview_command_parser(subparsers=subparsers) push_command_parser(subparsers=subparsers) + translate_command_parser(subparsers=subparsers) # Let's go args = parser.parse_args() diff --git a/src/doc_builder/commands/translate.py b/src/doc_builder/commands/translate.py new file mode 100644 index 00000000..9654ed74 --- /dev/null +++ b/src/doc_builder/commands/translate.py @@ -0,0 +1,357 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +`doc-builder translate` -- translate a library's English docs into another language. + +Built to run as one Hugging Face Job, once a night. It works out what has changed before it +loads the model, so a night where nothing changed finishes in a couple of minutes without +ever downloading 52GB of weights: + + hf jobs uv run --namespace hf-doc-build --flavor a100-large --timeout 6h \ + -v hf://buckets/hf-doc-build/doc-translate:/bucket --secrets HF_TOKEN \ + --with "hf-doc-builder[translate] @ git+https://github.com/huggingface/doc-builder@main" \ + doc-builder translate transformers --lang ja --bucket /bucket + +To try it on your own machine, point --bucket at any folder and --source at a docs checkout. +Nothing gets translated without a GPU, but everything up to that point runs: + + doc-builder translate transformers --lang ja \ + --source ~/hf/transformers/docs/source --bucket /tmp/bucket --dry-run +""" + +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +from doc_builder.translate import pipeline, validate +from doc_builder.translate.cache import SegmentCache, segment_key + +DEFAULT_MODEL = "google/gemma-4-26B-A4B-it" +REPO_URL = "https://github.com/huggingface/{package}.git" +TOCTREE = "_toctree.yml" + + +def clone_docs(package, into): + """Grab a copy of the library's repo, latest commit only. + + We fetch the whole repo rather than just the docs folder, which looks wasteful but is not. + A couple of doc pages are shortcuts pointing outside the docs folder -- in transformers, + `en/notebooks.md` points at `notebooks/README.md` and `en/contributing.md` at + `CONTRIBUTING.md`. Fetch only the docs folder and those shortcuts point at nothing, and the + run dies partway through. The extra few hundred MB is nothing next to a 52GB model. + """ + url = REPO_URL.format(package=package) + print(f"[translate] cloning {url}") + subprocess.run(["git", "clone", "--depth", "1", url, str(into)], check=True) + return into / "docs" / "source" + + +def select_pages(source_dir, pages_file): + pages = sorted(p.relative_to(source_dir).as_posix() for p in source_dir.rglob("*.md")) + if not pages_file: + return pages + wanted = { + line.strip() + for line in Path(pages_file).read_text(encoding="utf-8").splitlines() + if line.strip() and not line.startswith("#") + } + missing = wanted - set(pages) + if missing: + print(f"[translate] WARNING {len(missing)} requested page(s) not found: {sorted(missing)[:5]}") + return [p for p in pages if p in wanted] + + +def plan_all(source_dir, pages, lang, model, gloss_sha): + """Break every page into paragraphs, ready to translate. + + A page we cannot read is skipped with a warning rather than stopping everything. A couple of + pages are shortcuts to files outside the docs folder, so if one of those ever points + somewhere we did not fetch, it should cost us that page and not the whole night. + """ + plans, unreadable = {}, [] + for page in pages: + try: + text = (source_dir / page).read_text(encoding="utf-8") + except OSError as exc: + unreadable.append(f"{page} ({exc.__class__.__name__})") + continue + plans[page] = pipeline.PagePlan(page, text, lang, model, gloss_sha) + if unreadable: + print(f"[translate] WARNING skipped {len(unreadable)} unreadable page(s): {unreadable[:5]}") + return plans + + +def load_toctree(source_dir, lang, model, gloss_sha, pages=None): + """Read the sidebar file and work out an ID for each of its titles. + + Pass `pages` to cut the sidebar down to just those pages -- that is for test runs on a + handful of pages. Leave it out and the whole sidebar is kept, which is the normal case. + """ + path = source_dir / TOCTREE + if not path.is_file(): + return None, {} + tree = yaml.safe_load(path.read_text(encoding="utf-8")) + + if pages is not None: + # the sidebar refers to pages without the .md on the end + keep = {p[:-3] if p.endswith(".md") else p for p in pages} + tree = pipeline.prune_toctree(tree, keep) + if tree is None: + print("[translate] WARNING no toctree entries match the selected pages; skipping it") + return None, {} + + keys = { + segment_key(title, model, pipeline.PROMPT_VERSION, gloss_sha, lang): title + for title in pipeline.toctree_titles(tree) + } + return tree, keys + + +def write_toctree(out_dir, tree, titles): + """Write out the sidebar with translated titles, but only if it still reads back correctly. + + We check by writing the file out, reading it straight back, and making sure every page is + still listed. Checking the version in memory would prove nothing, since swapping titles + cannot change the shape of anything. What could actually go wrong is the writing and + re-reading itself, and a broken sidebar takes down the entire language -- unlike one bad + page, which only affects itself. + """ + source_locals = pipeline.toctree_values(tree, "local") + pipeline.apply_toctree_titles(tree, titles) + dumped = yaml.safe_dump(tree, sort_keys=False, allow_unicode=True) + if pipeline.toctree_values(yaml.safe_load(dumped), "local") != source_locals: + print("[translate] ERROR toctree did not survive a re-parse; keeping the existing one") + return False + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / TOCTREE).write_text(dumped, encoding="utf-8") + return True + + +def write_page(out_dir, page, text): + dest = out_dir / page + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(text, encoding="utf-8") + + +def keep_published(path, source): + """Is the page already in the bucket still good enough to leave alone? + + Only the checks that read a finished page apply here -- we have the published text, not the + marked-up version it was built from, so the marker checks cannot run. In practice that is + the check that matters, since it is the one that has caught pages published under older, + weaker rules. + """ + if not path.is_file(): + return False + try: + published = path.read_text(encoding="utf-8") + except OSError: + return False + problems = validate.check_links(source, published) + if problems: + print(f"[translate] replacing published {path.name}: {problems[0]}") + return False + return True + + +def run(args, source_dir): + bucket = Path(args.bucket) + out_dir = bucket / "translations" / args.package / args.lang + cache = SegmentCache(bucket) + + glossary = pipeline.load_glossary(args.glossary or pipeline.glossary_path(args.lang)) + gloss_sha = pipeline.glossary_sha(glossary) + print(f"[translate] {args.package}/{args.lang} model={args.model} glossary={gloss_sha[:12]}") + + pages = select_pages(source_dir, args.pages_file) + plans = plan_all(source_dir, pages, args.lang, args.model, gloss_sha) + # If we are only translating some pages, the sidebar has to be trimmed to match, or the + # result cannot be built. + subset = pages if args.pages_file else None + toc_tree, toc_keys = load_toctree(source_dir, args.lang, args.model, gloss_sha, subset) + + wanted = {} + for plan in plans.values(): + wanted.update(plan.segments) + wanted.update(toc_keys) + + known = cache.load_index() + missing = {k: v for k, v in wanted.items() if k not in known} + print( + f"[translate] {len(plans)} pages, {len(wanted)} unique segments, " + f"{len(missing)} missing ({len(wanted) - len(missing)} cached)" + ) + + # This is the moment the whole design is built around: decide whether there is anything to + # do before going anywhere near the model. + # + # --rebuild carries on anyway, using the cache. Nothing here needs a GPU when there is + # nothing to translate, so it is a cheap way to republish every page after a change to how + # pages are assembled or checked. Without it those changes never reach the bucket, because + # a warm run stops before it ever rebuilds a page. + if not missing and not args.rebuild: + print("[translate] cache is warm, nothing to translate -- exiting before model load") + return 0 + if args.dry_run: + print(f"[translate] dry run, would translate {len(missing)} segment(s)") + return 0 + + fresh, failures = ({}, []) + if missing: + fresh, failures = pipeline.translate_segments( + missing, + args.lang, + glossary, + args.model, + attn_implementation=args.attn_implementation, + use_cuda_graph=args.cuda_graphs, + ) + print(f"[translate] translated {len(fresh)}, {len(failures)} request failure(s)") + for key, why in failures[:10]: + print(f"[translate] FAILED {key[:12]} {why}") + + cache.put_many(fresh) + cache.save_index() + + # Pull together what we had already and what we just translated. + available = cache.get_many([k for k in wanted if k in known and k not in fresh]) + available.update(fresh) + + results, written, skipped = [], 0, 0 + for page, plan in plans.items(): + masked_translation, page_text, rejected = pipeline.assemble_page(plan, available) + result = pipeline.validate_plan(plan, masked_translation, glossary, page_text) + if rejected: + result.warnings.append(f"{len(rejected)} paragraph(s) kept in English: markers not preserved") + results.append(result) + if result.ok: + disclosed = pipeline.add_disclosure(page_text, page, args.lang, args.package) + write_page(out_dir, page, disclosed) + written += 1 + else: + # This page failed its checks, so keep whatever is already published -- but only if + # that still passes today's checks. "The last good translation" meant good under the + # rules at the time, and the rules get stricter: continuous_batching.md sat in the + # bucket with 8 broken links for two runs, kept each time because the fresh attempt + # failed for an unrelated reason. Anything that no longer passes drops to English. + if not keep_published(out_dir / page, plan.source): + write_page(out_dir, page, plan.source) + skipped += 1 + + toctree_ok = True + if toc_tree is not None: + toctree_ok = write_toctree( + out_dir, + toc_tree, + # Sidebar titles never go through assemble_page, so they need the same clean-up. + {title: pipeline.strip_echoed_markers(available.get(key, title)) for key, title in toc_keys.items()}, + ) + + print(validate.summarize(results)) + print(f"[translate] wrote {written} page(s), kept/fell back on {skipped}") + # If the sidebar was rejected, fail the run. The pages have already gone out, so reporting + # success here would leave new pages sitting behind an old sidebar that does not list them. + return 0 if toctree_ok else 1 + + +def translate_command(args): + """Where the command starts. + + It exits with an error code rather than returning one. The CLI throws away whatever a + command returns, so a returned code would vanish -- and this runs unattended every night, + where the exit code is the only way anyone finds out something went wrong. `check_links` and + `light_install` do the same. + """ + if args.source: + code = run(args, Path(args.source) / "en") + else: + with tempfile.TemporaryDirectory() as tmp: + code = run(args, clone_docs(args.package, Path(tmp)) / "en") + if code: + sys.exit(code) + + +def translate_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("translate") + else: + parser = argparse.ArgumentParser("Doc Builder translate command") + + parser.add_argument("package", type=str, help="Name of the GitHub repo whose docs to translate.") + parser.add_argument("--lang", type=str, default="ja", help="Target language code.") + parser.add_argument("--model", type=str, default=DEFAULT_MODEL, help="Translation model id.") + parser.add_argument( + "--bucket", + type=str, + default="/bucket", + help="Mounted HF storage bucket (or a local directory) holding `cache/` and `translations/`.", + ) + parser.add_argument( + "--source", + type=str, + default=None, + help="Existing docs/source checkout to translate from. Clones the package repo if omitted.", + ) + parser.add_argument( + "--glossary", + type=str, + default=None, + help="Glossary YAML. Defaults to the packaged `doc_builder/glossaries/.yml`.", + ) + parser.add_argument( + "--pages-file", + type=str, + default=None, + help="Newline-separated subset of page paths to translate, for smoke runs.", + ) + parser.add_argument( + "--attn-implementation", + type=str, + default=pipeline.DEFAULT_ATTENTION, + help=( + "How attention is computed, e.g. 'paged|sdpa' (the default, always available) or " + "'paged|flash_attention_2' (faster, but needs flash-attn or a matching Hub kernel)." + ), + ) + parser.add_argument( + "--cuda-graphs", + action="store_true", + default=pipeline.DEFAULT_CUDA_GRAPHS, + help=( + "Record and replay the GPU work for speed. Off by default because it is incompatible " + "with mixture-of-experts models, which copy between CPU and GPU to route experts." + ), + ) + parser.add_argument( + "--rebuild", + action="store_true", + help=( + "Rebuild and republish every page from the cache even when nothing needs " + "translating. Use after changing how pages are assembled or checked. Needs no GPU " + "if the cache is warm." + ), + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Report how many segments are missing and exit without loading the model.", + ) + if subparsers is not None: + parser.set_defaults(func=translate_command) + return parser diff --git a/src/doc_builder/glossaries/ja.yml b/src/doc_builder/glossaries/ja.yml new file mode 100644 index 00000000..74bf7d8b --- /dev/null +++ b/src/doc_builder/glossaries/ja.yml @@ -0,0 +1,60 @@ +# How we want certain words translated into Japanese. +# +# Without this, the same word comes out three different ways across 733 pages and nobody +# notices. These were written by hand, but they are not guesses: they come from the 152 +# pages that human translators had already done. Where those translators disagreed with each +# other, the more common choice won. The numbers below are how many of those pages used each +# one, which is the evidence for picking it. +# +# Matching ignores capitals and matches partial words, so `fine-tune` also covers +# "fine-tuned". Always write whole words here, never a truncated stem: whatever is on the left +# is shown to the model as the English term, and a stem like `fine-tun` comes back rendered +# literally in the page as "微調整（fine-tun）". Add a second entry for a form that +# is not a simple suffix, the way fine-tuning does below. +# +# Editing this file re-translates any text containing the words you changed, and nothing +# else. Keep the list short: matching terms are added to every request we send the model, +# and it cannot reuse that work between requests. + +pin: + # These had a competing translation. The number is how many pages used this one. + fine-tune: 微調整 # 54 files, vs ファインチューニング 15 + fine-tuning: 微調整 # "fine-tune" does not match this one + pretrained: 事前トレーニング済み # 30 files, vs 事前学習済みモデル 12 + pretraining: 事前トレーニング + training: トレーニング # 112 files, vs 学習 74 + tokenizer: トークナイザー # 35 files + sequence: シーケンス # 49 files, vs 系列 6 + + # Nobody disagreed on these. They are here to stop the model inventing alternatives. + inference: 推論 # 69 + input: 入力 # 83 + output: 出力 # 55 + dataset: データセット # 60 + checkpoint: チェックポイント # 50 + batch: バッチ # 43 + embedding: 埋め込み # 40 + padding: パディング # 35 + weight: 重み # 34 + vocabulary: 語彙 # 20 + mixed precision: 混合精度 # 15 + gradient: 勾配 # 13 + quantization: 量子化 # 10 + distillation: 蒸留 # 7 + learning rate: 学習率 # 6 + +# Words to leave in English. Only product names belong here. Things like `from_pretrained` +# or `AutoModel` are written as code in the docs, so they are hidden from the model already +# and need no help from us. +keep: + - Hugging Face + - Hub + - Spaces + - Transformers + - Inference Endpoints + - Trainer + - pipeline + +# Left off on purpose: "layer" is split almost evenly between 層 (35 pages) and レイヤー +# (23), with no clear winner, and 層 also turns up glued onto other words. Forcing either +# one would read worse than letting the model pick whichever fits the sentence. diff --git a/src/doc_builder/translate/__init__.py b/src/doc_builder/translate/__init__.py new file mode 100644 index 00000000..9d55d3f3 --- /dev/null +++ b/src/doc_builder/translate/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Translates a library's English docs into another language, remembering what it has already +done so it only pays for what changed. + + doc-builder translate transformers --lang ja --bucket /bucket + +Running it needs `torch` and `transformers`, so they live behind the `translate` extra rather +than being installed for everyone. Only the function that actually calls the model imports +them, and it does so at the last moment -- so everything else here, including the whole test +suite, runs without them. + +Start with `segment.py`. Hiding the parts of a page that must not be translated is the idea +the rest of this is built on. +""" diff --git a/src/doc_builder/translate/cache.py b/src/doc_builder/translate/cache.py new file mode 100644 index 00000000..82db8aeb --- /dev/null +++ b/src/doc_builder/translate/cache.py @@ -0,0 +1,152 @@ +""" +Remembers paragraphs we have already translated, so we only pay for new ones. + +Every paragraph gets an ID worked out from the English text itself (plus the model, the +prompt and the glossary). Same English in, same ID out. So to find out whether we have +already translated something, we work out its ID and look for it -- there is no separate +list mapping pages to paragraphs to keep in step. + +What lives on disk: + + cache/index.json a list of every ID we have stored + cache/blobs//.txt one translated paragraph per file + +The index file is just a shortcut. Without it, working out what is already translated +would mean listing roughly 20,000 files over the network; with it, that is a single read. + +Two habits borrowed from build_cache.py, for the same reasons it has them: + + - A plain folder works just as well as a bucket, so tests never touch the network. + - Nothing here is ever allowed to crash the run. If the cache misbehaves, the worst + that happens is we translate the paragraph again. +""" + +import hashlib +import json +import os +import traceback +from pathlib import Path + + +def segment_key(masked_text, model_id, prompt_version, glossary_sha, language): + """Work out the ID for one paragraph. + + Two things are deliberately left out of the ID. + + The surrounding paragraphs: if the translation depended on its neighbours but the ID + did not, then a paragraph pulled from the cache could differ from the same paragraph + translated fresh, and we would have no way to tell which one we were looking at. + + The heading it sits under: including that would mean renaming one section throws away + the translation of everything below it. + """ + sha = hashlib.sha256() + for part in (masked_text, model_id, prompt_version, glossary_sha, language): + sha.update(part.encode("utf-8")) + sha.update(b"\0") + return sha.hexdigest() + + +def sha256_text(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +class SegmentCache: + """Reads and writes translated paragraphs, filed under their ID. + + `root` can be an ordinary folder or a storage bucket mounted as one. Jobs mount + buckets so they look like normal folders, so either way this is just reading and + writing files -- there is no separate network path to write or test. + """ + + def __init__(self, root): + self.root = Path(root) / "cache" + self.blobs = self.root / "blobs" + self.index_path = self.root / "index.json" + self._known = None + + # -- the list of what we have -------------------------------------------------- + + def load_index(self): + """Which paragraphs do we already have? An unreadable index means we assume none.""" + if self._known is not None: + return self._known + try: + self._known = set(json.loads(self.index_path.read_text(encoding="utf-8"))) + except FileNotFoundError: + self._known = set() + except Exception: + traceback.print_exc() + print("[cache] could not read the index; carrying on as if nothing is cached") + self._known = set() + return self._known + + def save_index(self): + """Rebuild the index by looking at what is actually on disk. + + It is written to a temporary file and then moved into place. If we wrote it + directly and the run died halfway through, we would be left with a half-written + list, and every translation missing from it would be quietly redone. + """ + try: + keys = sorted(p.stem for p in self.blobs.rglob("*.txt")) + self.root.mkdir(parents=True, exist_ok=True) + tmp = self.index_path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(keys), encoding="utf-8") + os.replace(tmp, self.index_path) + self._known = set(keys) + return len(keys) + except Exception: + traceback.print_exc() + print("[cache] could not write the index; the translations are still saved") + return 0 + + # -- the translations themselves ------------------------------------------------ + + def _blob_path(self, key): + return self.blobs / key[:2] / f"{key}.txt" + + def get(self, key): + """The translation for this ID, or None if we do not have it or cannot read it. + + Any filesystem trouble counts as "we do not have it", quietly. It used to be only + "file not found", so when a bucket mount presented one blob path as a directory the + run printed a full traceback -- which looks like a crash in a job nobody is watching, + for something that is handled perfectly well by translating the paragraph again. + """ + try: + return self._blob_path(key).read_text(encoding="utf-8") + except OSError: + return None + except Exception: + traceback.print_exc() + return None + + def get_many(self, keys): + """Look up several IDs at once. Anything we do not have is simply left out.""" + found = {} + for key in keys: + text = self.get(key) + if text is not None: + found[key] = text + return found + + def put(self, key, text): + """Save one translated paragraph. Says whether it worked.""" + try: + path = self._blob_path(key) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".txt.tmp") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + return True + except Exception: + traceback.print_exc() + return False + + def put_many(self, items): + """Save a batch of translations and report how many were written. + + Remember to call save_index() afterwards, or the next run will not know they exist. + """ + return sum(1 for key, text in items.items() if self.put(key, text)) diff --git a/src/doc_builder/translate/pipeline.py b/src/doc_builder/translate/pipeline.py new file mode 100644 index 00000000..2824f168 --- /dev/null +++ b/src/doc_builder/translate/pipeline.py @@ -0,0 +1,511 @@ +""" +The middle of the pipeline: build the prompt, take a page apart, put it back together, and +run the model. + +Everything here except `translate_segments` is ordinary text handling with no GPU involved, +which is why the tests can cover it in under a second on a laptop. + +`torch` and `transformers` are imported inside `translate_segments` rather than at the top +of the file. That way a night where nothing has changed never loads them at all, and the +rest of this file can be imported anywhere. +""" + +import re +from functools import lru_cache +from pathlib import Path +from typing import NamedTuple + +import yaml + +from . import validate +from .cache import segment_key, sha256_text +from .segment import is_translatable, join_blocks, mask, placeholder_indices, restore, split_blocks + +# Change this to redo every translation from scratch. It is part of each paragraph's ID, so +# editing it throws the whole cache away -- about $2.50-10 of GPU time for transformers. +PROMPT_VERSION = "v4" + +LANGUAGE_NAMES = {"ja": "Japanese"} + +# How attention is computed. Continuous batching needs a paged backend, and the `paged|` +# prefix asks for one. +# +# sdpa is the default because it is built into PyTorch and always works. FlashAttention is +# faster, but it needs either the compiled flash-attn package or a prebuilt kernel from the +# Hub that matches the exact torch and CUDA version in the image -- and on a job image running +# torch 2.13/CUDA 13, kernels-community/flash-attn2 published nothing newer than torch 2.12, +# so loading the model failed outright. For a job that runs unattended overnight, a crash +# costs a whole day of translations while slower decoding costs minutes. +# +# Override with --attn-implementation when you know the image has FlashAttention available. +DEFAULT_ATTENTION = "paged|sdpa" + +# CUDA graphs record the GPU work once and replay it, which is faster -- but recording forbids +# copying between CPU and GPU, and a mixture-of-experts model does exactly that when it picks +# which experts to route each token to. Qwen3-30B-A3B died on this inside its MoE layer, so the +# safe default is off. Turn it on with --cuda-graphs for a dense model. +DEFAULT_CUDA_GRAPHS = False + +# On purpose, this does not name a particular library. The prompt is part of each +# paragraph's ID, so keeping it generic means the same boilerplate sentence translated for +# one library can be reused for another instead of being paid for twice. +SYSTEM_PROMPT = """You are translating technical documentation for a Hugging Face \ +library from English into {language}. + +Rules: +- Translate only the prose. Preserve the Markdown structure exactly. +- Tokens like {ph_open}0{ph_close} stand in for code, tags and link targets that were taken \ +out before you saw the text. Copy every one of them into your translation exactly once, \ +unchanged. Never translate, renumber, drop or repeat one. +- A phrase wrapped in two tokens, like {ph_open}0{ph_close}some text{ph_open}1{ph_close}, is a \ +link. Translate the words between the tokens and leave both tokens where they are. +- Keep heading levels (`#`, `##`) exactly as they are. +- Output only the translation. No preamble, no explanation, no code fences.{glossary}""" + +GLOSSARY_HEADER = "\n- Use these renderings exactly:" + +# Reasoning models wrap their working in tags like ... before giving an answer. +# We ask them not to (see enable_thinking below), but not every model honours that, so strip it +# here too -- otherwise the model's notes get cached and published as if they were a translation. +REASONING_RE = re.compile(r"\A\s*<(think|thinking|reasoning)>.*?\s*", re.DOTALL | re.IGNORECASE) + + +def strip_reasoning(text): + """Remove a leading block of model 'thinking' from a translation.""" + return REASONING_RE.sub("", text) + + +# The model sometimes copies a marker back in a different pair of brackets, right next to the +# real one, so `¤0¤the guide¤1¤` comes back as `⟦0⟧¤0¤the guide¤1¤⟦1⟧`. The real markers are +# present and correct, so every check passed and the page shipped with `⟦0⟧` sitting in the +# text -- 176 of them across 26 pages in the first full run. +# +# This is the one place where taking the syntax away is not an option: the model invents these +# unprompted, so there is nothing exposed to hide. Changing the marker delimiters was already +# tried and did not stop it. +# +# Only the numbered form is removed here. It is unambiguous -- `⟦` and `⟧` appear nowhere in +# the 732 English pages -- and the translation around it is intact, so throwing the paragraph +# away would lose good work for a bit of litter. Anything else using these brackets is left +# alone on purpose, so validate.py can reject it and we hear about a new habit instead of +# quietly cleaning up after it forever. +# +# Either bracket on either side, because the model is not consistent about which it uses: as +# well as `⟦1⟧` it writes `⟧1⟧`, with the closing one at both ends. Some of those sit in +# paragraphs that never had a marker to echo in the first place -- the English behind +# `ポジティブ⟧1⟧、🙁 ネガティブ⟧1⟧` is plain prose, "🙂 positive, 🙁 negative" -- so the number +# refers to nothing and there is no content at risk of being removed with it. +ECHOED_MARKER_RE = re.compile(r"[⟦⟧]\d+[⟦⟧]") + + +def strip_echoed_markers(text): + """Remove markers the model rewrote in the wrong brackets, e.g. `⟦0⟧`.""" + return ECHOED_MARKER_RE.sub("", text) + + +def glossary_path(language): + """Where the glossary for this language lives inside the installed package. + + Same approach `mock_imports` uses to find its own data files. + """ + return Path(__file__).parent.parent / "glossaries" / f"{language}.yml" + + +def load_glossary(path): + """Read a glossary file. Returns an empty one if there is no file there.""" + try: + with open(path, encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + except FileNotFoundError: + return {} + + +def glossary_sha(glossary): + """A short fingerprint of the glossary, so editing it re-translates the affected text.""" + return sha256_text(yaml.safe_dump(glossary or {}, sort_keys=True, allow_unicode=True)) + + +def glossary_for_segment(segment_text, glossary): + """Pick out just the glossary terms that actually appear in this paragraph. + + We could send the whole glossary every time, but there are around 14,000 paragraphs and + the model cannot reuse any of that work between them, so every unused line would be paid + for 14,000 times over. + """ + if not glossary: + return {} + low = segment_text.lower() + return {term: rendering for term, lowered, rendering in _pins(glossary) if lowered in low} + + +@lru_cache(maxsize=8) +def _pins_cached(items): + return tuple((term, term.lower(), rendering) for term, rendering in items) + + +def _pins(glossary): + """The glossary terms with their lowercase form worked out once, instead of per paragraph.""" + return _pins_cached(tuple(sorted((glossary.get("pin") or {}).items()))) + + +def build_prompt(segment_text, language, glossary): + """Build the instructions and the paragraph into a chat message for the model.""" + terms = glossary_for_segment(segment_text, glossary) + if terms: + lines = "".join(f"\n - {t} -> {r}" for t, r in sorted(terms.items())) + glossary_block = GLOSSARY_HEADER + lines + else: + glossary_block = "" + + system = SYSTEM_PROMPT.format( + language=LANGUAGE_NAMES.get(language, language), + ph_open="⟦", + ph_close="⟧", + glossary=glossary_block, + ) + return [ + {"role": "system", "content": system}, + {"role": "user", "content": segment_text}, + ] + + +# -- page pipeline (pure, no GPU) ------------------------------------------------ + + +class Unit(NamedTuple): + """One paragraph to translate, with its surrounding blank space kept to one side. + + Trimming the blank space before working out the ID matters more than it sounds. The last + chunk on a page keeps a trailing newline, so without trimming, the same sentence would + get two different IDs and be translated twice. The docs repeat a lot of boilerplate -- + "The abstract from the paper is the following:" appears across 513 model pages -- and + trimming means all of those share one translation. We keep the spacing so the page can be + rebuilt exactly as it was. + """ + + key: str + text: str + lead: str + trail: str + + +class PagePlan: + """A page broken into pieces, ready to translate. + + `parts` is the whole page in chunks. `units` picks out just the chunks with prose in + them and notes each one's ID. Everything else passes straight through untouched. + """ + + def __init__(self, page, source, language, model_id, gloss_sha): + self.page = page + self.source = source + self.masked, self.placeholders = mask(source) + self.parts = split_blocks(self.masked) + self.units = {} + for i, part in enumerate(self.parts): + if i % 2 != 0 or not is_translatable(part): + continue + core = part.strip() + lead = part[: len(part) - len(part.lstrip())] + trail = part[len(part.rstrip()) :] + key = segment_key(core, model_id, PROMPT_VERSION, gloss_sha, language) + self.units[i] = Unit(key, core, lead, trail) + + @property + def segments(self): + """The paragraphs to send to the model. Repeats collapse into one.""" + return {u.key: u.text for u in self.units.values()} + + +def assemble_page(plan, translations): + """Put a page back together from its translated paragraphs. + + If a paragraph is missing a translation, its English is left in place. That is on + purpose: a page that is mostly translated beats no page at all, and it still has to pass + the checks before anyone sees it. + """ + parts = list(plan.parts) + rejected = [] + for index, unit in plan.units.items(): + translated = translations.get(unit.key) + if translated is None: + continue + # Cleaned on the way out rather than on the way in, so a cache full of translations + # written before this existed is fixed by a --rebuild, with nothing retranslated. + translated = strip_echoed_markers(translated) + # Check this paragraph's markers before accepting it. Sorted, not in order: Japanese + # word order differs from English, so a model that moves a marker to the other end of + # the sentence is doing its job -- only dropping, repeating or inventing one is wrong. + # + # Doing this per paragraph rather than per page is what stops one bad paragraph costing + # the whole page. The model sometimes paraphrases a marker away when it stands for short + # inline code -- writing "from the checkpoint" instead of keeping `config.json`, or + # guessing the hidden text and typing it out. That was 4 paragraphs out of 402, and it + # failed 3 entire pages. Now those 4 stay English inside otherwise Japanese pages. + if sorted(placeholder_indices(translated)) != sorted(placeholder_indices(unit.text)): + rejected.append(unit.key) + continue + # Same treatment for brackets the model made up but did not number, like a lone `⟧`. + # These are dropped a paragraph at a time for the same reason as the markers above: a + # single stray character would otherwise cost a whole page of good Japanese. Two pages + # in the first full run came down to exactly one character each. + if validate.check_invented_brackets(unit.text, translated): + rejected.append(unit.key) + continue + parts[index] = f"{unit.lead}{translated}{unit.trail}" + masked_translation = join_blocks(parts) + return masked_translation, restore(masked_translation, plan.placeholders), rejected + + +def validate_plan(plan, masked_translation, glossary=None, restored=None): + return validate.validate_page( + plan.page, + plan.masked, + masked_translation, + glossary, + source=plan.source, + restored=restored, + ) + + +# -- disclosure ----------------------------------------------------------------- + +DISCLOSURE = { + "ja": ( + "> [!TIP]\n" + "> このページは機械翻訳されています。原文は[英語版]({en_url})を参照してください。\n" + "> 翻訳の問題は[こちら]({issue_url})から報告できます。\n" + ) +} + +# The library name is filled in rather than written in. This command works on any library, so +# hardcoding "transformers" would put a link to the wrong docs site on every page of every +# other library, and send its bug reports to the wrong repo. +EN_DOCS_URL = "https://huggingface.co/docs/{package}/en/{slug}" +DISCLOSURE_FALLBACK = ( + "> [!TIP]\n" + "> This page was machine-translated. See the [English original]({en_url}).\n" + "> Report translation problems [here]({issue_url}).\n" +) + +ISSUE_URL = "https://github.com/huggingface/{package}/issues/new?labels=documentation" +LICENSE_HEADER_RE = re.compile(r"\A(\n)", re.DOTALL) + + +def add_disclosure(page_text, page, language, package): + """Add the "this was machine-translated" notice, just below the licence header. + + This is us being upfront, not a substitute for review. Readers should know that nobody + checked this page. + """ + # If we have no notice written for this language, use the English one rather than adding + # nothing at all. Quietly publishing a machine translation with no warning on it is the + # exact thing this function exists to stop. + banner = DISCLOSURE.get(language, DISCLOSURE_FALLBACK) + slug = page[:-3] if page.endswith(".md") else page + banner = banner.format( + en_url=EN_DOCS_URL.format(package=package, slug=slug), + issue_url=ISSUE_URL.format(package=package), + ) + match = LICENSE_HEADER_RE.match(page_text) + if match: + return f"{match.group(1)}\n{banner}\n{page_text[match.end() :].lstrip(chr(10))}" + return f"{banner}\n{page_text}" + + +# -- toctree -------------------------------------------------------------------- + + +def toctree_dicts(node): + """Walk every entry in the sidebar file, top to bottom. + + Reading and writing both go through here, so if the sidebar format ever grows a new kind + of entry, this is the only place that needs to learn about it. + """ + if isinstance(node, dict): + yield node + for value in node.values(): + yield from toctree_dicts(value) + elif isinstance(node, list): + for item in node: + yield from toctree_dicts(item) + + +def toctree_values(node, field): + """Collect one field from every sidebar entry -- the titles, or the page names.""" + return [d[field] for d in toctree_dicts(node) if isinstance(d.get(field), str)] + + +def toctree_titles(node): + return toctree_values(node, "title") + + +def prune_toctree(node, keep_locals): + """Cut the sidebar down to just the pages we are translating. + + This is for test runs on a handful of pages. The sidebar lists every page in the docs, so + if we copied it over unchanged next to three translated pages, doc-builder would refuse to + build and tell us to remove the missing entries. It also stops a three-page test run from + translating all 756 sidebar titles. + + Sidebar entries come in two shapes: a page, or a group of pages. A group is kept only if + something inside it survived, so we do not leave empty headings behind. Returns None if + nothing is left at all. + """ + if isinstance(node, list): + kept = [p for p in (prune_toctree(item, keep_locals) for item in node) if p is not None] + return kept or None + if isinstance(node, dict): + if "local" in node: + return dict(node) if node["local"] in keep_locals else None + if "sections" in node: + sections = prune_toctree(node["sections"], keep_locals) + if sections is None: + return None + return {**node, "sections": sections} + return node + + +def apply_toctree_titles(node, translations): + """Swap the sidebar titles for their translations.""" + for d in toctree_dicts(node): + if isinstance(d.get("title"), str): + d["title"] = translations.get(d["title"], d["title"]) + return node + + +# -- model ---------------------------------------------------------------------- + + +def build_requests(segments, tokenizer, language, glossary, max_new_token_ratio=2.5): + """Turn each paragraph into something the model can read, plus a length limit. + + The length limit is worked out from the paragraph alone, not the whole prompt. The + instructions are about 150 tokens and a typical paragraph is only about 16, so measuring + the whole thing would hand a one-line heading roughly six times the room it needs. + """ + requests = [] + for key, text in segments.items(): + prompt = tokenizer.apply_chat_template( + build_prompt(text, language, glossary), + tokenize=True, + add_generation_prompt=True, + # Transformers v5 defaults this to True, which hands back a BatchEncoding rather + # than a plain list of token ids. Passing that straight to add_request makes the + # batcher iterate the dict's keys, so it ends up trying to build a tensor out of + # the strings "input_ids" and "attention_mask" -- which fails a long way from here + # with "too many dimensions 'str'". Ask for the list directly. + return_dict=False, + # Reasoning models think out loud before answering, and that thinking eats the + # whole token budget: Qwen3 returned pages of "Okay, the user wants me to + # translate..." and never reached the translation. Templates that don't know this + # option ignore it. + enable_thinking=False, + ) + if not (prompt and isinstance(prompt, list) and isinstance(prompt[0], int)): + raise TypeError( + f"expected a list of token ids from apply_chat_template, got {type(prompt).__name__}. " + "The tokenizer may have changed what it returns; see the note above." + ) + # Japanese output runs longer in tokens than English input, so one global cap + # would either truncate long blocks or waste KV budget. + content_tokens = len(tokenizer.encode(text, add_special_tokens=False)) + budget = int(content_tokens * max_new_token_ratio) + 48 + requests.append((key, prompt, budget)) + return requests + + +def translate_segments( + segments, + language, + glossary, + model_id, + max_new_token_ratio=2.5, + attn_implementation=DEFAULT_ATTENTION, + use_cuda_graph=DEFAULT_CUDA_GRAPHS, +): + """Translate a batch of paragraphs on the GPU. + + Paragraphs range from a few words to a couple of thousand, and continuous batching is + built for exactly that: as each one finishes, the next joins in, instead of everything + waiting for the longest one in the group. + + We drive it through the manager rather than `generate_batch` because the manager lets us + label each request ourselves. Labelling each one with its cache ID means results file + themselves away as they arrive, and it does not matter what order they come back in. + """ + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + from transformers.generation import ContinuousBatchingConfig, GenerationConfig + from transformers.generation.continuous_batching.utils import WorkloadHints + + if not segments: + return {}, [] + + tokenizer = AutoTokenizer.from_pretrained(model_id) + model = AutoModelForCausalLM.from_pretrained( + model_id, + attn_implementation=attn_implementation, + device_map="cuda", + dtype=torch.bfloat16, + ) + + requests = build_requests(segments, tokenizer, language, glossary, max_new_token_ratio) + max_prompt = max(len(p) for _, p, _ in requests) + max_generated = max(b for _, _, b in requests) + + cb_config = ContinuousBatchingConfig( + # Leave the GPU some room. By default the KV cache grows to fill whatever memory is + # left after the weights, which on an 80GB card meant 72GB in use and only 6.4GB free + # -- so the CUDA-graph warmup could not get the 9.9GB it wanted and gave up. Losing + # warmup only costs speed, but there is no reason to pay it. + max_memory_percent=0.8, + max_batch_tokens=16384, + use_cuda_graph=use_cuda_graph, + # Compiling the model is worth it on a long run but not a short one, where the + # setup time would be most of the job. + default_compile_level=1 if len(requests) > 500 else 0, + max_requests_per_batch=256, # keeps memory use in check on big batches + ) + generation_config = GenerationConfig( + max_new_tokens=max_generated, + # Always pick the most likely word rather than sampling, so running the same + # paragraph twice gives the same answer. Otherwise a cached translation and a fresh + # one could differ, with no way to tell which we were looking at. + do_sample=False, + eos_token_id=tokenizer.eos_token_id, + ) + # Telling it roughly what to expect lets it set aside the right amount of memory up + # front, instead of guessing. + hints = WorkloadHints( + max_prompt_length=max_prompt, + max_generated_length=max_generated, + num_requests=len(requests), + ) + + translations, failures = {}, [] + with model.continuous_batching_context_manager( + generation_config=generation_config, + continuous_batching_config=cb_config, + workload_hints=hints, + ) as manager: + for key, prompt, budget in requests: + manager.add_request(input_ids=prompt, request_id=key, max_new_tokens=budget) + + # Stop once we have heard back about every request. We cannot just loop until the + # results run out: the loop keeps going while the background worker is alive, and + # that worker is only shut down when we leave this block -- so waiting for it to + # finish from in here would hang forever. + for result in manager: + if result.error or not result.is_finished(): + failures.append((result.request_id, result.error or str(result.status))) + else: + decoded = tokenizer.decode(result.generated_tokens, skip_special_tokens=True) + translations[result.request_id] = strip_reasoning(decoded).strip() + if len(translations) + len(failures) >= len(requests): + break + + missing = set(segments) - set(translations) - {k for k, _ in failures} + for key in missing: + failures.append((key, "no result returned")) + return translations, failures diff --git a/src/doc_builder/translate/segment.py b/src/doc_builder/translate/segment.py new file mode 100644 index 00000000..cc9e236e --- /dev/null +++ b/src/doc_builder/translate/segment.py @@ -0,0 +1,205 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Hide everything in a doc page that must not be translated, then put it back afterwards. + +A doc page is a mix of prose and things that have to stay exactly as they are: code +samples, URLs, HTML tags, `[[autodoc]]` directives. If we handed the whole page to a +translation model, it would happily "translate" a variable name or a link. + +So we do this instead: + +1. Find each of those things and swap it for a numbered marker like `¤0¤`. +2. Send only what is left -- the prose -- to the model. +3. Swap the real content back in afterwards. + +The model never sees the code, so it cannot damage it. And because the markers have to +come back unchanged, we can check afterwards whether the model kept them (see +validate.py). That check is the main thing protecting these pages. + +Each thing on the page is either hidden or translated, never partly both. +""" + +import re + +# The marker delimiters. `¤` is the Unicode "generic currency sign" -- a character that exists +# to stand in for something else -- and it appears nowhere in the English or Japanese docs. +# +# It also looks nothing like a bracket, which was meant to stop the model muddling a marker +# with the `]` of a link. That turned out not to be the problem: with `¤` markers the model +# still returns `Gemma 4⟧¤396¤` -- dropping the `[` and inventing a `⟧` it was never shown. +# So the delimiter was never the cause. What fixed it was hiding both of a link's brackets +# (see `link_open` below), which turns a mangled link into an ordinary missing marker. `¤` is +# kept anyway; there is no reason to prefer a bracket-shaped marker. +PH_OPEN = "¤" +PH_CLOSE = "¤" + +PLACEHOLDER_RE = re.compile(f"{PH_OPEN}(\\d+){PH_CLOSE}") + +# The patterns run top to bottom. Once something is hidden, later patterns cannot see it, +# so the order is doing real work: +# +# comments go first, because a comment can contain literally anything, even code blocks +# code blocks before inline code, so ``` is not mistaken for a short `snippet` +# [[autodoc]] before the general [[...]] rule, so it keeps its indented list of methods +# inline code before tags, so `` in backticks is hidden as one piece +# +# Two of the patterns are deliberately fussy, to stop them swallowing half the page: +# +# "tag" needs a letter or / right after the `<`, because real prose says things like +# "when n < m". It stops at the next `<` and gives up after 600 characters. It is +# allowed to run across several lines: 88 tags in the docs are `` or `