Add a translation pipeline

.github/workflows/build_main_documentation.yml

@@ -49,6 +49,13 @@ on:
       python_version:
         type: string
         description: "Python version for the build venv (e.g. '3.11'). Defaults to the runner's system Python."
+      translated_languages:
+        type: string
+        description: >
+          Languages whose pages are machine-translated, separated by spaces. Their source is
+          pulled from the hf-doc-build/doc-translate bucket instead of from the repo. Anything
+          listed here must also appear in `languages`. Leave it empty (the default) and every
+          language comes from the repo as usual.
     secrets:
       hf_token:
         required: true
@@ -186,6 +193,48 @@ jobs:
 
           doc-builder notebook-to-mdx ${{ env.doc_folder }} --open_notebook_prefix https://colab.research.google.com/github/${{ inputs.repo_owner }}/${{ inputs.package }}/blob/$branch$remaining_part
 
+      # Machine-translated pages are not kept in the repo. A separate nightly job writes them to
+      # a bucket, and this step fetches them so the build has something to work from. It does
+      # nothing at all unless a workflow asks for it, since every HF library shares this file.
+      #
+      # The bucket path uses the repo name, not the Python package name. The job that writes
+      # those files names them after the repo it cloned, so both sides have to agree.
+      - name: Sync machine translations
+        if: inputs.translated_languages != ''
+        env:
+          HF_TOKEN: ${{ secrets.hf_token }}
+          BUCKET: hf://buckets/hf-doc-build/doc-translate/translations/${{ inputs.package }}
+          # This is handed over as an environment variable rather than written straight into
+          # the script below. GitHub pastes a ${{ }} value into the script text before bash
+          # reads any of it, so quoting it there would not stop someone slipping in extra
+          # commands -- and this job is holding a token that can write to our buckets.
+          TRANSLATED_LANGUAGES: ${{ inputs.translated_languages }}
+        run: |
+          for lang in $TRANSLATED_LANGUAGES; do
+            target="${{ env.doc_folder }}/$lang"
+            # Download next to where the files are going, rather than into /tmp. Same disk, so
+            # putting them in place is a rename instead of copying 700 files. Cleared out first
+            # in case an earlier run was interrupted halfway through.
+            staged="$target.incoming"
+            rm -rf "$staged"
+            uvx --from huggingface_hub hf sync "$BUCKET/$lang" "$staged"
+
+            # Download everything first, then swap it in. If we wrote straight into place and
+            # the download died halfway, the build would carry on quite happily with pages
+            # missing and nobody would notice. The sidebar file is the last thing we expect to
+            # see, so if it is there the download finished.
+            if [ ! -f "$staged/_toctree.yml" ]; then
+              echo "::error::no _toctree.yml in $BUCKET/$lang - refusing to build a partial tree"
+              exit 1
+            fi
+
+            # Replace the folder outright rather than merging into it. That also clears out
+            # pages whose English original has since been deleted -- transformers has 20 of
+            # those sitting in its Japanese docs today.
+            rm -rf "$target" && mv "$staged" "$target"
+            echo "$lang: $(find "$target" -name '*.md' | wc -l) page(s) from $BUCKET/$lang"
+          done
+
       - name: Make documentation
         shell: bash
         env:

pyproject.toml

@@ -38,6 +38,16 @@ Repository = "https://github.com/huggingface/doc-builder"
 
 [project.optional-dependencies]
 transformers = ["transformers[dev]"]
+# `doc-builder translate` runs a real model, so these cannot be mocked away like the heavy
+# deps in `mock_deps/`. Kept an extra so the doc-build path never installs them.
+# accelerate is required by `device_map="cuda"` in translate/pipeline.py -- without it
+# from_pretrained raises before the model is even downloaded.
+#
+# kernels lets `flash_attention_2` fall back to the kernels-community/flash-attn2 kernel on
+# the Hub when the compiled flash-attn package isn't installed. Without it the fallback can't
+# happen and from_pretrained raises instead. Compiling flash-attn inside the job would be slow
+# and fragile, so the Hub kernel is the path we want.
+translate = ["torch", "transformers", "accelerate", "kernels>=0.11.0"]
 testing = [
     "pytest",
     "pytest-xdist",
@@ -92,6 +102,9 @@ build-backend = "setuptools.build_meta"
 [tool.setuptools]
 package-dir = {"" = "src"}
 
+[tool.setuptools.package-data]
+doc_builder = ["mock_deps/*.txt", "glossaries/*.yml"]
+
 [tool.setuptools.packages.find]
 where = ["src"]
 

setup.py

@@ -18,6 +18,7 @@
 extras = {}
 
 extras["transformers"] = ["transformers[dev]"]
+extras["translate"] = ["torch", "transformers", "accelerate", "kernels>=0.11.0"]
 extras["testing"] = [
     "pytest",
     "pytest-xdist",
@@ -49,7 +50,7 @@
     keywords="doc documentation doc-builder huggingface hugging face",
     url="https://github.com/huggingface/doc-builder",
     package_dir={"": "src"},
-    package_data={"doc_builder": ["mock_deps/*.txt"]},
+    package_data={"doc_builder": ["mock_deps/*.txt", "glossaries/*.yml"]},
     include_package_data=True,
     packages=find_packages("src"),
     extras_require=extras,

src/doc_builder/commands/doc_builder_cli.py

@@ -24,6 +24,7 @@
 from doc_builder.commands.preview import preview_command_parser
 from doc_builder.commands.push import push_command_parser
 from doc_builder.commands.style import style_command_parser
+from doc_builder.commands.translate import translate_command_parser
 
 
 def main():
@@ -40,6 +41,7 @@ def main():
     style_command_parser(subparsers=subparsers)
     preview_command_parser(subparsers=subparsers)
     push_command_parser(subparsers=subparsers)
+    translate_command_parser(subparsers=subparsers)
 
     # Let's go
     args = parser.parse_args()