Sync llms.txt #35
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync llms.txt | |
| # **What it does**: Generates docs.github.com/llms.txt, github.com/llms.txt, and | |
| # github.com/llms-full.txt from the page catalog and popularity data, then | |
| # opens PRs to update them. | |
| # **Why we have it**: Agents discover docs through llms.txt; the page list keeps | |
| # pace with what's actually popular without writers updating it by hand. | |
| # **Who does it impact**: Docs consumers via agents, and anyone landing on | |
| # github.com/llms.txt, github.com/llms-full.txt, or docs.github.com/llms.txt. | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - '.github/workflows/sync-llms-txt.yml' | |
| - 'data/llms-txt/**' | |
| - 'src/workflows/generate-llms-txt.ts' | |
| schedule: | |
| - cron: '20 16 * * 1' # Run every Monday at 16:20 UTC / 9:20 PDT / 8:20 PST | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' | |
| cancel-in-progress: true | |
| jobs: | |
| sync: | |
| name: Sync llms.txt | |
| if: github.repository == 'github/docs-internal' | |
| runs-on: ubuntu-latest | |
| env: | |
| BRANCH: sync-llms-txt | |
| steps: | |
| - name: Checkout docs-internal | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: ./.github/actions/node-npm-setup | |
| - name: Generate llms.txt for docs.github.com | |
| env: | |
| DOCS_BOT_PAT_BASE: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| npm run generate-llms-txt --silent -- \ | |
| --config data/llms-txt/config-docs.yml \ | |
| --output /tmp/docs-llms.txt | |
| echo "Generated docs llms.txt ($(wc -l < /tmp/docs-llms.txt) lines, $(wc -c < /tmp/docs-llms.txt) bytes)" | |
| - name: Generate llms.txt for github.com | |
| env: | |
| DOCS_BOT_PAT_BASE: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| npm run generate-llms-txt --silent -- \ | |
| --config data/llms-txt/config-monolith.yml \ | |
| --output /tmp/monolith-llms.txt | |
| echo "Generated monolith llms.txt ($(wc -l < /tmp/monolith-llms.txt) lines, $(wc -c < /tmp/monolith-llms.txt) bytes)" | |
| # ---------- PR to docs-internal: update data/llms-txt/docs.md ---------- | |
| - name: Diff docs llms.txt against committed copy | |
| id: diff_docs | |
| run: | | |
| if diff -q /tmp/docs-llms.txt data/llms-txt/docs.md > /dev/null 2>&1; then | |
| echo "No docs changes, skipping" | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Docs changes detected" | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Ensure sync branch exists in docs-internal | |
| if: steps.diff_docs.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| REPO="github/docs-internal" | |
| if gh api "repos/$REPO/git/ref/heads/$BRANCH" --jq '.object.sha' > /dev/null 2>&1; then | |
| echo "Branch $BRANCH exists, fetching" | |
| git -c url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf="https://github.com/" \ | |
| fetch origin "$BRANCH" | |
| git checkout "$BRANCH" | |
| else | |
| echo "Branch $BRANCH does not exist, creating from main" | |
| git checkout -b "$BRANCH" | |
| fi | |
| - name: Commit and push docs.md | |
| if: steps.diff_docs.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| cp /tmp/docs-llms.txt data/llms-txt/docs.md | |
| git config user.name "docs-bot" | |
| git config user.email "77750099+docs-bot@users.noreply.github.com" | |
| git add data/llms-txt/docs.md | |
| # diff_docs compares against main, but the sync branch may already | |
| # exist with this exact content (open PR from a prior run). In that | |
| # case there is nothing new to stage, and `git commit` would exit 1 | |
| # and fail the whole workflow. Skip the commit and push when the | |
| # branch is already up to date. | |
| if git diff --cached --quiet; then | |
| echo "Sync branch already has the latest generated docs.md; nothing to commit." | |
| else | |
| git commit -m "Update data/llms-txt/docs.md from popularity data" | |
| git push "https://x-access-token:${GH_TOKEN}@github.com/github/docs-internal.git" "$BRANCH" | |
| fi | |
| - name: Create or update docs-internal PR | |
| if: steps.diff_docs.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| REPO="github/docs-internal" | |
| if EXISTING_PR=$(gh pr list --repo "$REPO" --head "$BRANCH" \ | |
| --json number --jq '.[0].number' 2>/dev/null) && [ -n "$EXISTING_PR" ]; then | |
| echo "Docs PR #$EXISTING_PR already exists, updated with new commit" | |
| exit 0 | |
| fi | |
| RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| PR_BODY="The [sync-llms-txt workflow]($RUN_URL) generated this PR. | |
| Updates \`data/llms-txt/docs.md\`, served at https://docs.github.com/llms.txt. Built from the page catalog and popularity data using \`data/llms-txt/config-default.yml\` + \`config-docs.yml\`." | |
| gh pr create \ | |
| --repo "$REPO" \ | |
| --title "Update data/llms-txt/docs.md" \ | |
| --body "$PR_BODY" \ | |
| --head "$BRANCH" \ | |
| --base main \ | |
| --draft \ | |
| --label "llm-generated" | |
| # ---------- PR to github/github: update public/llms*.txt ---------- | |
| - name: Fetch current public llms files from github/github | |
| id: fetch_monolith | |
| env: | |
| GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| fetch_current() { | |
| local target_file="$1" | |
| local output_file="$2" | |
| if gh api -H "Accept: application/vnd.github.raw" \ | |
| "repos/github/github/contents/$target_file" \ | |
| > "$output_file" 2>/dev/null; then | |
| echo "Fetched current $target_file" | |
| else | |
| rm -f "$output_file" | |
| fi | |
| } | |
| fetch_current public/llms.txt /tmp/monolith-current-llms.txt | |
| fetch_current public/llms-full.txt /tmp/monolith-current-llms-full.txt | |
| - name: Diff monolith llms files | |
| id: diff_monolith | |
| run: | | |
| if [ -f /tmp/monolith-current-llms.txt ] && \ | |
| diff -q /tmp/monolith-llms.txt /tmp/monolith-current-llms.txt > /dev/null 2>&1 && \ | |
| [ -f /tmp/monolith-current-llms-full.txt ] && \ | |
| diff -q /tmp/monolith-llms.txt /tmp/monolith-current-llms-full.txt > /dev/null 2>&1; then | |
| echo "No monolith changes, skipping" | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Monolith changes detected" | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Ensure sync branch exists in github/github | |
| if: steps.diff_monolith.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| REPO="github/github" | |
| if gh api "repos/$REPO/git/ref/heads/$BRANCH" --jq '.object.sha' > /dev/null 2>&1; then | |
| echo "Branch $BRANCH exists" | |
| else | |
| DEFAULT_BRANCH=$(gh api "repos/$REPO" --jq '.default_branch') | |
| BASE_SHA=$(gh api "repos/$REPO/git/ref/heads/$DEFAULT_BRANCH" --jq '.object.sha') | |
| gh api "repos/$REPO/git/refs" \ | |
| --method POST \ | |
| -f ref="refs/heads/$BRANCH" \ | |
| -f sha="$BASE_SHA" | |
| echo "Created branch $BRANCH from $DEFAULT_BRANCH at $BASE_SHA" | |
| fi | |
| - name: Commit monolith llms files to github/github | |
| if: steps.diff_monolith.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| REPO="github/github" | |
| CONTENT=$(base64 -w 0 /tmp/monolith-llms.txt) | |
| sync_file() { | |
| local target_file="$1" | |
| local branch_copy="/tmp/monolith-branch-$(basename "$target_file")" | |
| if gh api -H "Accept: application/vnd.github.raw" \ | |
| "repos/$REPO/contents/$target_file?ref=$BRANCH" \ | |
| > "$branch_copy" 2>/dev/null && \ | |
| diff -q /tmp/monolith-llms.txt "$branch_copy" > /dev/null 2>&1; then | |
| echo "$target_file is already current on $BRANCH" | |
| return | |
| fi | |
| local existing_sha | |
| existing_sha=$(gh api "repos/$REPO/contents/$target_file?ref=$BRANCH" \ | |
| --jq '.sha' 2>/dev/null || true) | |
| local commit_args=(-f "message=Sync $(basename "$target_file") from docs.github.com" | |
| -f "content=$CONTENT" | |
| -f "branch=$BRANCH") | |
| if [ -n "$existing_sha" ]; then | |
| commit_args+=(-f "sha=$existing_sha") | |
| fi | |
| gh api "repos/$REPO/contents/$target_file" \ | |
| --method PUT \ | |
| "${commit_args[@]}" --jq '.commit.sha' | |
| } | |
| sync_file public/llms.txt | |
| sync_file public/llms-full.txt | |
| - name: Create or update github/github PR | |
| if: steps.diff_monolith.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| run: | | |
| REPO="github/github" | |
| RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| DEFAULT_BRANCH=$(gh api "repos/$REPO" --jq '.default_branch') | |
| PR_BODY="The [sync-llms-txt workflow]($RUN_URL) generated this PR. | |
| Updates the static generated files \`public/llms.txt\` and \`public/llms-full.txt\`, served at https://github.com/llms.txt and https://github.com/llms-full.txt. Both are built in docs-internal from the page catalog and popularity data using \`data/llms-txt/config-default.yml\` + \`config-monolith.yml\`. | |
| Initially, \`llms-full.txt\` is intentionally an exact duplicate of \`llms.txt\`. | |
| No feature flags. Static file in \`public/\`, no code changes. | |
| <!-- | |
| Labels for github/github PR template automation: | |
| (\`environment:production-dotcom\`) | |
| (\`risk:low\`) | |
| (\`validate:other\`) | |
| (\`mitigate:rollback\`) | |
| (\`backend/rails/api-only\`) | |
| pull_request_template_version=2 | |
| -->" | |
| if EXISTING_PR=$(gh pr list --repo "$REPO" --head "$BRANCH" \ | |
| --json number --jq '.[0].number' 2>/dev/null) && [ -n "$EXISTING_PR" ]; then | |
| gh api "repos/$REPO/pulls/$EXISTING_PR" \ | |
| --method PATCH \ | |
| -f title="Sync llms.txt and llms-full.txt from docs.github.com" \ | |
| -f body="$PR_BODY" > /dev/null | |
| echo "Monolith PR #$EXISTING_PR already exists, updated with new commit" | |
| exit 0 | |
| fi | |
| gh pr create \ | |
| --repo "$REPO" \ | |
| --title "Sync llms.txt and llms-full.txt from docs.github.com" \ | |
| --body "$PR_BODY" \ | |
| --head "$BRANCH" \ | |
| --base "$DEFAULT_BRANCH" \ | |
| --label "docs" | |
| - uses: ./.github/actions/create-workflow-failure-issue | |
| id: create-failure-issue | |
| if: ${{ failure() && github.event_name != 'workflow_dispatch' }} | |
| with: | |
| token: ${{ secrets.DOCS_BOT_PAT_BASE }} | |
| - uses: ./.github/actions/slack-alert | |
| if: ${{ failure() && github.event_name != 'workflow_dispatch' }} | |
| with: | |
| slack_token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} | |
| issue_url: ${{ steps.create-failure-issue.outputs.issue_url }} |