From be215a58d49a8a5208363b70294dd9313d006b51 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Fri, 2 Oct 2026 16:19:51 +0100 Subject: [PATCH 01/18] ci: split push.yml into reusable workflows, share build steps via composite action Breaks the single push.yml into comparison-tests.yml, execution-tests.yml, and lint.yml, called from push.yml as reusable workflows. Each OS job's install/build steps (plus the Windows Defender exclusion and the copy-to-C:\source\ts-loader workaround) are now shared via the build-ts-loader composite action, also reused by the Windows test probe. Also generalizes the probe to run on Ubuntu/macOS/Windows via a new os input, defaulting to all three. Co-Authored-By: Claude Sonnet 5 --- .github/actions/build-ts-loader/action.yml | 37 ++++++ .github/workflows/comparison-tests.yml | 30 +++++ .github/workflows/execution-tests.yml | 66 +++++++++++ .github/workflows/lint.yml | 17 +++ .github/workflows/push.yml | 129 +-------------------- .github/workflows/windows-test-probe.yml | 104 +++++++++++------ AGENTS.md | 2 +- 7 files changed, 227 insertions(+), 158 deletions(-) create mode 100644 .github/actions/build-ts-loader/action.yml create mode 100644 .github/workflows/comparison-tests.yml create mode 100644 .github/workflows/execution-tests.yml create mode 100644 .github/workflows/lint.yml diff --git a/.github/actions/build-ts-loader/action.yml b/.github/actions/build-ts-loader/action.yml new file mode 100644 index 000000000..a874b7aba --- /dev/null +++ b/.github/actions/build-ts-loader/action.yml @@ -0,0 +1,37 @@ +name: Build ts-loader +description: > + Install and build ts-loader (expects actions/checkout to have already run). + On Windows runners this excludes the working directory from Windows + Defender's real-time scanning and copies the checkout to + C:\source\ts-loader first - Defender's scanning of the heavy file I/O that + yarn does during install/build/test is a major source of slowdown and + flakiness on hosted Windows runners. +inputs: + working-directory: + description: 'Directory to run install/build in. Use C:\source\ts-loader on Windows runners.' + required: false + default: '.' +runs: + using: composite + steps: + - name: exclude working directory from Windows Defender + if: runner.os == 'Windows' + shell: pwsh + run: Add-MpPreference -ExclusionPath "C:\source" + + - name: copy files + if: runner.os == 'Windows' + shell: pwsh + run: | + New-Item C:\source\ts-loader -ItemType Directory + Copy-Item .\* C:\source\ts-loader -Recurse -Force + + - name: install + shell: bash + working-directory: ${{ inputs.working-directory }} + run: yarn install + + - name: build + shell: bash + working-directory: ${{ inputs.working-directory }} + run: yarn build diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml new file mode 100644 index 000000000..5ba39a1a3 --- /dev/null +++ b/.github/workflows/comparison-tests.yml @@ -0,0 +1,30 @@ +name: Comparison Tests + +on: + workflow_call: + +jobs: + comparison_test_ubuntu: + name: Comparison Tests Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader + + - name: test + run: sudo yarn comparison-tests + + comparison_test_windows: + name: Comparison Tests Windows + runs-on: windows-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader + with: + working-directory: C:\source\ts-loader + + - name: test + run: yarn comparison-tests + working-directory: C:\source\ts-loader diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml new file mode 100644 index 000000000..c3d6c1a1c --- /dev/null +++ b/.github/workflows/execution-tests.yml @@ -0,0 +1,66 @@ +name: Execution Tests + +on: + workflow_call: + +jobs: + execution_test_ubuntu: + name: Execution Tests Ubuntu + strategy: + matrix: + node: [26, 24, 22] + ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + webpack: [4, 5] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: install node + uses: actions/setup-node@v7 + with: + node-version: ${{ matrix.node }} + + - uses: ./.github/actions/build-ts-loader + + - name: install typescript + run: yarn add typescript@${{ matrix.ts }} + + - name: install webpack 4 + if: matrix.webpack == 4 + run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 + + - name: test + run: yarn execution-tests + + execution_test_windows: + name: Execution Tests Windows + strategy: + matrix: + node: [26, 24, 22] + ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + webpack: [4, 5] + runs-on: windows-latest + steps: + - uses: actions/checkout@v7 + + - name: install node + uses: actions/setup-node@v7 + with: + node-version: ${{ matrix.node }} + + - uses: ./.github/actions/build-ts-loader + with: + working-directory: C:\source\ts-loader + + - name: install typescript + run: yarn add typescript@${{ matrix.ts }} + working-directory: C:\source\ts-loader + + - name: install webpack 4 + if: matrix.webpack == 4 + run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 + working-directory: C:\source\ts-loader + + - name: test + run: yarn execution-tests + working-directory: C:\source\ts-loader diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..ae88d1b6d --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,17 @@ +name: Lint + +on: + workflow_call: + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: install + run: yarn install + + - name: lint + run: yarn lint diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c13c85822..54943db96 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -7,130 +7,11 @@ concurrency: cancel-in-progress: true jobs: - comparison_test_ubuntu: - name: Comparison Tests Ubuntu - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 + comparison-tests: + uses: ./.github/workflows/comparison-tests.yml - - name: install - run: yarn install - - - name: build - run: yarn build - - - name: test - run: sudo yarn comparison-tests - - comparison_test_windows: - name: Comparison Tests Windows - runs-on: windows-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - - - name: copy files - shell: pwsh - run: | - New-Item C:\source\ts-loader -ItemType Directory - Copy-Item .\* C:\source\ts-loader -Recurse -Force - - - name: install - run: yarn install - working-directory: C:\source\ts-loader - - - name: build - run: yarn build - working-directory: C:\source\ts-loader - - - name: test - run: yarn comparison-tests - working-directory: C:\source\ts-loader - - execution_test_ubuntu: - name: Execution Tests Ubuntu - strategy: - matrix: - node: [26, 24, 22] - ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now - webpack: [4, 5] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: install node - uses: actions/setup-node@v7 - with: - node-version: ${{ matrix.node }} - - - name: install - run: yarn install - - - name: build - run: yarn build - - - name: install typescript - run: yarn add typescript@${{ matrix.ts }} - - - name: install webpack 4 - if: matrix.webpack == 4 - run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 - - - name: test - run: yarn execution-tests - - execution_test_windows: - name: Execution Tests Windows - strategy: - matrix: - node: [26, 24, 22] - ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now - webpack: [4, 5] - runs-on: windows-latest - steps: - - uses: actions/checkout@v7 - - - name: install node - uses: actions/setup-node@v7 - with: - node-version: ${{ matrix.node }} - - - name: copy files - shell: pwsh - run: | - New-Item C:\source\ts-loader -ItemType Directory - Copy-Item .\* C:\source\ts-loader -Recurse -Force - - - name: install - run: yarn install - working-directory: C:\source\ts-loader - - - name: build - run: yarn build - working-directory: C:\source\ts-loader - - - name: install typescript - run: yarn add typescript@${{ matrix.ts }} - working-directory: C:\source\ts-loader - - - name: install webpack 4 - if: matrix.webpack == 4 - run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 - working-directory: C:\source\ts-loader - - - name: test - run: yarn execution-tests - working-directory: C:\source\ts-loader + execution-tests: + uses: ./.github/workflows/execution-tests.yml lint: - name: Lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: install - run: yarn install - - - name: lint - run: yarn lint + uses: ./.github/workflows/lint.yml diff --git a/.github/workflows/windows-test-probe.yml b/.github/workflows/windows-test-probe.yml index 0894608b8..cd1ce5aef 100644 --- a/.github/workflows/windows-test-probe.yml +++ b/.github/workflows/windows-test-probe.yml @@ -1,14 +1,24 @@ name: Windows test probe -# A deliberately lean, on-demand counterpart to the `comparison_test_windows` -# job in push.yml - triggered only via workflow_dispatch (never on push/PR, -# so it adds no cost/noise to normal CI) and able to target a single test or -# a name pattern, so a single iteration comes back in a couple of minutes -# instead of running the full comparison/execution test matrix. +# A deliberately lean, on-demand counterpart to the `comparison_test_*` jobs +# in comparison-tests.yml - triggered only via workflow_dispatch (never on +# push/PR, so it adds no cost/noise to normal CI) and able to target a single +# test or a name pattern, so a single iteration comes back in a couple of +# minutes instead of running the full comparison/execution test matrix. on: workflow_dispatch: inputs: + os: + description: 'Which OS(es) to run the probe on' + required: false + type: choice + default: all + options: + - all + - ubuntu-latest + - macos-latest + - windows-latest single_test: description: 'Run only this comparison test (matches the test directory name, e.g. "dependencyErrors")' required: false @@ -19,46 +29,74 @@ on: type: string jobs: - probe: - name: Windows comparison test probe - runs-on: windows-latest + probe_ubuntu: + name: Comparison test probe (Ubuntu) + if: inputs.os == 'all' || inputs.os == 'ubuntu-latest' + runs-on: ubuntu-latest timeout-minutes: 25 steps: - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader - # Windows Defender's real-time scanning adds unpredictable latency to - # file I/O, and this probe does a lot of it (installing, building, then - # running comparison tests) - excluding the working directory keeps - # that noise out of the results. - - name: exclude working directory from Windows Defender - shell: pwsh - run: Add-MpPreference -ExclusionPath "C:\source" - - - name: copy files - shell: pwsh + - name: test + shell: bash + env: + SINGLE_TEST: ${{ inputs.single_test }} + MATCH_TEST: ${{ inputs.match_test }} run: | - New-Item C:\source\ts-loader -ItemType Directory - Copy-Item .\* C:\source\ts-loader -Recurse -Force + if [ -n "$SINGLE_TEST" ]; then + sudo yarn comparison-tests --single-test "$SINGLE_TEST" + elif [ -n "$MATCH_TEST" ]; then + sudo yarn comparison-tests --match-test "$MATCH_TEST" + else + sudo yarn comparison-tests + fi - - name: install - run: yarn install - working-directory: C:\source\ts-loader + probe_macos: + name: Comparison test probe (macOS) + if: inputs.os == 'all' || inputs.os == 'macos-latest' + runs-on: macos-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader - - name: build - run: yarn build - working-directory: C:\source\ts-loader + - name: test + shell: bash + env: + SINGLE_TEST: ${{ inputs.single_test }} + MATCH_TEST: ${{ inputs.match_test }} + run: | + if [ -n "$SINGLE_TEST" ]; then + yarn comparison-tests --single-test "$SINGLE_TEST" + elif [ -n "$MATCH_TEST" ]; then + yarn comparison-tests --match-test "$MATCH_TEST" + else + yarn comparison-tests + fi + + probe_windows: + name: Comparison test probe (Windows) + if: inputs.os == 'all' || inputs.os == 'windows-latest' + runs-on: windows-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader + with: + working-directory: C:\source\ts-loader - name: test - shell: pwsh + shell: bash working-directory: C:\source\ts-loader env: SINGLE_TEST: ${{ inputs.single_test }} MATCH_TEST: ${{ inputs.match_test }} run: | - if ($env:SINGLE_TEST -ne "") { - yarn comparison-tests --single-test "$env:SINGLE_TEST" - } elseif ($env:MATCH_TEST -ne "") { - yarn comparison-tests --match-test "$env:MATCH_TEST" - } else { + if [ -n "$SINGLE_TEST" ]; then + yarn comparison-tests --single-test "$SINGLE_TEST" + elif [ -n "$MATCH_TEST" ]; then + yarn comparison-tests --match-test "$MATCH_TEST" + else yarn comparison-tests - } + fi diff --git a/AGENTS.md b/AGENTS.md index 957fa43c1..33ea4e393 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ yarn comparison-tests -- --save-output --single-test # regenerate o ## Execution tests (`test/execution-tests/`) -Each sub-directory is a mini webpack project with a Karma/Jasmine test suite. The harness compiles the project and **runs the compiled code** — useful for asserting correct runtime behaviour. These are matrix-tested in CI across multiple Node and TypeScript versions (see `.github/workflows/push.yml` for the current matrix). +Each sub-directory is a mini webpack project with a Karma/Jasmine test suite. The harness compiles the project and **runs the compiled code** — useful for asserting correct runtime behaviour. These are matrix-tested in CI across multiple Node and TypeScript versions (see `.github/workflows/execution-tests.yml` for the current matrix). Full docs: [`test/execution-tests/README.md`](test/execution-tests/README.md) From 86cd9c4326b15f9aee9f29beafffb109fb24ebe3 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 09:59:18 +0100 Subject: [PATCH 02/18] ci: rename windows-test-probe.yml to test-probe.yml Reflects that it now probes comparison tests on Ubuntu/macOS/Windows, not just Windows. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/{windows-test-probe.yml => test-probe.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{windows-test-probe.yml => test-probe.yml} (100%) diff --git a/.github/workflows/windows-test-probe.yml b/.github/workflows/test-probe.yml similarity index 100% rename from .github/workflows/windows-test-probe.yml rename to .github/workflows/test-probe.yml From e1ba1aadc67ea84fb7413e04dcf1c6fd44b44ccf Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 10:05:22 +0100 Subject: [PATCH 03/18] docs: document test-probe.yml usage in AGENTS.md Co-Authored-By: Claude Sonnet 5 --- .github/workflows/test-probe.yml | 2 +- AGENTS.md | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-probe.yml b/.github/workflows/test-probe.yml index cd1ce5aef..65f3390d2 100644 --- a/.github/workflows/test-probe.yml +++ b/.github/workflows/test-probe.yml @@ -1,4 +1,4 @@ -name: Windows test probe +name: Test probe # A deliberately lean, on-demand counterpart to the `comparison_test_*` jobs # in comparison-tests.yml - triggered only via workflow_dispatch (never on diff --git a/AGENTS.md b/AGENTS.md index 33ea4e393..7fcd07a35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,32 @@ yarn comparison-tests -- --save-output --single-test # regenerate o > Note: test name casing must be exact when using `--single-test`. +### Test probe workflow + +`.github/workflows/test-probe.yml` (registered on `main`, so dispatchable against any branch/ref) runs just the comparison tests via `workflow_dispatch` on Ubuntu, macOS, and/or Windows — much faster than the full `push.yml` matrix (which also runs the full Node/TS/webpack execution-test matrix). Use it to iterate on a failing test, or an OS-specific failure, without asking a human to relay CI output. + +Three separate jobs (`probe_ubuntu`/`probe_macos`/`probe_windows`), each gated by the `os` input — defaults to `all`; pass `ubuntu-latest`, `macos-latest`, or `windows-latest` to run just one. All three share their install/build steps (including the Windows Defender-exclusion + copy-to-`C:\source\ts-loader` workaround) via the composite action `.github/actions/build-ts-loader/action.yml`, which is also used by `comparison-tests.yml` and `execution-tests.yml` — edit that action, not each workflow, if the build steps themselves need to change. + +Requires `gh` CLI authenticated with the `workflow` scope (`gh auth login`, then `gh auth refresh -s workflow` if `gh auth status` doesn't already list `workflow` — both scopes need a human to complete the browser device-flow prompt, they can't be scripted). + +````bash +# trigger — omit single_test/match_test to run the full comparison-test suite; +# omit os (or pass os=all) to run on every platform +gh workflow run test-probe.yml --repo TypeStrong/ts-loader \ + --ref -f os=windows-latest -f single_test= # one test, Windows only +gh workflow run test-probe.yml --repo TypeStrong/ts-loader \ + --ref -f match_test='^(testA|testB)$' # several, by regex, all OSes + +# the trigger command prints the run URL directly - grab the numeric id from it, then: +gh run watch --repo TypeStrong/ts-loader --exit-status # blocks until done + +# `gh run watch` can itself fail on a transient network blip even when the run +# succeeded - always verify conclusion this way rather than trusting its exit code +gh run view --repo TypeStrong/ts-loader --json status,conclusion + +gh run view --repo TypeStrong/ts-loader --log-failed # full failure log text +```` + ## Execution tests (`test/execution-tests/`) Each sub-directory is a mini webpack project with a Karma/Jasmine test suite. The harness compiles the project and **runs the compiled code** — useful for asserting correct runtime behaviour. These are matrix-tested in CI across multiple Node and TypeScript versions (see `.github/workflows/execution-tests.yml` for the current matrix). From efbe534c3510fbe7270c513810f036dfb11c157a Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 13:24:14 +0100 Subject: [PATCH 04/18] ci: add macOS jobs to comparison-tests, execution-tests, and benchmark Co-Authored-By: Claude Sonnet 5 --- .github/workflows/benchmark.yml | 70 ++++++++++++++++++++++++++ .github/workflows/comparison-tests.yml | 11 ++++ .github/workflows/execution-tests.yml | 28 +++++++++++ 3 files changed, 109 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e1cd1578c..e5303538a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -86,6 +86,76 @@ jobs: title: 'Benchmark (Ubuntu)', }); + benchmark_macos: + name: Benchmark macOS + runs-on: macos-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + + - name: install & build (PR branch) + run: | + yarn install + yarn build + + # Checked out into a sibling worktree rather than a second checkout step + # so both builds can run from disk at the same time. + - name: checkout base branch into a worktree + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || 'origin/main' }} + run: git worktree add ../ts-loader-main "$BASE_SHA" + + - name: install & build (base branch) + working-directory: ../ts-loader-main + run: | + yarn install + yarn build + + # --root-a is the PR branch checked out at $GITHUB_WORKSPACE above; + # --root-b is the base branch worktree built in the previous step. + - name: run benchmark + run: | + yarn benchmark -- \ + --root-a "$GITHUB_WORKSPACE" --label-a "PR branch" \ + --root-b "$GITHUB_WORKSPACE/../ts-loader-main" --label-b "base branch" + + - name: write job summary + if: always() + run: cat .benchmark/benchmark-results.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + + - name: upload results + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-results-macos + path: .benchmark/benchmark-results.* + if-no-files-found: ignore + + # Only runs with a writable token - GitHub automatically issues a + # read-only GITHUB_TOKEN for fork PRs, so this step is a silent no-op + # there. The step summary above still carries the numbers for everyone. + - name: comment on PR + if: always() && github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const path = require('path'); + const postComment = require(path.join(process.env.GITHUB_WORKSPACE, '.github/scripts/post-benchmark-comment.js')); + await postComment({ + github, + context, + resultsPath: '.benchmark/benchmark-results.md', + marker: '', + title: 'Benchmark (macOS)', + }); + benchmark_windows: name: Benchmark Windows runs-on: windows-latest diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml index 5ba39a1a3..b82cc6b56 100644 --- a/.github/workflows/comparison-tests.yml +++ b/.github/workflows/comparison-tests.yml @@ -15,6 +15,17 @@ jobs: - name: test run: sudo yarn comparison-tests + comparison_test_macos: + name: Comparison Tests macOS + runs-on: macos-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader + + - name: test + run: yarn comparison-tests + comparison_test_windows: name: Comparison Tests Windows runs-on: windows-latest diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml index c3d6c1a1c..aa5c52aaf 100644 --- a/.github/workflows/execution-tests.yml +++ b/.github/workflows/execution-tests.yml @@ -32,6 +32,34 @@ jobs: - name: test run: yarn execution-tests + execution_test_macos: + name: Execution Tests macOS + strategy: + matrix: + node: [26, 24, 22] + ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + webpack: [4, 5] + runs-on: macos-latest + steps: + - uses: actions/checkout@v7 + + - name: install node + uses: actions/setup-node@v7 + with: + node-version: ${{ matrix.node }} + + - uses: ./.github/actions/build-ts-loader + + - name: install typescript + run: yarn add typescript@${{ matrix.ts }} + + - name: install webpack 4 + if: matrix.webpack == 4 + run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 + + - name: test + run: yarn execution-tests + execution_test_windows: name: Execution Tests Windows strategy: From 661837b8f91336d7fb6ca0d2c3fccf871d80e6a1 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 13:44:40 +0100 Subject: [PATCH 05/18] chore: remove lower version of ts from exectuion test matrix --- .github/workflows/execution-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml index aa5c52aaf..342c39276 100644 --- a/.github/workflows/execution-tests.yml +++ b/.github/workflows/execution-tests.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: node: [26, 24, 22] - ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + ts: [5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now webpack: [4, 5] runs-on: ubuntu-latest steps: @@ -37,7 +37,7 @@ jobs: strategy: matrix: node: [26, 24, 22] - ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + ts: [5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now webpack: [4, 5] runs-on: macos-latest steps: @@ -65,7 +65,7 @@ jobs: strategy: matrix: node: [26, 24, 22] - ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + ts: [5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now webpack: [4, 5] runs-on: windows-latest steps: From c28fc602fa53a0678d23b3975b01b9442fccbf96 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 17:03:37 +0100 Subject: [PATCH 06/18] chore: drop sudo --- .github/workflows/comparison-tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml index b82cc6b56..d04fff1c3 100644 --- a/.github/workflows/comparison-tests.yml +++ b/.github/workflows/comparison-tests.yml @@ -13,18 +13,18 @@ jobs: - uses: ./.github/actions/build-ts-loader - name: test - run: sudo yarn comparison-tests + run: yarn comparison-tests - comparison_test_macos: - name: Comparison Tests macOS - runs-on: macos-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - - uses: ./.github/actions/build-ts-loader + # comparison_test_macos: + # name: Comparison Tests macOS + # runs-on: macos-latest + # timeout-minutes: 25 + # steps: + # - uses: actions/checkout@v7 + # - uses: ./.github/actions/build-ts-loader - - name: test - run: yarn comparison-tests + # - name: test + # run: yarn comparison-tests comparison_test_windows: name: Comparison Tests Windows From c2428c89ea16db010be98a1e9ea00b4abc6349c3 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 17:09:48 +0100 Subject: [PATCH 07/18] chore: drop sudo --- .github/workflows/comparison-tests.yml | 18 ++++----- .github/workflows/execution-tests.yml | 54 +++++++++++++------------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml index d04fff1c3..d7853d1ed 100644 --- a/.github/workflows/comparison-tests.yml +++ b/.github/workflows/comparison-tests.yml @@ -15,16 +15,16 @@ jobs: - name: test run: yarn comparison-tests - # comparison_test_macos: - # name: Comparison Tests macOS - # runs-on: macos-latest - # timeout-minutes: 25 - # steps: - # - uses: actions/checkout@v7 - # - uses: ./.github/actions/build-ts-loader + comparison_test_macos: + name: Comparison Tests macOS + runs-on: macos-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader - # - name: test - # run: yarn comparison-tests + - name: test + run: yarn comparison-tests comparison_test_windows: name: Comparison Tests Windows diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml index 342c39276..ed5e41c3c 100644 --- a/.github/workflows/execution-tests.yml +++ b/.github/workflows/execution-tests.yml @@ -32,33 +32,33 @@ jobs: - name: test run: yarn execution-tests - execution_test_macos: - name: Execution Tests macOS - strategy: - matrix: - node: [26, 24, 22] - ts: [5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now - webpack: [4, 5] - runs-on: macos-latest - steps: - - uses: actions/checkout@v7 - - - name: install node - uses: actions/setup-node@v7 - with: - node-version: ${{ matrix.node }} - - - uses: ./.github/actions/build-ts-loader - - - name: install typescript - run: yarn add typescript@${{ matrix.ts }} - - - name: install webpack 4 - if: matrix.webpack == 4 - run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 - - - name: test - run: yarn execution-tests + # execution_test_macos: + # name: Execution Tests macOS + # strategy: + # matrix: + # node: [26, 24, 22] + # ts: [5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + # webpack: [4, 5] + # runs-on: macos-latest + # steps: + # - uses: actions/checkout@v7 + + # - name: install node + # uses: actions/setup-node@v7 + # with: + # node-version: ${{ matrix.node }} + + # - uses: ./.github/actions/build-ts-loader + + # - name: install typescript + # run: yarn add typescript@${{ matrix.ts }} + + # - name: install webpack 4 + # if: matrix.webpack == 4 + # run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 + + # - name: test + # run: yarn execution-tests execution_test_windows: name: Execution Tests Windows From f8a8553bb2bcdc917b8626ac7affc9186bbde2b2 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 17:22:25 +0100 Subject: [PATCH 08/18] chore: shorter names --- .github/workflows/{push.yml => ci.yml} | 4 +++- .github/workflows/comparison-tests.yml | 6 +++--- .github/workflows/execution-tests.yml | 6 +++--- .github/workflows/test-probe.yml | 6 +++--- 4 files changed, 12 insertions(+), 10 deletions(-) rename .github/workflows/{push.yml => ci.yml} (85%) diff --git a/.github/workflows/push.yml b/.github/workflows/ci.yml similarity index 85% rename from .github/workflows/push.yml rename to .github/workflows/ci.yml index 54943db96..ce44b4138 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: build and test +name: ci on: [push, pull_request, workflow_dispatch] @@ -8,9 +8,11 @@ concurrency: jobs: comparison-tests: + name: Comparison tests uses: ./.github/workflows/comparison-tests.yml execution-tests: + name: Execution tests uses: ./.github/workflows/execution-tests.yml lint: diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml index d7853d1ed..a2885c6e6 100644 --- a/.github/workflows/comparison-tests.yml +++ b/.github/workflows/comparison-tests.yml @@ -5,7 +5,7 @@ on: jobs: comparison_test_ubuntu: - name: Comparison Tests Ubuntu + name: ubuntu runs-on: ubuntu-latest timeout-minutes: 25 steps: @@ -16,7 +16,7 @@ jobs: run: yarn comparison-tests comparison_test_macos: - name: Comparison Tests macOS + name: macOS runs-on: macos-latest timeout-minutes: 25 steps: @@ -27,7 +27,7 @@ jobs: run: yarn comparison-tests comparison_test_windows: - name: Comparison Tests Windows + name: windows runs-on: windows-latest timeout-minutes: 25 steps: diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml index ed5e41c3c..e2ad36b10 100644 --- a/.github/workflows/execution-tests.yml +++ b/.github/workflows/execution-tests.yml @@ -5,7 +5,7 @@ on: jobs: execution_test_ubuntu: - name: Execution Tests Ubuntu + name: ubuntu strategy: matrix: node: [26, 24, 22] @@ -33,7 +33,7 @@ jobs: run: yarn execution-tests # execution_test_macos: - # name: Execution Tests macOS + # name: macOS # strategy: # matrix: # node: [26, 24, 22] @@ -61,7 +61,7 @@ jobs: # run: yarn execution-tests execution_test_windows: - name: Execution Tests Windows + name: windows strategy: matrix: node: [26, 24, 22] diff --git a/.github/workflows/test-probe.yml b/.github/workflows/test-probe.yml index 65f3390d2..490f6cc18 100644 --- a/.github/workflows/test-probe.yml +++ b/.github/workflows/test-probe.yml @@ -45,11 +45,11 @@ jobs: MATCH_TEST: ${{ inputs.match_test }} run: | if [ -n "$SINGLE_TEST" ]; then - sudo yarn comparison-tests --single-test "$SINGLE_TEST" + yarn comparison-tests --single-test "$SINGLE_TEST" elif [ -n "$MATCH_TEST" ]; then - sudo yarn comparison-tests --match-test "$MATCH_TEST" + yarn comparison-tests --match-test "$MATCH_TEST" else - sudo yarn comparison-tests + yarn comparison-tests fi probe_macos: From 4a47dede56eb38b085fde6c5e52d747fdcdf6cc1 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 17:36:51 +0100 Subject: [PATCH 09/18] chore: renames --- .github/workflows/benchmark.yml | 6 +++--- .github/workflows/comparison-tests.yml | 2 +- .github/workflows/execution-tests.yml | 2 +- .github/workflows/lint.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e5303538a..5d05dd36c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,7 +17,7 @@ permissions: jobs: benchmark_ubuntu: - name: Benchmark Ubuntu + name: ubuntu runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -87,7 +87,7 @@ jobs: }); benchmark_macos: - name: Benchmark macOS + name: macos runs-on: macos-latest timeout-minutes: 20 steps: @@ -157,7 +157,7 @@ jobs: }); benchmark_windows: - name: Benchmark Windows + name: windows runs-on: windows-latest timeout-minutes: 20 steps: diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml index a2885c6e6..86ebf11cd 100644 --- a/.github/workflows/comparison-tests.yml +++ b/.github/workflows/comparison-tests.yml @@ -1,4 +1,4 @@ -name: Comparison Tests +name: comparison tests on: workflow_call: diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml index e2ad36b10..5ee626fe1 100644 --- a/.github/workflows/execution-tests.yml +++ b/.github/workflows/execution-tests.yml @@ -1,4 +1,4 @@ -name: Execution Tests +name: execution tests on: workflow_call: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ae88d1b6d..4f8d49d0f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,11 +1,11 @@ -name: Lint +name: lint on: workflow_call: jobs: lint: - name: Lint + name: lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 From 91c3ac3240f9093fad2f9461b5f9a0de929a936b Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 17:41:26 +0100 Subject: [PATCH 10/18] fix: reduce excess runs --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce44b4138..30f986770 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,14 @@ name: ci -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: comparison-tests: From 227485f36369e09ae794965ad2d17fd99651e192 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 18:18:40 +0100 Subject: [PATCH 11/18] chore: build and lint --- .github/workflows/build-and-lint.yml | 22 ++++++++++++++++++++++ .github/workflows/ci.yml | 7 ++++--- .github/workflows/lint.yml | 17 ----------------- 3 files changed, 26 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/build-and-lint.yml delete mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/build-and-lint.yml b/.github/workflows/build-and-lint.yml new file mode 100644 index 000000000..63ae351ce --- /dev/null +++ b/.github/workflows/build-and-lint.yml @@ -0,0 +1,22 @@ +name: lint + +on: + workflow_call: + +jobs: + build: + name: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader + + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/build-ts-loader + + - name: lint + run: yarn lint diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30f986770..bf7014cbe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,12 +12,13 @@ concurrency: jobs: comparison-tests: - name: Comparison tests + name: comparison tests uses: ./.github/workflows/comparison-tests.yml execution-tests: - name: Execution tests + name: execution tests uses: ./.github/workflows/execution-tests.yml lint: - uses: ./.github/workflows/lint.yml + name: build and lint + uses: ./.github/workflows/build-and-lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 4f8d49d0f..000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: lint - -on: - workflow_call: - -jobs: - lint: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: install - run: yarn install - - - name: lint - run: yarn lint From 188609c408f087a42293bace19a8ba08f72fc4b6 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 18:33:11 +0100 Subject: [PATCH 12/18] chore: chck --- .github/actions/build-ts-loader/action.yml | 22 +++++++++++-- .github/workflows/benchmark.yml | 38 +++++----------------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/.github/actions/build-ts-loader/action.yml b/.github/actions/build-ts-loader/action.yml index a874b7aba..5f676cde9 100644 --- a/.github/actions/build-ts-loader/action.yml +++ b/.github/actions/build-ts-loader/action.yml @@ -1,7 +1,7 @@ name: Build ts-loader description: > Install and build ts-loader (expects actions/checkout to have already run). - On Windows runners this excludes the working directory from Windows + On Windows runners this excludes the checkout and C:\source from Windows Defender's real-time scanning and copies the checkout to C:\source\ts-loader first - Defender's scanning of the heavy file I/O that yarn does during install/build/test is a major source of slowdown and @@ -14,10 +14,26 @@ inputs: runs: using: composite steps: - - name: exclude working directory from Windows Defender + # TEMPORARY: checks whether Defender real-time scanning is actually on + # for hosted Windows runners. If it isn't, the exclusions below (and + # possibly the copy to C:\source) are unnecessary. Remove once answered. + - name: report Windows Defender status if: runner.os == 'Windows' shell: pwsh - run: Add-MpPreference -ExclusionPath "C:\source" + run: | + try { + Get-MpComputerStatus | Format-List AMServiceEnabled, AntivirusEnabled, RealTimeProtectionEnabled, OnAccessProtectionEnabled + (Get-MpPreference).ExclusionPath + } catch { + Write-Output "Could not query Defender: $_" + } + + - name: exclude working directories from Windows Defender + if: runner.os == 'Windows' + shell: pwsh + run: | + Add-MpPreference -ExclusionPath "C:\source" + Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" - name: copy files if: runner.os == 'Windows' diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5d05dd36c..0319be0da 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,10 +30,7 @@ jobs: node-version: 24 cache: yarn - - name: install & build (PR branch) - run: | - yarn install - yarn build + - uses: ./.github/actions/build-ts-loader # Checked out into a sibling worktree rather than a second checkout step # so both builds can run from disk at the same time. @@ -100,10 +97,7 @@ jobs: node-version: 24 cache: yarn - - name: install & build (PR branch) - run: | - yarn install - yarn build + - uses: ./.github/actions/build-ts-loader # Checked out into a sibling worktree rather than a second checkout step # so both builds can run from disk at the same time. @@ -170,28 +164,12 @@ jobs: node-version: 24 cache: yarn - # Windows Defender's real-time scanning adds unpredictable latency to - # file I/O, and this benchmark does a lot of it (a 300-file fixture, - # webpack output per build, a touched file on every incremental - # iteration) - excluding the working directories keeps that noise out - # of the measurements. - - name: exclude working directories from Windows Defender - shell: pwsh - run: | - Add-MpPreference -ExclusionPath "C:\source" - Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" - - - name: copy files - shell: pwsh - run: | - New-Item C:\source\ts-loader -ItemType Directory - Copy-Item .\* C:\source\ts-loader -Recurse -Force - - - name: install & build (PR branch) - run: | - yarn install - yarn build - working-directory: C:\source\ts-loader + # Excludes the working directories from Windows Defender (this + # benchmark does a lot of file I/O, so scanning would add noise to the + # measurements) and copies the checkout to C:\source\ts-loader. + - uses: ./.github/actions/build-ts-loader + with: + working-directory: C:\source\ts-loader # Checked out into a sibling worktree rather than a second checkout step # so both builds can run from disk at the same time. From ea9e6a7eef902fd38898b586c5aa70b713b4d016 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 18:38:37 +0100 Subject: [PATCH 13/18] refactor: windows hardcodee --- .github/actions/build-ts-loader/action.yml | 39 ++++------------------ .github/workflows/benchmark.yml | 6 +--- .github/workflows/comparison-tests.yml | 2 -- .github/workflows/execution-tests.yml | 2 -- .github/workflows/test-probe.yml | 2 -- 5 files changed, 7 insertions(+), 44 deletions(-) diff --git a/.github/actions/build-ts-loader/action.yml b/.github/actions/build-ts-loader/action.yml index 5f676cde9..ea2b92aa5 100644 --- a/.github/actions/build-ts-loader/action.yml +++ b/.github/actions/build-ts-loader/action.yml @@ -1,40 +1,13 @@ name: Build ts-loader description: > Install and build ts-loader (expects actions/checkout to have already run). - On Windows runners this excludes the checkout and C:\source from Windows - Defender's real-time scanning and copies the checkout to - C:\source\ts-loader first - Defender's scanning of the heavy file I/O that - yarn does during install/build/test is a major source of slowdown and - flakiness on hosted Windows runners. -inputs: - working-directory: - description: 'Directory to run install/build in. Use C:\source\ts-loader on Windows runners.' - required: false - default: '.' + On Windows runners this first copies the checkout to C:\source\ts-loader + and installs/builds there, giving a consistent path that the comparison + tests' output normalisation expects - later Windows steps should use + working-directory: C:\source\ts-loader. runs: using: composite steps: - # TEMPORARY: checks whether Defender real-time scanning is actually on - # for hosted Windows runners. If it isn't, the exclusions below (and - # possibly the copy to C:\source) are unnecessary. Remove once answered. - - name: report Windows Defender status - if: runner.os == 'Windows' - shell: pwsh - run: | - try { - Get-MpComputerStatus | Format-List AMServiceEnabled, AntivirusEnabled, RealTimeProtectionEnabled, OnAccessProtectionEnabled - (Get-MpPreference).ExclusionPath - } catch { - Write-Output "Could not query Defender: $_" - } - - - name: exclude working directories from Windows Defender - if: runner.os == 'Windows' - shell: pwsh - run: | - Add-MpPreference -ExclusionPath "C:\source" - Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" - - name: copy files if: runner.os == 'Windows' shell: pwsh @@ -44,10 +17,10 @@ runs: - name: install shell: bash - working-directory: ${{ inputs.working-directory }} + working-directory: ${{ runner.os == 'Windows' && 'C:\source\ts-loader' || '.' }} run: yarn install - name: build shell: bash - working-directory: ${{ inputs.working-directory }} + working-directory: ${{ runner.os == 'Windows' && 'C:\source\ts-loader' || '.' }} run: yarn build diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0319be0da..cfe6da613 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -164,12 +164,8 @@ jobs: node-version: 24 cache: yarn - # Excludes the working directories from Windows Defender (this - # benchmark does a lot of file I/O, so scanning would add noise to the - # measurements) and copies the checkout to C:\source\ts-loader. + # Copies the checkout to C:\source\ts-loader before building. - uses: ./.github/actions/build-ts-loader - with: - working-directory: C:\source\ts-loader # Checked out into a sibling worktree rather than a second checkout step # so both builds can run from disk at the same time. diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml index 86ebf11cd..3ecc84256 100644 --- a/.github/workflows/comparison-tests.yml +++ b/.github/workflows/comparison-tests.yml @@ -33,8 +33,6 @@ jobs: steps: - uses: actions/checkout@v7 - uses: ./.github/actions/build-ts-loader - with: - working-directory: C:\source\ts-loader - name: test run: yarn comparison-tests diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml index 5ee626fe1..3f399e469 100644 --- a/.github/workflows/execution-tests.yml +++ b/.github/workflows/execution-tests.yml @@ -77,8 +77,6 @@ jobs: node-version: ${{ matrix.node }} - uses: ./.github/actions/build-ts-loader - with: - working-directory: C:\source\ts-loader - name: install typescript run: yarn add typescript@${{ matrix.ts }} diff --git a/.github/workflows/test-probe.yml b/.github/workflows/test-probe.yml index 490f6cc18..8f3347cf2 100644 --- a/.github/workflows/test-probe.yml +++ b/.github/workflows/test-probe.yml @@ -83,8 +83,6 @@ jobs: steps: - uses: actions/checkout@v7 - uses: ./.github/actions/build-ts-loader - with: - working-directory: C:\source\ts-loader - name: test shell: bash From 1f6f510751b3b29e9576d426cd1868aa2e225c06 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sat, 3 Oct 2026 18:53:01 +0100 Subject: [PATCH 14/18] chore: fiddling --- .github/workflows/build-and-lint.yml | 12 ++++++++++++ .github/workflows/comparison-tests.yml | 18 ++++++++++++++++++ .github/workflows/release.yml | 6 +----- .github/workflows/test-probe.yml | 18 ++++++++++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-lint.yml b/.github/workflows/build-and-lint.yml index 63ae351ce..9c829ac11 100644 --- a/.github/workflows/build-and-lint.yml +++ b/.github/workflows/build-and-lint.yml @@ -9,6 +9,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + - uses: ./.github/actions/build-ts-loader lint: @@ -16,6 +22,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + - uses: ./.github/actions/build-ts-loader - name: lint diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml index 3ecc84256..2c5e6f595 100644 --- a/.github/workflows/comparison-tests.yml +++ b/.github/workflows/comparison-tests.yml @@ -10,6 +10,12 @@ jobs: timeout-minutes: 25 steps: - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + - uses: ./.github/actions/build-ts-loader - name: test @@ -21,6 +27,12 @@ jobs: timeout-minutes: 25 steps: - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + - uses: ./.github/actions/build-ts-loader - name: test @@ -32,6 +44,12 @@ jobs: timeout-minutes: 25 steps: - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + - uses: ./.github/actions/build-ts-loader - name: test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0fee4f4a3..722ec989d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,11 +21,7 @@ jobs: node-version: 24 registry-url: https://registry.npmjs.org/ - - name: install - run: yarn install - - - name: build - run: yarn build + - uses: ./.github/actions/build-ts-loader - run: npm install -g npm@latest diff --git a/.github/workflows/test-probe.yml b/.github/workflows/test-probe.yml index 8f3347cf2..93e94bca8 100644 --- a/.github/workflows/test-probe.yml +++ b/.github/workflows/test-probe.yml @@ -36,6 +36,12 @@ jobs: timeout-minutes: 25 steps: - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + - uses: ./.github/actions/build-ts-loader - name: test @@ -59,6 +65,12 @@ jobs: timeout-minutes: 25 steps: - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + - uses: ./.github/actions/build-ts-loader - name: test @@ -82,6 +94,12 @@ jobs: timeout-minutes: 25 steps: - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + - uses: ./.github/actions/build-ts-loader - name: test From d5a6439d45fcf49e804ba5fc24a4bd9df71a4897 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sun, 4 Oct 2026 09:14:26 +0100 Subject: [PATCH 15/18] ci: cache yarn in execution tests, restrict CI token, tidy names and docs - add cache: yarn to execution-tests setup-node steps - give ci.yml's token contents: read only - rename build-and-lint workflow to "build and lint" and strip trailing whitespace - prefix benchmark job names and use "macOS" casing consistently - update AGENTS.md: ci.yml replaces push.yml, C:\source copy is for path consistency (not Defender), list all workflows using the build action Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark.yml | 6 +++--- .github/workflows/build-and-lint.yml | 4 ++-- .github/workflows/ci.yml | 3 +++ .github/workflows/execution-tests.yml | 3 +++ AGENTS.md | 4 ++-- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cfe6da613..a381bf2d6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,7 +17,7 @@ permissions: jobs: benchmark_ubuntu: - name: ubuntu + name: benchmark ubuntu runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -84,7 +84,7 @@ jobs: }); benchmark_macos: - name: macos + name: benchmark macOS runs-on: macos-latest timeout-minutes: 20 steps: @@ -151,7 +151,7 @@ jobs: }); benchmark_windows: - name: windows + name: benchmark windows runs-on: windows-latest timeout-minutes: 20 steps: diff --git a/.github/workflows/build-and-lint.yml b/.github/workflows/build-and-lint.yml index 9c829ac11..94dc1d383 100644 --- a/.github/workflows/build-and-lint.yml +++ b/.github/workflows/build-and-lint.yml @@ -1,4 +1,4 @@ -name: lint +name: build and lint on: workflow_call: @@ -16,7 +16,7 @@ jobs: cache: yarn - uses: ./.github/actions/build-ts-loader - + lint: name: lint runs-on: ubuntu-latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf7014cbe..18d9cfb2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,9 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml index 3f399e469..57083bf00 100644 --- a/.github/workflows/execution-tests.yml +++ b/.github/workflows/execution-tests.yml @@ -19,6 +19,7 @@ jobs: uses: actions/setup-node@v7 with: node-version: ${{ matrix.node }} + cache: yarn - uses: ./.github/actions/build-ts-loader @@ -47,6 +48,7 @@ jobs: # uses: actions/setup-node@v7 # with: # node-version: ${{ matrix.node }} + # cache: yarn # - uses: ./.github/actions/build-ts-loader @@ -75,6 +77,7 @@ jobs: uses: actions/setup-node@v7 with: node-version: ${{ matrix.node }} + cache: yarn - uses: ./.github/actions/build-ts-loader diff --git a/AGENTS.md b/AGENTS.md index 7fcd07a35..c313953bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,9 +46,9 @@ yarn comparison-tests -- --save-output --single-test # regenerate o ### Test probe workflow -`.github/workflows/test-probe.yml` (registered on `main`, so dispatchable against any branch/ref) runs just the comparison tests via `workflow_dispatch` on Ubuntu, macOS, and/or Windows — much faster than the full `push.yml` matrix (which also runs the full Node/TS/webpack execution-test matrix). Use it to iterate on a failing test, or an OS-specific failure, without asking a human to relay CI output. +`.github/workflows/test-probe.yml` (registered on `main`, so dispatchable against any branch/ref) runs just the comparison tests via `workflow_dispatch` on Ubuntu, macOS, and/or Windows — much faster than the full CI run from `ci.yml` (which also runs the full Node/TS/webpack execution-test matrix). Use it to iterate on a failing test, or an OS-specific failure, without asking a human to relay CI output. -Three separate jobs (`probe_ubuntu`/`probe_macos`/`probe_windows`), each gated by the `os` input — defaults to `all`; pass `ubuntu-latest`, `macos-latest`, or `windows-latest` to run just one. All three share their install/build steps (including the Windows Defender-exclusion + copy-to-`C:\source\ts-loader` workaround) via the composite action `.github/actions/build-ts-loader/action.yml`, which is also used by `comparison-tests.yml` and `execution-tests.yml` — edit that action, not each workflow, if the build steps themselves need to change. +Three separate jobs (`probe_ubuntu`/`probe_macos`/`probe_windows`), each gated by the `os` input — defaults to `all`; pass `ubuntu-latest`, `macos-latest`, or `windows-latest` to run just one. All three share their install/build steps (including, on Windows, copying the checkout to `C:\source\ts-loader` so the comparison tests' path normalisation sees a consistent location) via the composite action `.github/actions/build-ts-loader/action.yml`, which every workflow that builds ts-loader uses (`comparison-tests.yml`, `execution-tests.yml`, `build-and-lint.yml`, `benchmark.yml`, `release.yml`) — edit that action, not each workflow, if the build steps themselves need to change. Requires `gh` CLI authenticated with the `workflow` scope (`gh auth login`, then `gh auth refresh -s workflow` if `gh auth status` doesn't already list `workflow` — both scopes need a human to complete the browser device-flow prompt, they can't be scripted). From adcaa4d9f74c3ce08f17f417658f33aca7b05923 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sun, 4 Oct 2026 09:19:57 +0100 Subject: [PATCH 16/18] ci: replace build-and-lint with a lint-only workflow The standalone build job duplicated work every other CI job already does via the build-ts-loader composite action, so drop it and keep just lint (which builds before linting). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 ++-- .../workflows/{build-and-lint.yml => lint.yml} | 15 +-------------- AGENTS.md | 2 +- 3 files changed, 4 insertions(+), 17 deletions(-) rename .github/workflows/{build-and-lint.yml => lint.yml} (54%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18d9cfb2e..f1e77284e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,5 +23,5 @@ jobs: uses: ./.github/workflows/execution-tests.yml lint: - name: build and lint - uses: ./.github/workflows/build-and-lint.yml + name: lint + uses: ./.github/workflows/lint.yml diff --git a/.github/workflows/build-and-lint.yml b/.github/workflows/lint.yml similarity index 54% rename from .github/workflows/build-and-lint.yml rename to .github/workflows/lint.yml index 94dc1d383..c8c1359db 100644 --- a/.github/workflows/build-and-lint.yml +++ b/.github/workflows/lint.yml @@ -1,22 +1,9 @@ -name: build and lint +name: lint on: workflow_call: jobs: - build: - name: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-node@v7 - with: - node-version: 24 - cache: yarn - - - uses: ./.github/actions/build-ts-loader - lint: name: lint runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index c313953bf..97eb0c3ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,7 +48,7 @@ yarn comparison-tests -- --save-output --single-test # regenerate o `.github/workflows/test-probe.yml` (registered on `main`, so dispatchable against any branch/ref) runs just the comparison tests via `workflow_dispatch` on Ubuntu, macOS, and/or Windows — much faster than the full CI run from `ci.yml` (which also runs the full Node/TS/webpack execution-test matrix). Use it to iterate on a failing test, or an OS-specific failure, without asking a human to relay CI output. -Three separate jobs (`probe_ubuntu`/`probe_macos`/`probe_windows`), each gated by the `os` input — defaults to `all`; pass `ubuntu-latest`, `macos-latest`, or `windows-latest` to run just one. All three share their install/build steps (including, on Windows, copying the checkout to `C:\source\ts-loader` so the comparison tests' path normalisation sees a consistent location) via the composite action `.github/actions/build-ts-loader/action.yml`, which every workflow that builds ts-loader uses (`comparison-tests.yml`, `execution-tests.yml`, `build-and-lint.yml`, `benchmark.yml`, `release.yml`) — edit that action, not each workflow, if the build steps themselves need to change. +Three separate jobs (`probe_ubuntu`/`probe_macos`/`probe_windows`), each gated by the `os` input — defaults to `all`; pass `ubuntu-latest`, `macos-latest`, or `windows-latest` to run just one. All three share their install/build steps (including, on Windows, copying the checkout to `C:\source\ts-loader` so the comparison tests' path normalisation sees a consistent location) via the composite action `.github/actions/build-ts-loader/action.yml`, which every workflow that builds ts-loader uses (`comparison-tests.yml`, `execution-tests.yml`, `lint.yml`, `benchmark.yml`, `release.yml`) — edit that action, not each workflow, if the build steps themselves need to change. Requires `gh` CLI authenticated with the `workflow` scope (`gh auth login`, then `gh auth refresh -s workflow` if `gh auth status` doesn't already list `workflow` — both scopes need a human to complete the browser device-flow prompt, they can't be scripted). From 5e1f27e2fb4b34d3bbce6b19bd0ced418d319af5 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sun, 4 Oct 2026 15:05:26 +0100 Subject: [PATCH 17/18] ci: collapse benchmark jobs into a single OS matrix The Ubuntu, macOS and Windows jobs were the same steps three times over and had started to drift. A matrix with a per-OS ROOT (the workspace, or C:/source/ts-loader on Windows where build-ts-loader builds) lets every step run in bash on all three OSes. Check names, artifact names and PR comment markers are unchanged so existing comments still update in place. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/post-benchmark-comment.js | 2 +- .github/workflows/benchmark.yml | 196 +++++----------------- 2 files changed, 43 insertions(+), 155 deletions(-) diff --git a/.github/scripts/post-benchmark-comment.js b/.github/scripts/post-benchmark-comment.js index c8f8d72e9..a8c19814e 100644 --- a/.github/scripts/post-benchmark-comment.js +++ b/.github/scripts/post-benchmark-comment.js @@ -2,7 +2,7 @@ const fs = require('fs'); /** * Creates or updates the benchmark result comment identified by `marker`. - * Shared by the "comment on PR" steps in benchmark.yml (Ubuntu + Windows jobs). + * Used by the "comment on PR" step of each OS in benchmark.yml's matrix. * * @param {{ * github: { rest: { issues: { diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a381bf2d6..9804da2f8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -16,10 +16,34 @@ permissions: pull-requests: write jobs: - benchmark_ubuntu: - name: benchmark ubuntu - runs-on: ubuntu-latest + benchmark: + name: benchmark ${{ matrix.name }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + id: ubuntu + name: ubuntu + title: Ubuntu + - os: macos-latest + id: macos + name: macOS + title: macOS + - os: windows-latest + id: windows + name: windows + title: Windows + runs-on: ${{ matrix.os }} timeout-minutes: 20 + defaults: + run: + shell: bash + env: + # Where the build-ts-loader action builds the PR branch - on Windows it + # copies the checkout to C:\source\ts-loader first. The base branch is + # checked out alongside it at "$ROOT-main". + ROOT: ${{ matrix.os == 'windows-latest' && 'C:/source/ts-loader' || github.workspace }} steps: - uses: actions/checkout@v7 with: @@ -35,34 +59,36 @@ jobs: # Checked out into a sibling worktree rather than a second checkout step # so both builds can run from disk at the same time. - name: checkout base branch into a worktree + working-directory: ${{ env.ROOT }} env: BASE_SHA: ${{ github.event.pull_request.base.sha || 'origin/main' }} - run: git worktree add ../ts-loader-main "$BASE_SHA" + run: git worktree add "$ROOT-main" "$BASE_SHA" - name: install & build (base branch) - working-directory: ../ts-loader-main + working-directory: ${{ env.ROOT }}-main run: | yarn install yarn build - # --root-a is the PR branch checked out at $GITHUB_WORKSPACE above; + # --root-a is the PR branch built by the build-ts-loader action; # --root-b is the base branch worktree built in the previous step. - name: run benchmark + working-directory: ${{ env.ROOT }} run: | yarn benchmark -- \ - --root-a "$GITHUB_WORKSPACE" --label-a "PR branch" \ - --root-b "$GITHUB_WORKSPACE/../ts-loader-main" --label-b "base branch" + --root-a "$ROOT" --label-a "PR branch" \ + --root-b "$ROOT-main" --label-b "base branch" - name: write job summary if: always() - run: cat .benchmark/benchmark-results.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + run: cat "$ROOT/.benchmark/benchmark-results.md" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true - name: upload results if: always() uses: actions/upload-artifact@v7 with: - name: benchmark-results-ubuntu - path: .benchmark/benchmark-results.* + name: benchmark-results-${{ matrix.id }} + path: ${{ env.ROOT }}/.benchmark/benchmark-results.* if-no-files-found: ignore # Only runs with a writable token - GitHub automatically issues a @@ -71,147 +97,9 @@ jobs: - name: comment on PR if: always() && github.event_name == 'pull_request' uses: actions/github-script@v7 - with: - script: | - const path = require('path'); - const postComment = require(path.join(process.env.GITHUB_WORKSPACE, '.github/scripts/post-benchmark-comment.js')); - await postComment({ - github, - context, - resultsPath: '.benchmark/benchmark-results.md', - marker: '', - title: 'Benchmark (Ubuntu)', - }); - - benchmark_macos: - name: benchmark macOS - runs-on: macos-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v7 - with: - node-version: 24 - cache: yarn - - - uses: ./.github/actions/build-ts-loader - - # Checked out into a sibling worktree rather than a second checkout step - # so both builds can run from disk at the same time. - - name: checkout base branch into a worktree - env: - BASE_SHA: ${{ github.event.pull_request.base.sha || 'origin/main' }} - run: git worktree add ../ts-loader-main "$BASE_SHA" - - - name: install & build (base branch) - working-directory: ../ts-loader-main - run: | - yarn install - yarn build - - # --root-a is the PR branch checked out at $GITHUB_WORKSPACE above; - # --root-b is the base branch worktree built in the previous step. - - name: run benchmark - run: | - yarn benchmark -- \ - --root-a "$GITHUB_WORKSPACE" --label-a "PR branch" \ - --root-b "$GITHUB_WORKSPACE/../ts-loader-main" --label-b "base branch" - - - name: write job summary - if: always() - run: cat .benchmark/benchmark-results.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true - - - name: upload results - if: always() - uses: actions/upload-artifact@v7 - with: - name: benchmark-results-macos - path: .benchmark/benchmark-results.* - if-no-files-found: ignore - - # Only runs with a writable token - GitHub automatically issues a - # read-only GITHUB_TOKEN for fork PRs, so this step is a silent no-op - # there. The step summary above still carries the numbers for everyone. - - name: comment on PR - if: always() && github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const path = require('path'); - const postComment = require(path.join(process.env.GITHUB_WORKSPACE, '.github/scripts/post-benchmark-comment.js')); - await postComment({ - github, - context, - resultsPath: '.benchmark/benchmark-results.md', - marker: '', - title: 'Benchmark (macOS)', - }); - - benchmark_windows: - name: benchmark windows - runs-on: windows-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v7 - with: - node-version: 24 - cache: yarn - - # Copies the checkout to C:\source\ts-loader before building. - - uses: ./.github/actions/build-ts-loader - - # Checked out into a sibling worktree rather than a second checkout step - # so both builds can run from disk at the same time. - - name: checkout base branch into a worktree - shell: pwsh env: - BASE_SHA: ${{ github.event.pull_request.base.sha || 'origin/main' }} - run: git worktree add C:\source\ts-loader-main $env:BASE_SHA - working-directory: C:\source\ts-loader - - - name: install & build (base branch) - run: | - yarn install - yarn build - working-directory: C:\source\ts-loader-main - - # --root-a is the PR branch copied to C:\source\ts-loader above; - # --root-b is the base branch worktree built in the previous step. - - name: run benchmark - shell: pwsh - run: | - yarn benchmark -- ` - --root-a C:\source\ts-loader --label-a "PR branch" ` - --root-b C:\source\ts-loader-main --label-b "base branch" - working-directory: C:\source\ts-loader - - - name: write job summary - if: always() - shell: pwsh - run: | - if (Test-Path .benchmark\benchmark-results.md) { - Get-Content .benchmark\benchmark-results.md | Out-File -Append -Encoding utf8 $env:GITHUB_STEP_SUMMARY - } - working-directory: C:\source\ts-loader - - - name: upload results - if: always() - uses: actions/upload-artifact@v7 - with: - name: benchmark-results-windows - path: C:\source\ts-loader\.benchmark\benchmark-results.* - if-no-files-found: ignore - - - name: comment on PR - if: always() && github.event_name == 'pull_request' - uses: actions/github-script@v7 + MARKER: + TITLE: Benchmark (${{ matrix.title }}) with: script: | const path = require('path'); @@ -219,7 +107,7 @@ jobs: await postComment({ github, context, - resultsPath: 'C:\\source\\ts-loader\\.benchmark\\benchmark-results.md', - marker: '', - title: 'Benchmark (Windows)', + resultsPath: path.join(process.env.ROOT, '.benchmark/benchmark-results.md'), + marker: process.env.MARKER, + title: process.env.TITLE, }); From dbe4534700937b266bdf887175150d52596afca8 Mon Sep 17 00:00:00 2001 From: johnnyreilly Date: Sun, 4 Oct 2026 15:13:01 +0100 Subject: [PATCH 18/18] test: raise benchmark scenario time budget to 90s and update docs Cold typeCheck builds were regularly hitting the 60s per-scenario budget (e.g. stopping after 44/62 iterations). 90s lets them complete all iterations while still bounding pathologically slow scenarios. Also document the budget in the benchmark README, replace its stale Windows Defender paragraph with how CI runs the benchmark now, and fix AGENTS.md describing the two sides as running back-to-back in one process. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- test/benchmark-tests/README.md | 4 +++- test/benchmark-tests/run-side.mts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 97eb0c3ad..d2019564f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ yarn execution-tests -- --single-test --watch # watch mode (open http: ## Benchmark tests (`test/benchmark-tests/`) -Answers "did this get faster or slower?", not "is the output correct?" - the comparison/execution packs never record timing, so this is a separate harness. It generates a synthetic project on the fly and times ts-loader compiling it (cold build, and incremental rebuild after touching a low-fan-out vs. high-fan-out file, under both `transpileOnly: true`/`false`), comparing two ts-loader checkouts (this build vs. another, e.g. `main`) back-to-back in one process so the relative numbers are meaningful despite noisy CI hosts. +Answers "did this get faster or slower?", not "is the output correct?" - the comparison/execution packs never record timing, so this is a separate harness. It generates a synthetic project on the fly and times ts-loader compiling it (cold build, and incremental rebuild after touching a low-fan-out vs. high-fan-out file, under both `transpileOnly: true`/`false`), comparing two ts-loader checkouts (this build vs. another, e.g. `main`), each in its own process but run concurrently so host noise hits both sides equally and the relative numbers stay meaningful on noisy CI hosts. Full docs: [`test/benchmark-tests/README.md`](test/benchmark-tests/README.md) diff --git a/test/benchmark-tests/README.md b/test/benchmark-tests/README.md index f9665d25e..553aee4a5 100644 --- a/test/benchmark-tests/README.md +++ b/test/benchmark-tests/README.md @@ -20,6 +20,8 @@ Their fixtures are tiny (a handful of files, 1-2 watch patches) and built for ou For each scenario, both sides' `run-side.mts` subprocesses are spawned and run **concurrently** (not one after the other) - this is what keeps the delta near zero when the two checkouts are identical, since any host noise (CPU contention, scheduling) hits both sides at the same wall-clock moment instead of biasing whichever side happened to run in a noisier window. Cold builds run 2 discarded warmup iterations plus several times the requested `--iterations` (see `ITERATION_MULTIPLIER` in `run-benchmark.mts`); incremental rebuilds get a much larger multiplier again, since each rebuild is only tens of ms and cheap to re-run many times to average out that much noisier measurement. The **median** of the measured iterations is reported (robust to the odd stalled one). + Each side also has a 90s wall-clock budget per scenario (`SCENARIO_TIME_BUDGET_MS` in `run-side.mts`): once it's past that budget *and* has its warmup plus at least 2 measured iterations, it stops early and logs `stopped early after / iterations`. That bounds scenarios that are far more expensive per iteration than expected (e.g. a compiler that spawns a native process per instance) so they can't blow the CI job's timeout. Seeing it occasionally for cold typeCheck builds is normal - the median of the iterations that did run is still reported; it's only worth a closer look if a scenario stops after just a handful of iterations, or the two sides stop at very different counts. + ## Running it ```bash @@ -38,7 +40,7 @@ Other flags: `--files ` (default 300, the fixture's module count), `--warmup Results are written to `.benchmark/benchmark-results.json` (raw samples) and `.benchmark/benchmark-results.md` (summary table), and also printed to stdout. -On Windows CI specifically, `.github/workflows/benchmark.yml` also excludes the working directories from Windows Defender's real-time scanning before running the benchmark, since that scanning otherwise adds unpredictable latency to this benchmark's heavy file I/O. +In CI, `.github/workflows/benchmark.yml` runs this on Ubuntu, macOS and Windows (one matrix job per OS), building the PR branch with the shared `.github/actions/build-ts-loader` action and the base branch in a sibling git worktree. On Windows that action copies the checkout to `C:\source\ts-loader`, so the benchmark runs from there (base branch at `C:\source\ts-loader-main`). ## Interpreting deltas diff --git a/test/benchmark-tests/run-side.mts b/test/benchmark-tests/run-side.mts index a805856a8..6dca588ae 100644 --- a/test/benchmark-tests/run-side.mts +++ b/test/benchmark-tests/run-side.mts @@ -35,7 +35,7 @@ const REBUILD_TIMEOUT_MS = 30_000; // timeout long before every scenario finishes. Capping wall-clock instead of // guessing a smaller fixed iteration count keeps full statistical power for // scenarios that stay cheap while still bounding the expensive ones. -const SCENARIO_TIME_BUDGET_MS = 60_000; +const SCENARIO_TIME_BUDGET_MS = 90_000; const MIN_MEASURED_ITERATIONS = 2; function get(flag: string): string {