diff --git a/.github/actions/build-ts-loader/action.yml b/.github/actions/build-ts-loader/action.yml new file mode 100644 index 000000000..ea2b92aa5 --- /dev/null +++ b/.github/actions/build-ts-loader/action.yml @@ -0,0 +1,26 @@ +name: Build ts-loader +description: > + Install and build ts-loader (expects actions/checkout to have already run). + On Windows runners this first copies the checkout to C:\source\ts-loader + and installs/builds there, giving a consistent path that the comparison + tests' output normalisation expects - later Windows steps should use + working-directory: C:\source\ts-loader. +runs: + using: composite + steps: + - name: copy files + if: runner.os == 'Windows' + shell: pwsh + run: | + New-Item C:\source\ts-loader -ItemType Directory + Copy-Item .\* C:\source\ts-loader -Recurse -Force + + - name: install + shell: bash + working-directory: ${{ runner.os == 'Windows' && 'C:\source\ts-loader' || '.' }} + run: yarn install + + - name: build + shell: bash + working-directory: ${{ runner.os == 'Windows' && 'C:\source\ts-loader' || '.' }} + run: yarn build diff --git a/.github/scripts/post-benchmark-comment.js b/.github/scripts/post-benchmark-comment.js index c8f8d72e9..a8c19814e 100644 --- a/.github/scripts/post-benchmark-comment.js +++ b/.github/scripts/post-benchmark-comment.js @@ -2,7 +2,7 @@ const fs = require('fs'); /** * Creates or updates the benchmark result comment identified by `marker`. - * Shared by the "comment on PR" steps in benchmark.yml (Ubuntu + Windows jobs). + * Used by the "comment on PR" step of each OS in benchmark.yml's matrix. * * @param {{ * github: { rest: { issues: { diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e1cd1578c..9804da2f8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -16,10 +16,34 @@ permissions: pull-requests: write jobs: - benchmark_ubuntu: - name: Benchmark Ubuntu - runs-on: ubuntu-latest + benchmark: + name: benchmark ${{ matrix.name }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + id: ubuntu + name: ubuntu + title: Ubuntu + - os: macos-latest + id: macos + name: macOS + title: macOS + - os: windows-latest + id: windows + name: windows + title: Windows + runs-on: ${{ matrix.os }} timeout-minutes: 20 + defaults: + run: + shell: bash + env: + # Where the build-ts-loader action builds the PR branch - on Windows it + # copies the checkout to C:\source\ts-loader first. The base branch is + # checked out alongside it at "$ROOT-main". + ROOT: ${{ matrix.os == 'windows-latest' && 'C:/source/ts-loader' || github.workspace }} steps: - uses: actions/checkout@v7 with: @@ -30,42 +54,41 @@ jobs: node-version: 24 cache: yarn - - name: install & build (PR branch) - run: | - yarn install - yarn build + - uses: ./.github/actions/build-ts-loader # Checked out into a sibling worktree rather than a second checkout step # so both builds can run from disk at the same time. - name: checkout base branch into a worktree + working-directory: ${{ env.ROOT }} env: BASE_SHA: ${{ github.event.pull_request.base.sha || 'origin/main' }} - run: git worktree add ../ts-loader-main "$BASE_SHA" + run: git worktree add "$ROOT-main" "$BASE_SHA" - name: install & build (base branch) - working-directory: ../ts-loader-main + working-directory: ${{ env.ROOT }}-main run: | yarn install yarn build - # --root-a is the PR branch checked out at $GITHUB_WORKSPACE above; + # --root-a is the PR branch built by the build-ts-loader action; # --root-b is the base branch worktree built in the previous step. - name: run benchmark + working-directory: ${{ env.ROOT }} run: | yarn benchmark -- \ - --root-a "$GITHUB_WORKSPACE" --label-a "PR branch" \ - --root-b "$GITHUB_WORKSPACE/../ts-loader-main" --label-b "base branch" + --root-a "$ROOT" --label-a "PR branch" \ + --root-b "$ROOT-main" --label-b "base branch" - name: write job summary if: always() - run: cat .benchmark/benchmark-results.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + run: cat "$ROOT/.benchmark/benchmark-results.md" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true - name: upload results if: always() uses: actions/upload-artifact@v7 with: - name: benchmark-results-ubuntu - path: .benchmark/benchmark-results.* + name: benchmark-results-${{ matrix.id }} + path: ${{ env.ROOT }}/.benchmark/benchmark-results.* if-no-files-found: ignore # Only runs with a writable token - GitHub automatically issues a @@ -74,100 +97,9 @@ jobs: - name: comment on PR if: always() && github.event_name == 'pull_request' uses: actions/github-script@v7 - with: - script: | - const path = require('path'); - const postComment = require(path.join(process.env.GITHUB_WORKSPACE, '.github/scripts/post-benchmark-comment.js')); - await postComment({ - github, - context, - resultsPath: '.benchmark/benchmark-results.md', - marker: '', - title: 'Benchmark (Ubuntu)', - }); - - benchmark_windows: - name: Benchmark Windows - runs-on: windows-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - uses: actions/setup-node@v7 - with: - node-version: 24 - cache: yarn - - # Windows Defender's real-time scanning adds unpredictable latency to - # file I/O, and this benchmark does a lot of it (a 300-file fixture, - # webpack output per build, a touched file on every incremental - # iteration) - excluding the working directories keeps that noise out - # of the measurements. - - name: exclude working directories from Windows Defender - shell: pwsh - run: | - Add-MpPreference -ExclusionPath "C:\source" - Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" - - - name: copy files - shell: pwsh - run: | - New-Item C:\source\ts-loader -ItemType Directory - Copy-Item .\* C:\source\ts-loader -Recurse -Force - - - name: install & build (PR branch) - run: | - yarn install - yarn build - working-directory: C:\source\ts-loader - - # Checked out into a sibling worktree rather than a second checkout step - # so both builds can run from disk at the same time. - - name: checkout base branch into a worktree - shell: pwsh env: - BASE_SHA: ${{ github.event.pull_request.base.sha || 'origin/main' }} - run: git worktree add C:\source\ts-loader-main $env:BASE_SHA - working-directory: C:\source\ts-loader - - - name: install & build (base branch) - run: | - yarn install - yarn build - working-directory: C:\source\ts-loader-main - - # --root-a is the PR branch copied to C:\source\ts-loader above; - # --root-b is the base branch worktree built in the previous step. - - name: run benchmark - shell: pwsh - run: | - yarn benchmark -- ` - --root-a C:\source\ts-loader --label-a "PR branch" ` - --root-b C:\source\ts-loader-main --label-b "base branch" - working-directory: C:\source\ts-loader - - - name: write job summary - if: always() - shell: pwsh - run: | - if (Test-Path .benchmark\benchmark-results.md) { - Get-Content .benchmark\benchmark-results.md | Out-File -Append -Encoding utf8 $env:GITHUB_STEP_SUMMARY - } - working-directory: C:\source\ts-loader - - - name: upload results - if: always() - uses: actions/upload-artifact@v7 - with: - name: benchmark-results-windows - path: C:\source\ts-loader\.benchmark\benchmark-results.* - if-no-files-found: ignore - - - name: comment on PR - if: always() && github.event_name == 'pull_request' - uses: actions/github-script@v7 + MARKER: + TITLE: Benchmark (${{ matrix.title }}) with: script: | const path = require('path'); @@ -175,7 +107,7 @@ jobs: await postComment({ github, context, - resultsPath: 'C:\\source\\ts-loader\\.benchmark\\benchmark-results.md', - marker: '', - title: 'Benchmark (Windows)', + resultsPath: path.join(process.env.ROOT, '.benchmark/benchmark-results.md'), + marker: process.env.MARKER, + title: process.env.TITLE, }); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..f1e77284e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + comparison-tests: + name: comparison tests + uses: ./.github/workflows/comparison-tests.yml + + execution-tests: + name: execution tests + uses: ./.github/workflows/execution-tests.yml + + lint: + name: lint + uses: ./.github/workflows/lint.yml diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml new file mode 100644 index 000000000..2c5e6f595 --- /dev/null +++ b/.github/workflows/comparison-tests.yml @@ -0,0 +1,57 @@ +name: comparison tests + +on: + workflow_call: + +jobs: + comparison_test_ubuntu: + name: ubuntu + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: test + run: yarn comparison-tests + + comparison_test_macos: + name: macOS + runs-on: macos-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: test + run: yarn comparison-tests + + comparison_test_windows: + name: windows + runs-on: windows-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: test + run: yarn comparison-tests + working-directory: C:\source\ts-loader diff --git a/.github/workflows/execution-tests.yml b/.github/workflows/execution-tests.yml new file mode 100644 index 000000000..57083bf00 --- /dev/null +++ b/.github/workflows/execution-tests.yml @@ -0,0 +1,95 @@ +name: execution tests + +on: + workflow_call: + +jobs: + execution_test_ubuntu: + name: ubuntu + strategy: + matrix: + node: [26, 24, 22] + ts: [5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + webpack: [4, 5] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: install node + uses: actions/setup-node@v7 + with: + node-version: ${{ matrix.node }} + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: install typescript + run: yarn add typescript@${{ matrix.ts }} + + - name: install webpack 4 + if: matrix.webpack == 4 + run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 + + - name: test + run: yarn execution-tests + + # execution_test_macos: + # name: macOS + # strategy: + # matrix: + # node: [26, 24, 22] + # ts: [5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + # webpack: [4, 5] + # runs-on: macos-latest + # steps: + # - uses: actions/checkout@v7 + + # - name: install node + # uses: actions/setup-node@v7 + # with: + # node-version: ${{ matrix.node }} + # cache: yarn + + # - uses: ./.github/actions/build-ts-loader + + # - name: install typescript + # run: yarn add typescript@${{ matrix.ts }} + + # - name: install webpack 4 + # if: matrix.webpack == 4 + # run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 + + # - name: test + # run: yarn execution-tests + + execution_test_windows: + name: windows + strategy: + matrix: + node: [26, 24, 22] + ts: [5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now + webpack: [4, 5] + runs-on: windows-latest + steps: + - uses: actions/checkout@v7 + + - name: install node + uses: actions/setup-node@v7 + with: + node-version: ${{ matrix.node }} + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: install typescript + run: yarn add typescript@${{ matrix.ts }} + working-directory: C:\source\ts-loader + + - name: install webpack 4 + if: matrix.webpack == 4 + run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 + working-directory: C:\source\ts-loader + + - name: test + run: yarn execution-tests + working-directory: C:\source\ts-loader diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..c8c1359db --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,21 @@ +name: lint + +on: + workflow_call: + +jobs: + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: lint + run: yarn lint diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml deleted file mode 100644 index c13c85822..000000000 --- a/.github/workflows/push.yml +++ /dev/null @@ -1,136 +0,0 @@ -name: build and test - -on: [push, pull_request, workflow_dispatch] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - comparison_test_ubuntu: - name: Comparison Tests Ubuntu - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - - - name: install - run: yarn install - - - name: build - run: yarn build - - - name: test - run: sudo yarn comparison-tests - - comparison_test_windows: - name: Comparison Tests Windows - runs-on: windows-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - - - name: copy files - shell: pwsh - run: | - New-Item C:\source\ts-loader -ItemType Directory - Copy-Item .\* C:\source\ts-loader -Recurse -Force - - - name: install - run: yarn install - working-directory: C:\source\ts-loader - - - name: build - run: yarn build - working-directory: C:\source\ts-loader - - - name: test - run: yarn comparison-tests - working-directory: C:\source\ts-loader - - execution_test_ubuntu: - name: Execution Tests Ubuntu - strategy: - matrix: - node: [26, 24, 22] - ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now - webpack: [4, 5] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: install node - uses: actions/setup-node@v7 - with: - node-version: ${{ matrix.node }} - - - name: install - run: yarn install - - - name: build - run: yarn build - - - name: install typescript - run: yarn add typescript@${{ matrix.ts }} - - - name: install webpack 4 - if: matrix.webpack == 4 - run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 - - - name: test - run: yarn execution-tests - - execution_test_windows: - name: Execution Tests Windows - strategy: - matrix: - node: [26, 24, 22] - ts: [5.6.2, 5.7.2, 5.8.2, 5.9.2, 6.0.2] # next excluded for now - webpack: [4, 5] - runs-on: windows-latest - steps: - - uses: actions/checkout@v7 - - - name: install node - uses: actions/setup-node@v7 - with: - node-version: ${{ matrix.node }} - - - name: copy files - shell: pwsh - run: | - New-Item C:\source\ts-loader -ItemType Directory - Copy-Item .\* C:\source\ts-loader -Recurse -Force - - - name: install - run: yarn install - working-directory: C:\source\ts-loader - - - name: build - run: yarn build - working-directory: C:\source\ts-loader - - - name: install typescript - run: yarn add typescript@${{ matrix.ts }} - working-directory: C:\source\ts-loader - - - name: install webpack 4 - if: matrix.webpack == 4 - run: yarn add webpack@^4.47.0 karma-webpack@^4.0.2 - working-directory: C:\source\ts-loader - - - name: test - run: yarn execution-tests - working-directory: C:\source\ts-loader - - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: install - run: yarn install - - - name: lint - run: yarn lint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0fee4f4a3..722ec989d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,11 +21,7 @@ jobs: node-version: 24 registry-url: https://registry.npmjs.org/ - - name: install - run: yarn install - - - name: build - run: yarn build + - uses: ./.github/actions/build-ts-loader - run: npm install -g npm@latest diff --git a/.github/workflows/test-probe.yml b/.github/workflows/test-probe.yml new file mode 100644 index 000000000..93e94bca8 --- /dev/null +++ b/.github/workflows/test-probe.yml @@ -0,0 +1,118 @@ +name: Test probe + +# A deliberately lean, on-demand counterpart to the `comparison_test_*` jobs +# in comparison-tests.yml - triggered only via workflow_dispatch (never on +# push/PR, so it adds no cost/noise to normal CI) and able to target a single +# test or a name pattern, so a single iteration comes back in a couple of +# minutes instead of running the full comparison/execution test matrix. + +on: + workflow_dispatch: + inputs: + os: + description: 'Which OS(es) to run the probe on' + required: false + type: choice + default: all + options: + - all + - ubuntu-latest + - macos-latest + - windows-latest + single_test: + description: 'Run only this comparison test (matches the test directory name, e.g. "dependencyErrors")' + required: false + type: string + match_test: + description: 'Run only comparison tests whose name matches this regex (ignored if single_test is set)' + required: false + type: string + +jobs: + probe_ubuntu: + name: Comparison test probe (Ubuntu) + if: inputs.os == 'all' || inputs.os == 'ubuntu-latest' + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: test + shell: bash + env: + SINGLE_TEST: ${{ inputs.single_test }} + MATCH_TEST: ${{ inputs.match_test }} + run: | + if [ -n "$SINGLE_TEST" ]; then + yarn comparison-tests --single-test "$SINGLE_TEST" + elif [ -n "$MATCH_TEST" ]; then + yarn comparison-tests --match-test "$MATCH_TEST" + else + yarn comparison-tests + fi + + probe_macos: + name: Comparison test probe (macOS) + if: inputs.os == 'all' || inputs.os == 'macos-latest' + runs-on: macos-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: test + shell: bash + env: + SINGLE_TEST: ${{ inputs.single_test }} + MATCH_TEST: ${{ inputs.match_test }} + run: | + if [ -n "$SINGLE_TEST" ]; then + yarn comparison-tests --single-test "$SINGLE_TEST" + elif [ -n "$MATCH_TEST" ]; then + yarn comparison-tests --match-test "$MATCH_TEST" + else + yarn comparison-tests + fi + + probe_windows: + name: Comparison test probe (Windows) + if: inputs.os == 'all' || inputs.os == 'windows-latest' + runs-on: windows-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: yarn + + - uses: ./.github/actions/build-ts-loader + + - name: test + shell: bash + working-directory: C:\source\ts-loader + env: + SINGLE_TEST: ${{ inputs.single_test }} + MATCH_TEST: ${{ inputs.match_test }} + run: | + if [ -n "$SINGLE_TEST" ]; then + yarn comparison-tests --single-test "$SINGLE_TEST" + elif [ -n "$MATCH_TEST" ]; then + yarn comparison-tests --match-test "$MATCH_TEST" + else + yarn comparison-tests + fi diff --git a/.github/workflows/windows-test-probe.yml b/.github/workflows/windows-test-probe.yml deleted file mode 100644 index 0894608b8..000000000 --- a/.github/workflows/windows-test-probe.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Windows test probe - -# A deliberately lean, on-demand counterpart to the `comparison_test_windows` -# job in push.yml - triggered only via workflow_dispatch (never on push/PR, -# so it adds no cost/noise to normal CI) and able to target a single test or -# a name pattern, so a single iteration comes back in a couple of minutes -# instead of running the full comparison/execution test matrix. - -on: - workflow_dispatch: - inputs: - single_test: - description: 'Run only this comparison test (matches the test directory name, e.g. "dependencyErrors")' - required: false - type: string - match_test: - description: 'Run only comparison tests whose name matches this regex (ignored if single_test is set)' - required: false - type: string - -jobs: - probe: - name: Windows comparison test probe - runs-on: windows-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - - # Windows Defender's real-time scanning adds unpredictable latency to - # file I/O, and this probe does a lot of it (installing, building, then - # running comparison tests) - excluding the working directory keeps - # that noise out of the results. - - name: exclude working directory from Windows Defender - shell: pwsh - run: Add-MpPreference -ExclusionPath "C:\source" - - - name: copy files - shell: pwsh - run: | - New-Item C:\source\ts-loader -ItemType Directory - Copy-Item .\* C:\source\ts-loader -Recurse -Force - - - name: install - run: yarn install - working-directory: C:\source\ts-loader - - - name: build - run: yarn build - working-directory: C:\source\ts-loader - - - name: test - shell: pwsh - working-directory: C:\source\ts-loader - env: - SINGLE_TEST: ${{ inputs.single_test }} - MATCH_TEST: ${{ inputs.match_test }} - run: | - if ($env:SINGLE_TEST -ne "") { - yarn comparison-tests --single-test "$env:SINGLE_TEST" - } elseif ($env:MATCH_TEST -ne "") { - yarn comparison-tests --match-test "$env:MATCH_TEST" - } else { - yarn comparison-tests - } diff --git a/AGENTS.md b/AGENTS.md index 957fa43c1..d2019564f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,9 +44,35 @@ yarn comparison-tests -- --save-output --single-test # regenerate o > Note: test name casing must be exact when using `--single-test`. +### Test probe workflow + +`.github/workflows/test-probe.yml` (registered on `main`, so dispatchable against any branch/ref) runs just the comparison tests via `workflow_dispatch` on Ubuntu, macOS, and/or Windows — much faster than the full CI run from `ci.yml` (which also runs the full Node/TS/webpack execution-test matrix). Use it to iterate on a failing test, or an OS-specific failure, without asking a human to relay CI output. + +Three separate jobs (`probe_ubuntu`/`probe_macos`/`probe_windows`), each gated by the `os` input — defaults to `all`; pass `ubuntu-latest`, `macos-latest`, or `windows-latest` to run just one. All three share their install/build steps (including, on Windows, copying the checkout to `C:\source\ts-loader` so the comparison tests' path normalisation sees a consistent location) via the composite action `.github/actions/build-ts-loader/action.yml`, which every workflow that builds ts-loader uses (`comparison-tests.yml`, `execution-tests.yml`, `lint.yml`, `benchmark.yml`, `release.yml`) — edit that action, not each workflow, if the build steps themselves need to change. + +Requires `gh` CLI authenticated with the `workflow` scope (`gh auth login`, then `gh auth refresh -s workflow` if `gh auth status` doesn't already list `workflow` — both scopes need a human to complete the browser device-flow prompt, they can't be scripted). + +````bash +# trigger — omit single_test/match_test to run the full comparison-test suite; +# omit os (or pass os=all) to run on every platform +gh workflow run test-probe.yml --repo TypeStrong/ts-loader \ + --ref -f os=windows-latest -f single_test= # one test, Windows only +gh workflow run test-probe.yml --repo TypeStrong/ts-loader \ + --ref -f match_test='^(testA|testB)$' # several, by regex, all OSes + +# the trigger command prints the run URL directly - grab the numeric id from it, then: +gh run watch --repo TypeStrong/ts-loader --exit-status # blocks until done + +# `gh run watch` can itself fail on a transient network blip even when the run +# succeeded - always verify conclusion this way rather than trusting its exit code +gh run view --repo TypeStrong/ts-loader --json status,conclusion + +gh run view --repo TypeStrong/ts-loader --log-failed # full failure log text +```` + ## Execution tests (`test/execution-tests/`) -Each sub-directory is a mini webpack project with a Karma/Jasmine test suite. The harness compiles the project and **runs the compiled code** — useful for asserting correct runtime behaviour. These are matrix-tested in CI across multiple Node and TypeScript versions (see `.github/workflows/push.yml` for the current matrix). +Each sub-directory is a mini webpack project with a Karma/Jasmine test suite. The harness compiles the project and **runs the compiled code** — useful for asserting correct runtime behaviour. These are matrix-tested in CI across multiple Node and TypeScript versions (see `.github/workflows/execution-tests.yml` for the current matrix). Full docs: [`test/execution-tests/README.md`](test/execution-tests/README.md) @@ -66,7 +92,7 @@ yarn execution-tests -- --single-test --watch # watch mode (open http: ## Benchmark tests (`test/benchmark-tests/`) -Answers "did this get faster or slower?", not "is the output correct?" - the comparison/execution packs never record timing, so this is a separate harness. It generates a synthetic project on the fly and times ts-loader compiling it (cold build, and incremental rebuild after touching a low-fan-out vs. high-fan-out file, under both `transpileOnly: true`/`false`), comparing two ts-loader checkouts (this build vs. another, e.g. `main`) back-to-back in one process so the relative numbers are meaningful despite noisy CI hosts. +Answers "did this get faster or slower?", not "is the output correct?" - the comparison/execution packs never record timing, so this is a separate harness. It generates a synthetic project on the fly and times ts-loader compiling it (cold build, and incremental rebuild after touching a low-fan-out vs. high-fan-out file, under both `transpileOnly: true`/`false`), comparing two ts-loader checkouts (this build vs. another, e.g. `main`), each in its own process but run concurrently so host noise hits both sides equally and the relative numbers stay meaningful on noisy CI hosts. Full docs: [`test/benchmark-tests/README.md`](test/benchmark-tests/README.md) diff --git a/test/benchmark-tests/README.md b/test/benchmark-tests/README.md index f9665d25e..553aee4a5 100644 --- a/test/benchmark-tests/README.md +++ b/test/benchmark-tests/README.md @@ -20,6 +20,8 @@ Their fixtures are tiny (a handful of files, 1-2 watch patches) and built for ou For each scenario, both sides' `run-side.mts` subprocesses are spawned and run **concurrently** (not one after the other) - this is what keeps the delta near zero when the two checkouts are identical, since any host noise (CPU contention, scheduling) hits both sides at the same wall-clock moment instead of biasing whichever side happened to run in a noisier window. Cold builds run 2 discarded warmup iterations plus several times the requested `--iterations` (see `ITERATION_MULTIPLIER` in `run-benchmark.mts`); incremental rebuilds get a much larger multiplier again, since each rebuild is only tens of ms and cheap to re-run many times to average out that much noisier measurement. The **median** of the measured iterations is reported (robust to the odd stalled one). + Each side also has a 90s wall-clock budget per scenario (`SCENARIO_TIME_BUDGET_MS` in `run-side.mts`): once it's past that budget *and* has its warmup plus at least 2 measured iterations, it stops early and logs `stopped early after / iterations`. That bounds scenarios that are far more expensive per iteration than expected (e.g. a compiler that spawns a native process per instance) so they can't blow the CI job's timeout. Seeing it occasionally for cold typeCheck builds is normal - the median of the iterations that did run is still reported; it's only worth a closer look if a scenario stops after just a handful of iterations, or the two sides stop at very different counts. + ## Running it ```bash @@ -38,7 +40,7 @@ Other flags: `--files ` (default 300, the fixture's module count), `--warmup Results are written to `.benchmark/benchmark-results.json` (raw samples) and `.benchmark/benchmark-results.md` (summary table), and also printed to stdout. -On Windows CI specifically, `.github/workflows/benchmark.yml` also excludes the working directories from Windows Defender's real-time scanning before running the benchmark, since that scanning otherwise adds unpredictable latency to this benchmark's heavy file I/O. +In CI, `.github/workflows/benchmark.yml` runs this on Ubuntu, macOS and Windows (one matrix job per OS), building the PR branch with the shared `.github/actions/build-ts-loader` action and the base branch in a sibling git worktree. On Windows that action copies the checkout to `C:\source\ts-loader`, so the benchmark runs from there (base branch at `C:\source\ts-loader-main`). ## Interpreting deltas diff --git a/test/benchmark-tests/run-side.mts b/test/benchmark-tests/run-side.mts index a805856a8..6dca588ae 100644 --- a/test/benchmark-tests/run-side.mts +++ b/test/benchmark-tests/run-side.mts @@ -35,7 +35,7 @@ const REBUILD_TIMEOUT_MS = 30_000; // timeout long before every scenario finishes. Capping wall-clock instead of // guessing a smaller fixed iteration count keeps full statistical power for // scenarios that stay cheap while still bounding the expensive ones. -const SCENARIO_TIME_BUDGET_MS = 60_000; +const SCENARIO_TIME_BUDGET_MS = 90_000; const MIN_MEASURED_ITERATIONS = 2; function get(flag: string): string {