From 6c23d9a28835029f47ee4887f5caeb7ba5c27cff Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Wed, 29 Jul 2026 23:55:13 +0800 Subject: [PATCH 01/34] Assert TLSWrap EncOut and StreamBase Write buffer counts Narrow nbufs mismatch provenance before uv__try_write (crash-0044). Co-authored-by: Cursor --- src/crypto/crypto_tls.cc | 1 + src/stream_base-inl.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/crypto/crypto_tls.cc b/src/crypto/crypto_tls.cc index cfe760adb3af..cbe0c699fff0 100644 --- a/src/crypto/crypto_tls.cc +++ b/src/crypto/crypto_tls.cc @@ -613,6 +613,7 @@ void TLSWrap::EncOut() { size_t count = arraysize(data); write_size_ = NodeBIO::FromBIO(enc_out_)->PeekMultiple(data, size, &count); CHECK(write_size_ != 0 && count != 0); + v8::recordreplay::Assert("TLSWrap::EncOut %zu %zu", count, write_size_); uv_buf_t buf[arraysize(data)]; uv_buf_t* bufs = buf; diff --git a/src/stream_base-inl.h b/src/stream_base-inl.h index ef86587c0bd2..5cb7f4168de6 100644 --- a/src/stream_base-inl.h +++ b/src/stream_base-inl.h @@ -165,6 +165,8 @@ StreamWriteResult StreamBase::Write( size_t count, uv_stream_t* send_handle, v8::Local req_wrap_obj) { + v8::recordreplay::Assert("StreamBase::Write %zu", count); + Environment* env = stream_env(); int err; From 784267f255234471bbb32716dd3eca71f7b0031a Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:07:45 +0800 Subject: [PATCH 02/34] RQD-14: add runtime-node-build-and-test PipelineUpload + DevSuffix. Chromium-twin BK graph in public node; pin BackendBranch for driver/build/test handoff. --- .buildkite/runtime-node-build-and-test.yml | 124 +++++++++++++++++++++ .gitignore | 1 + REPLAY_BACKEND_REV | 2 +- build.js | 29 ++++- replay_build_scripts/build-pipeline.py | 30 +++++ 5 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 .buildkite/runtime-node-build-and-test.yml create mode 100644 replay_build_scripts/build-pipeline.py diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml new file mode 100644 index 000000000000..c3a78cb70824 --- /dev/null +++ b/.buildkite/runtime-node-build-and-test.yml @@ -0,0 +1,124 @@ +# NodeBKWiring build/test steps (PipelineYamlHome). +# Emitted by replay_build_scripts/build-pipeline.py (bakes pin SHA from +# node REPLAY_BACKEND_REV into the PIN_TOKEN placeholders). +# ${BUILDKITE_*} remain for Buildkite interpolation at PipelineUpload. +steps: + - trigger: "build-driver-linker" + key: "build-driver-linker" + label: ":hammer: Build driver/linker at pin" + build: + commit: "__REPLAY_BACKEND_REV__" + message: "Triggered from node: ${BUILDKITE_MESSAGE}" + + - label: ":bust_in_silhouette: Build node (linux)" + key: "build-node-linux" + depends_on: + - "build-driver-linker" + agents: + - "deploy=true" + - "size=large" + plugins: + - seek-oss/aws-sm#v2.3.1: + region: us-east-2 + env: + EARTHLY_TOKEN: earthly-token + ACCESS_KEY_ID: + secret-id: "prod/deploy-access-keys" + json-key: ".aws_access_key_id" + SECRET_ACCESS_KEY: + secret-id: "prod/deploy-access-keys" + json-key: ".aws_secret_access_key" + - "ssh://git@github.com/replayio/replay-deploy-buildkite-plugin.git#v1.33": + name: "node" + buildcmd: > + export DOCKER_BUILDKIT=1; + # YAML folds this into one string; then two Buildkite upload passes + # interpolate `$VAR`, so shell variables need `$$$$`. `$(...)` needs `$$`. + # Pin SHA is baked at PipelineUpload. + deploycmd: > + set -e; + PIN="__REPLAY_BACKEND_REV__"; + BACKEND_DIR="$$$$(pwd)/../replay-backend-pin"; + rm -rf "$$$$BACKEND_DIR"; + mkdir -p "$$$$BACKEND_DIR"; + git -C "$$$$BACKEND_DIR" init; + git -C "$$$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git; + git -C "$$$$BACKEND_DIR" fetch --depth 1 origin "$$$$PIN"; + git -C "$$$$BACKEND_DIR" checkout FETCH_HEAD; + cd "$$$$BACKEND_DIR"; + BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$$$1}'); + BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)"; + REPLAYIO_NODE_REF="$$$$BUILDKITE_COMMIT" ./scripts/docker/build_node_builder && + docker run --rm + $$$$BUILDKITE_DOCKER_ENVS + -v "$$$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" + linux-node-build + secretenv: + SENTRY_AUTH_TOKEN: sentry-auth-token + HASURA_ADMIN_SECRET: prod/hasura-admin-secret + POSTGRES_PASSWORD: prod/replay-user-postgres-password + TAILSCALE_AUTH_KEY: prod/agent-tailscale-auth-key + + - label: ":clipboard: Node test-suite" + key: "node-test-suite" + depends_on: + - "build-node-linux" + agents: + - "deploy=true" + plugins: + - seek-oss/aws-sm#v2.3.1: + region: us-east-2 + env: + ACCESS_KEY_ID: + secret-id: "prod/deploy-access-keys" + json-key: ".aws_access_key_id" + SECRET_ACCESS_KEY: + secret-id: "prod/deploy-access-keys" + json-key: ".aws_secret_access_key" + command: | + set -e + PIN="__REPLAY_BACKEND_REV__" + BACKEND_DIR="$$(pwd)/../replay-backend-pin" + rm -rf "$$BACKEND_DIR" + mkdir -p "$$BACKEND_DIR" + git -C "$$BACKEND_DIR" init + git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git + git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN" + git -C "$$BACKEND_DIR" checkout FETCH_HEAD + buildkite-agent artifact download build_id . + BUILD_ID="$$(cat build_id)" + cd "$$BACKEND_DIR" + npm ci --prefer-offline --progress=false + npx tsx src/build/buildNode.ts test-suite --build-id "$$BUILD_ID" + + - label: ":jest: Node test-jest" + key: "node-test-jest" + depends_on: + - "build-node-linux" + agents: + - "deploy=true" + plugins: + - seek-oss/aws-sm#v2.3.1: + region: us-east-2 + env: + ACCESS_KEY_ID: + secret-id: "prod/deploy-access-keys" + json-key: ".aws_access_key_id" + SECRET_ACCESS_KEY: + secret-id: "prod/deploy-access-keys" + json-key: ".aws_secret_access_key" + command: | + set -e + PIN="__REPLAY_BACKEND_REV__" + BACKEND_DIR="$$(pwd)/../replay-backend-pin" + rm -rf "$$BACKEND_DIR" + mkdir -p "$$BACKEND_DIR" + git -C "$$BACKEND_DIR" init + git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git + git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN" + git -C "$$BACKEND_DIR" checkout FETCH_HEAD + buildkite-agent artifact download build_id . + BUILD_ID="$$(cat build_id)" + cd "$$BACKEND_DIR" + npm ci --prefer-offline --progress=false + npx tsx src/build/buildNode.ts test-jest --build-id "$$BUILD_ID" diff --git a/.gitignore b/.gitignore index bf8a1d945081..f557f37784dd 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ !.flake8 !.gitattributes !.github +!.buildkite !.gitignore !.gitkeep !.mailmap diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 43938271869c..d252adfee527 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -d887c4838d6a4e305f874edd8cf3d422a068d9a2 \ No newline at end of file +73a6869224308c3059861706d3d295030dc5a11b diff --git a/build.js b/build.js index 95d769705416..cdf2b9c1b00d 100644 --- a/build.js +++ b/build.js @@ -25,8 +25,9 @@ if (localDriverDir) { let driverArchive = `${currentPlatform()}-recordreplay.tgz`; let downloadDriverRevision = process.env.DRIVER_REVISION ? process.env.DRIVER_REVISION : fs.readFileSync("REPLAY_BACKEND_REV", "utf8"); let downloadArchive = `${currentPlatform()}-recordreplay-${downloadDriverRevision.trim().substring(0, 12)}.tgz`; + let downloadUrl = `https://static.replay.io/downloads/${downloadArchive}`; const driverArchivePath = path.join(OutDir, driverArchive); - spawnChecked("curl", [`https://static.replay.io/downloads/${downloadArchive}`, "-o", driverArchivePath], { stdio: "inherit" }); + downloadDriverArchive(downloadUrl, driverArchivePath); spawnChecked("tar", ["xf", driverArchivePath, "-C", OutDir]); fs.unlinkSync(driverArchivePath); @@ -82,6 +83,24 @@ spawnChecked("make", [`-j${numCPUs}`, "-C", OutDir, "BUILDTYPE=Release"], { }, }); +function downloadDriverArchive(downloadUrl, driverArchivePath) { + curl(downloadUrl, driverArchivePath); +} + +function curl(url, outputPath) { + const prettyCmd = ["curl", "--fail", url, "-o", outputPath].join(" "); + console.error(prettyCmd); + + const rv = spawnSync("curl", ["--fail", url, "-o", outputPath], { + stdio: "inherit", + }); + + if (rv.status != 0 || rv.error) { + console.error(rv.error); + throw new Error(`Target driver/linker was not found: ${url}`); + } +} + function spawnChecked(cmd, args, options) { const prettyCmd = [cmd].concat(args).join(" "); console.error(prettyCmd); @@ -142,5 +161,11 @@ function computeBuildId() { // Use the later of the two dates in the build ID. const date = +runtimeDate >= +driverDate ? runtimeDate : driverDate; - return `${currentPlatform()}-node-${date}-${runtimeRevision}-${driverRevision}`; + // Chromium twin: upload_build_artifacts.mjs buildIdExtension / backend utils.ts. + const buildIdExtension = + process.env.BUILDKITE_BRANCH !== process.env.BUILDKITE_PIPELINE_DEFAULT_BRANCH + ? "-dev" + : process.env.LOCAL_DEVELOPER_BUILD_EXTENSION || ""; + + return `${currentPlatform()}-node-${date}-${runtimeRevision}-${driverRevision}${buildIdExtension}`; } diff --git a/replay_build_scripts/build-pipeline.py b/replay_build_scripts/build-pipeline.py new file mode 100644 index 000000000000..c40dfe8de835 --- /dev/null +++ b/replay_build_scripts/build-pipeline.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""PipelineUpload for runtime-node-build-and-test (Chromium twin of build-pipeline.py). + +Reads REPLAY_BACKEND_REV from the node checkout, bakes the pin into +.buildkite/runtime-node-build-and-test.yml, and prints the graph for +`buildkite-agent pipeline upload`. + +Do not curl private replayio/backend for pipeline YAML. +""" + +from pathlib import Path + +PIN_TOKEN = "__REPLAY_BACKEND_REV__" + + +def read_commit_hash(file_path: Path) -> str: + return file_path.read_text().strip().split()[0] + + +def main() -> None: + root = Path(__file__).resolve().parent.parent + pin = read_commit_hash(root / "REPLAY_BACKEND_REV") + yaml_path = root / ".buildkite" / "runtime-node-build-and-test.yml" + # Bake pin so upload does not depend on REPLAY_BACKEND_REV in the agent env. + # Leave ${BUILDKITE_*} for Buildkite interpolation at upload time. + print(yaml_path.read_text().replace(PIN_TOKEN, pin)) + + +if __name__ == "__main__": + main() From f4183b2bd810ac088e08db329978e14c2e4e7ecb Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:12:29 +0800 Subject: [PATCH 03/34] RQD-14: PipelineUpload fetch pin/YAML without full node checkout. --- replay_build_scripts/build-pipeline.py | 36 +++++++++++++++++++------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/replay_build_scripts/build-pipeline.py b/replay_build_scripts/build-pipeline.py index c40dfe8de835..ab90fff1eb6a 100644 --- a/replay_build_scripts/build-pipeline.py +++ b/replay_build_scripts/build-pipeline.py @@ -1,29 +1,47 @@ #!/usr/bin/env python3 """PipelineUpload for runtime-node-build-and-test (Chromium twin of build-pipeline.py). -Reads REPLAY_BACKEND_REV from the node checkout, bakes the pin into -.buildkite/runtime-node-build-and-test.yml, and prints the graph for -`buildkite-agent pipeline upload`. +Bakes REPLAY_BACKEND_REV into .buildkite/runtime-node-build-and-test.yml and +prints the graph for `buildkite-agent pipeline upload`. + +In CI (skip-checkout), fetches pin + YAML from public raw.githubusercontent.com +using BUILDKITE_COMMIT. Locally, reads from the node checkout. Do not curl private replayio/backend for pipeline YAML. """ +import os +import urllib.request from pathlib import Path PIN_TOKEN = "__REPLAY_BACKEND_REV__" +REPO = "replayio/node" + + +def read_commit_hash(text: str) -> str: + return text.strip().split()[0] + + +def fetch_raw(path: str) -> str: + commit = os.environ["BUILDKITE_COMMIT"] + url = f"https://raw.githubusercontent.com/{REPO}/{commit}/{path}" + with urllib.request.urlopen(url) as resp: + return resp.read().decode() -def read_commit_hash(file_path: Path) -> str: - return file_path.read_text().strip().split()[0] +def read_text(rel_path: str) -> str: + local = Path(__file__).resolve().parent.parent / rel_path + if local.is_file(): + return local.read_text() + return fetch_raw(rel_path) def main() -> None: - root = Path(__file__).resolve().parent.parent - pin = read_commit_hash(root / "REPLAY_BACKEND_REV") - yaml_path = root / ".buildkite" / "runtime-node-build-and-test.yml" + pin = read_commit_hash(read_text("REPLAY_BACKEND_REV")) + yaml_text = read_text(".buildkite/runtime-node-build-and-test.yml") # Bake pin so upload does not depend on REPLAY_BACKEND_REV in the agent env. # Leave ${BUILDKITE_*} for Buildkite interpolation at upload time. - print(yaml_path.read_text().replace(PIN_TOKEN, pin)) + print(yaml_text.replace(PIN_TOKEN, pin)) if __name__ == "__main__": From d60b4e802dc5be154358c811be7a27cdebe371a1 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:16:28 +0800 Subject: [PATCH 04/34] RQD-14: match existing BK step labels (no invented trigger label). --- .buildkite/runtime-node-build-and-test.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index c3a78cb70824..df873c9e4a78 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -5,12 +5,11 @@ steps: - trigger: "build-driver-linker" key: "build-driver-linker" - label: ":hammer: Build driver/linker at pin" build: commit: "__REPLAY_BACKEND_REV__" message: "Triggered from node: ${BUILDKITE_MESSAGE}" - - label: ":bust_in_silhouette: Build node (linux)" + - label: ":bust_in_silhouette: Build node" key: "build-node-linux" depends_on: - "build-driver-linker" @@ -59,7 +58,7 @@ steps: POSTGRES_PASSWORD: prod/replay-user-postgres-password TAILSCALE_AUTH_KEY: prod/agent-tailscale-auth-key - - label: ":clipboard: Node test-suite" + - label: "test-suite" key: "node-test-suite" depends_on: - "build-node-linux" @@ -91,7 +90,7 @@ steps: npm ci --prefer-offline --progress=false npx tsx src/build/buildNode.ts test-suite --build-id "$$BUILD_ID" - - label: ":jest: Node test-jest" + - label: "test-jest" key: "node-test-jest" depends_on: - "build-node-linux" From e9c6245a2471f9377e6c1771bbef4b0d4d4a4120 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:23:32 +0800 Subject: [PATCH 05/34] RQD-14: clone node in parallel with build-driver-linker. --- .buildkite/runtime-node-build-and-test.yml | 25 +++++++++++++++++++++- REPLAY_BACKEND_REV | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index df873c9e4a78..8f29f4d76a70 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -9,10 +9,28 @@ steps: commit: "__REPLAY_BACKEND_REV__" message: "Triggered from node: ${BUILDKITE_MESSAGE}" + # Parallel with build-driver-linker: shallow clone for build_node_builder (REPLAYIO_NODE_DIR). + - label: "Clone node" + key: "clone-node" + agents: + - "deploy=true" + plugins: + - thedyrt/skip-checkout#v0.1.1: ~ + command: | + set -e + mkdir -p node-src + git -C node-src init + git -C node-src remote add origin git@github.com:replayio/node.git + git -C node-src fetch --depth 1 origin "$BUILDKITE_COMMIT" + git -C node-src checkout FETCH_HEAD + tar czf node-src.tgz -C node-src . + buildkite-agent artifact upload node-src.tgz + - label: ":bust_in_silhouette: Build node" key: "build-node-linux" depends_on: - "build-driver-linker" + - "clone-node" agents: - "deploy=true" - "size=large" @@ -44,10 +62,15 @@ steps: git -C "$$$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git; git -C "$$$$BACKEND_DIR" fetch --depth 1 origin "$$$$PIN"; git -C "$$$$BACKEND_DIR" checkout FETCH_HEAD; + buildkite-agent artifact download node-src.tgz .; + NODE_SRC="$$$$(pwd)/node-src"; + rm -rf "$$$$NODE_SRC"; + mkdir -p "$$$$NODE_SRC"; + tar xzf node-src.tgz -C "$$$$NODE_SRC"; cd "$$$$BACKEND_DIR"; BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$$$1}'); BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)"; - REPLAYIO_NODE_REF="$$$$BUILDKITE_COMMIT" ./scripts/docker/build_node_builder && + REPLAYIO_NODE_DIR="$$$$NODE_SRC" ./scripts/docker/build_node_builder && docker run --rm $$$$BUILDKITE_DOCKER_ENVS -v "$$$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index d252adfee527..801aaee98889 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -73a6869224308c3059861706d3d295030dc5a11b +dfb9a7ac1bbcd85a7551f7945f370f40700f7f1b From 6a6e1d81c34d1f8d8ddb0e9b80a45e1269e3944f Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:38:48 +0800 Subject: [PATCH 06/34] RQD-14: drop replay-deploy plugin from PR Build node step. --- .buildkite/runtime-node-build-and-test.yml | 66 ++++++++++------------ 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 8f29f4d76a70..5aa6a749cee7 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -26,6 +26,8 @@ steps: tar czf node-src.tgz -C node-src . buildkite-agent artifact upload node-src.tgz + # Plain command — do not use replay-deploy-buildkite-plugin (that emits + # "Deploy to environment" / preprod steps; wrong for PR CI). - label: ":bust_in_silhouette: Build node" key: "build-node-linux" depends_on: @@ -38,48 +40,40 @@ steps: - seek-oss/aws-sm#v2.3.1: region: us-east-2 env: - EARTHLY_TOKEN: earthly-token ACCESS_KEY_ID: secret-id: "prod/deploy-access-keys" json-key: ".aws_access_key_id" SECRET_ACCESS_KEY: secret-id: "prod/deploy-access-keys" json-key: ".aws_secret_access_key" - - "ssh://git@github.com/replayio/replay-deploy-buildkite-plugin.git#v1.33": - name: "node" - buildcmd: > - export DOCKER_BUILDKIT=1; - # YAML folds this into one string; then two Buildkite upload passes - # interpolate `$VAR`, so shell variables need `$$$$`. `$(...)` needs `$$`. - # Pin SHA is baked at PipelineUpload. - deploycmd: > - set -e; - PIN="__REPLAY_BACKEND_REV__"; - BACKEND_DIR="$$$$(pwd)/../replay-backend-pin"; - rm -rf "$$$$BACKEND_DIR"; - mkdir -p "$$$$BACKEND_DIR"; - git -C "$$$$BACKEND_DIR" init; - git -C "$$$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git; - git -C "$$$$BACKEND_DIR" fetch --depth 1 origin "$$$$PIN"; - git -C "$$$$BACKEND_DIR" checkout FETCH_HEAD; - buildkite-agent artifact download node-src.tgz .; - NODE_SRC="$$$$(pwd)/node-src"; - rm -rf "$$$$NODE_SRC"; - mkdir -p "$$$$NODE_SRC"; - tar xzf node-src.tgz -C "$$$$NODE_SRC"; - cd "$$$$BACKEND_DIR"; - BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$$$1}'); - BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)"; - REPLAYIO_NODE_DIR="$$$$NODE_SRC" ./scripts/docker/build_node_builder && - docker run --rm - $$$$BUILDKITE_DOCKER_ENVS - -v "$$$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" - linux-node-build - secretenv: - SENTRY_AUTH_TOKEN: sentry-auth-token - HASURA_ADMIN_SECRET: prod/hasura-admin-secret - POSTGRES_PASSWORD: prod/replay-user-postgres-password - TAILSCALE_AUTH_KEY: prod/agent-tailscale-auth-key + command: | + set -e + export DOCKER_BUILDKIT=1 + PIN="__REPLAY_BACKEND_REV__" + BACKEND_DIR="$$(pwd)/../replay-backend-pin" + rm -rf "$$BACKEND_DIR" + mkdir -p "$$BACKEND_DIR" + git -C "$$BACKEND_DIR" init + git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git + git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN" + git -C "$$BACKEND_DIR" checkout FETCH_HEAD + buildkite-agent artifact download node-src.tgz . + NODE_SRC="$$(pwd)/node-src" + rm -rf "$$NODE_SRC" + mkdir -p "$$NODE_SRC" + tar xzf node-src.tgz -C "$$NODE_SRC" + cd "$$BACKEND_DIR" + BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$1}') + BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)" + REPLAYIO_NODE_DIR="$$NODE_SRC" ./scripts/docker/build_node_builder + docker run --rm \ + $$BUILDKITE_DOCKER_ENVS \ + -e ACCESS_KEY_ID \ + -e SECRET_ACCESS_KEY \ + -e AWS_ACCESS_KEY_ID="$$ACCESS_KEY_ID" \ + -e AWS_SECRET_ACCESS_KEY="$$SECRET_ACCESS_KEY" \ + -v "$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" \ + linux-node-build - label: "test-suite" key: "node-test-suite" From 134b6699581d9653c0e3e86d696a8b9f3cf767a3 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:40:04 +0800 Subject: [PATCH 07/34] RQD-14: wipe node-src before Clone node (skip-checkout dirty workdir). --- .buildkite/runtime-node-build-and-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 5aa6a749cee7..79b0482aa139 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -18,6 +18,8 @@ steps: - thedyrt/skip-checkout#v0.1.1: ~ command: | set -e + # skip-checkout leaves the agent workdir dirty across builds. + rm -rf node-src node-src.tgz mkdir -p node-src git -C node-src init git -C node-src remote add origin git@github.com:replayio/node.git From 8f1a914479c938138fd740d67d31a23faaba4526 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:41:47 +0800 Subject: [PATCH 08/34] RQD-14: use per-job mktemp scratch dirs (agents are stateful). --- .buildkite/runtime-node-build-and-test.yml | 63 ++++++++++++---------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 79b0482aa139..5fd64fbaabf2 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -2,6 +2,9 @@ # Emitted by replay_build_scripts/build-pipeline.py (bakes pin SHA from # node REPLAY_BACKEND_REV into the PIN_TOKEN placeholders). # ${BUILDKITE_*} remain for Buildkite interpolation at PipelineUpload. +# +# Scratch dirs: always mktemp under /tmp (BK agents are partially stateful; +# skip-checkout reuses the pipeline workdir — never init git into a sticky path). steps: - trigger: "build-driver-linker" key: "build-driver-linker" @@ -17,16 +20,17 @@ steps: plugins: - thedyrt/skip-checkout#v0.1.1: ~ command: | - set -e - # skip-checkout leaves the agent workdir dirty across builds. - rm -rf node-src node-src.tgz - mkdir -p node-src - git -C node-src init - git -C node-src remote add origin git@github.com:replayio/node.git - git -C node-src fetch --depth 1 origin "$BUILDKITE_COMMIT" - git -C node-src checkout FETCH_HEAD - tar czf node-src.tgz -C node-src . - buildkite-agent artifact upload node-src.tgz + set -euo pipefail + WORKDIR=$$(mktemp -d "/tmp/clone-node-$${BUILDKITE_JOB_ID}.XXXXXX") + trap 'rm -rf "$$WORKDIR"' EXIT + mkdir "$$WORKDIR/node-src" + git -C "$$WORKDIR/node-src" init + git -C "$$WORKDIR/node-src" remote add origin git@github.com:replayio/node.git + git -C "$$WORKDIR/node-src" fetch --depth 1 origin "$$BUILDKITE_COMMIT" + git -C "$$WORKDIR/node-src" checkout FETCH_HEAD + tar czf "$$WORKDIR/node-src.tgz" -C "$$WORKDIR/node-src" . + # Upload basename so downloaders can use `node-src.tgz` (not an abs path). + buildkite-agent artifact upload "$$WORKDIR/node-src.tgz;node-src.tgz" # Plain command — do not use replay-deploy-buildkite-plugin (that emits # "Deploy to environment" / preprod steps; wrong for PR CI). @@ -49,21 +53,20 @@ steps: secret-id: "prod/deploy-access-keys" json-key: ".aws_secret_access_key" command: | - set -e + set -euo pipefail export DOCKER_BUILDKIT=1 PIN="__REPLAY_BACKEND_REV__" - BACKEND_DIR="$$(pwd)/../replay-backend-pin" - rm -rf "$$BACKEND_DIR" - mkdir -p "$$BACKEND_DIR" + WORKDIR=$$(mktemp -d "/tmp/build-node-$${BUILDKITE_JOB_ID}.XXXXXX") + trap 'rm -rf "$$WORKDIR"' EXIT + BACKEND_DIR="$$WORKDIR/backend" + NODE_SRC="$$WORKDIR/node-src" + mkdir -p "$$BACKEND_DIR" "$$NODE_SRC" git -C "$$BACKEND_DIR" init git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN" git -C "$$BACKEND_DIR" checkout FETCH_HEAD - buildkite-agent artifact download node-src.tgz . - NODE_SRC="$$(pwd)/node-src" - rm -rf "$$NODE_SRC" - mkdir -p "$$NODE_SRC" - tar xzf node-src.tgz -C "$$NODE_SRC" + buildkite-agent artifact download node-src.tgz "$$WORKDIR/" + tar xzf "$$WORKDIR/node-src.tgz" -C "$$NODE_SRC" cd "$$BACKEND_DIR" BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$1}') BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)" @@ -94,17 +97,18 @@ steps: secret-id: "prod/deploy-access-keys" json-key: ".aws_secret_access_key" command: | - set -e + set -euo pipefail PIN="__REPLAY_BACKEND_REV__" - BACKEND_DIR="$$(pwd)/../replay-backend-pin" - rm -rf "$$BACKEND_DIR" + WORKDIR=$$(mktemp -d "/tmp/node-test-suite-$${BUILDKITE_JOB_ID}.XXXXXX") + trap 'rm -rf "$$WORKDIR"' EXIT + BACKEND_DIR="$$WORKDIR/backend" mkdir -p "$$BACKEND_DIR" git -C "$$BACKEND_DIR" init git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN" git -C "$$BACKEND_DIR" checkout FETCH_HEAD - buildkite-agent artifact download build_id . - BUILD_ID="$$(cat build_id)" + buildkite-agent artifact download build_id "$$WORKDIR/" + BUILD_ID="$$(cat "$$WORKDIR/build_id")" cd "$$BACKEND_DIR" npm ci --prefer-offline --progress=false npx tsx src/build/buildNode.ts test-suite --build-id "$$BUILD_ID" @@ -126,17 +130,18 @@ steps: secret-id: "prod/deploy-access-keys" json-key: ".aws_secret_access_key" command: | - set -e + set -euo pipefail PIN="__REPLAY_BACKEND_REV__" - BACKEND_DIR="$$(pwd)/../replay-backend-pin" - rm -rf "$$BACKEND_DIR" + WORKDIR=$$(mktemp -d "/tmp/node-test-jest-$${BUILDKITE_JOB_ID}.XXXXXX") + trap 'rm -rf "$$WORKDIR"' EXIT + BACKEND_DIR="$$WORKDIR/backend" mkdir -p "$$BACKEND_DIR" git -C "$$BACKEND_DIR" init git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN" git -C "$$BACKEND_DIR" checkout FETCH_HEAD - buildkite-agent artifact download build_id . - BUILD_ID="$$(cat build_id)" + buildkite-agent artifact download build_id "$$WORKDIR/" + BUILD_ID="$$(cat "$$WORKDIR/build_id")" cd "$$BACKEND_DIR" npm ci --prefer-offline --progress=false npx tsx src/build/buildNode.ts test-jest --build-id "$$BUILD_ID" From 7e9d635dc31255fde3d5f570d60638c705ded0e0 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:42:40 +0800 Subject: [PATCH 09/34] RQD-14: drop Clone node artifact handoff; build_node_builder clones. --- .buildkite/runtime-node-build-and-test.yml | 33 ++++------------------ 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 5fd64fbaabf2..77e3b03b488f 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -3,8 +3,9 @@ # node REPLAY_BACKEND_REV into the PIN_TOKEN placeholders). # ${BUILDKITE_*} remain for Buildkite interpolation at PipelineUpload. # -# Scratch dirs: always mktemp under /tmp (BK agents are partially stateful; -# skip-checkout reuses the pipeline workdir — never init git into a sticky path). +# Scratch dirs: mktemp under /tmp (BK agents are partially stateful). +# Do not parallel-clone node into a BK artifact — different agents cannot share +# disk; tar upload/download of the tree is pure waste. build_node_builder clones. steps: - trigger: "build-driver-linker" key: "build-driver-linker" @@ -12,33 +13,12 @@ steps: commit: "__REPLAY_BACKEND_REV__" message: "Triggered from node: ${BUILDKITE_MESSAGE}" - # Parallel with build-driver-linker: shallow clone for build_node_builder (REPLAYIO_NODE_DIR). - - label: "Clone node" - key: "clone-node" - agents: - - "deploy=true" - plugins: - - thedyrt/skip-checkout#v0.1.1: ~ - command: | - set -euo pipefail - WORKDIR=$$(mktemp -d "/tmp/clone-node-$${BUILDKITE_JOB_ID}.XXXXXX") - trap 'rm -rf "$$WORKDIR"' EXIT - mkdir "$$WORKDIR/node-src" - git -C "$$WORKDIR/node-src" init - git -C "$$WORKDIR/node-src" remote add origin git@github.com:replayio/node.git - git -C "$$WORKDIR/node-src" fetch --depth 1 origin "$$BUILDKITE_COMMIT" - git -C "$$WORKDIR/node-src" checkout FETCH_HEAD - tar czf "$$WORKDIR/node-src.tgz" -C "$$WORKDIR/node-src" . - # Upload basename so downloaders can use `node-src.tgz` (not an abs path). - buildkite-agent artifact upload "$$WORKDIR/node-src.tgz;node-src.tgz" - # Plain command — do not use replay-deploy-buildkite-plugin (that emits # "Deploy to environment" / preprod steps; wrong for PR CI). - label: ":bust_in_silhouette: Build node" key: "build-node-linux" depends_on: - "build-driver-linker" - - "clone-node" agents: - "deploy=true" - "size=large" @@ -59,18 +39,15 @@ steps: WORKDIR=$$(mktemp -d "/tmp/build-node-$${BUILDKITE_JOB_ID}.XXXXXX") trap 'rm -rf "$$WORKDIR"' EXIT BACKEND_DIR="$$WORKDIR/backend" - NODE_SRC="$$WORKDIR/node-src" - mkdir -p "$$BACKEND_DIR" "$$NODE_SRC" + mkdir -p "$$BACKEND_DIR" git -C "$$BACKEND_DIR" init git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN" git -C "$$BACKEND_DIR" checkout FETCH_HEAD - buildkite-agent artifact download node-src.tgz "$$WORKDIR/" - tar xzf "$$WORKDIR/node-src.tgz" -C "$$NODE_SRC" cd "$$BACKEND_DIR" BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$1}') BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)" - REPLAYIO_NODE_DIR="$$NODE_SRC" ./scripts/docker/build_node_builder + REPLAYIO_NODE_REF="$$BUILDKITE_COMMIT" ./scripts/docker/build_node_builder docker run --rm \ $$BUILDKITE_DOCKER_ENVS \ -e ACCESS_KEY_ID \ From 29aea27d775d990d204c195f798169dd2e21cb3b Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:45:24 +0800 Subject: [PATCH 10/34] =?UTF-8?q?RQD-14:=20DriverBuildCheck=20=E2=80=94=20?= =?UTF-8?q?skip=20build-driver-linker=20when=20archive=20on=20S3.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- replay_build_scripts/build-pipeline.py | 43 +++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/replay_build_scripts/build-pipeline.py b/replay_build_scripts/build-pipeline.py index ab90fff1eb6a..0fa9f4b72046 100644 --- a/replay_build_scripts/build-pipeline.py +++ b/replay_build_scripts/build-pipeline.py @@ -4,6 +4,9 @@ Bakes REPLAY_BACKEND_REV into .buildkite/runtime-node-build-and-test.yml and prints the graph for `buildkite-agent pipeline upload`. +DriverBuildCheck: if linux-recordreplay-.tgz is already on S3, replace the +build-driver-linker trigger with a noop (same key) so depends_on stays valid. + In CI (skip-checkout), fetches pin + YAML from public raw.githubusercontent.com using BUILDKITE_COMMIT. Locally, reads from the node checkout. @@ -11,6 +14,7 @@ """ import os +import urllib.error import urllib.request from pathlib import Path @@ -36,12 +40,49 @@ def read_text(rel_path: str) -> str: return fetch_raw(rel_path) +def driver_archive_present(driver_revision: str) -> bool: + """DriverBuildCheck: True if linux driver archive for this rev is already on S3.""" + url = f"https://static.replay.io/downloads/linux-recordreplay-{driver_revision}.tgz" + req = urllib.request.Request(url, method="HEAD") + try: + with urllib.request.urlopen(req, timeout=10) as resp: + return 200 <= resp.status < 300 + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError): + return False + + +def apply_driver_build_check(yaml_text: str, pin: str) -> str: + """Replace build-driver-linker trigger with noop when archive already on S3.""" + driver_revision = pin[:12] + if not driver_archive_present(driver_revision): + return yaml_text + + trigger = f""" - trigger: "build-driver-linker" + key: "build-driver-linker" + build: + commit: "{pin}" + message: "Triggered from node: ${{BUILDKITE_MESSAGE}}" +""" + noop = f""" - label: "DriverBuildCheck (archive present)" + key: "build-driver-linker" + agents: + - "deploy=true" + plugins: + - thedyrt/skip-checkout#v0.1.1: ~ + command: echo "DriverBuildCheck: linux-recordreplay-{driver_revision}.tgz already on S3" +""" + if trigger not in yaml_text: + raise RuntimeError("DriverBuildCheck: expected build-driver-linker trigger block missing") + return yaml_text.replace(trigger, noop, 1) + + def main() -> None: pin = read_commit_hash(read_text("REPLAY_BACKEND_REV")) yaml_text = read_text(".buildkite/runtime-node-build-and-test.yml") # Bake pin so upload does not depend on REPLAY_BACKEND_REV in the agent env. # Leave ${BUILDKITE_*} for Buildkite interpolation at upload time. - print(yaml_text.replace(PIN_TOKEN, pin)) + yaml_text = yaml_text.replace(PIN_TOKEN, pin) + print(apply_driver_build_check(yaml_text, pin)) if __name__ == "__main__": From 5c59e3ff159149aa101b3982ad5df3047cb32904 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 04:45:56 +0800 Subject: [PATCH 11/34] RQD-14: quote DriverBuildCheck command (YAML colon parse). --- replay_build_scripts/build-pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/replay_build_scripts/build-pipeline.py b/replay_build_scripts/build-pipeline.py index 0fa9f4b72046..2e5b09cdce9f 100644 --- a/replay_build_scripts/build-pipeline.py +++ b/replay_build_scripts/build-pipeline.py @@ -69,7 +69,7 @@ def apply_driver_build_check(yaml_text: str, pin: str) -> str: - "deploy=true" plugins: - thedyrt/skip-checkout#v0.1.1: ~ - command: echo "DriverBuildCheck: linux-recordreplay-{driver_revision}.tgz already on S3" + command: "echo DriverBuildCheck linux-recordreplay-{driver_revision}.tgz already on S3" """ if trigger not in yaml_text: raise RuntimeError("DriverBuildCheck: expected build-driver-linker trigger block missing") From 74714d5805172213f6639ce9d5dbd5b32a6f118b Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 16:57:02 +0800 Subject: [PATCH 12/34] RQD-14: raise Build node timeout to 120m (cold make exceeds BK 60m default). Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 77e3b03b488f..917839354416 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -17,6 +17,8 @@ steps: # "Deploy to environment" / preprod steps; wrong for PR CI). - label: ":bust_in_silhouette: Build node" key: "build-node-linux" + # BK default is 60m; cold node make -j4 exceeds that (see build #12). + timeout_in_minutes: 120 depends_on: - "build-driver-linker" agents: From 9b3d63bef9d8095e09d7f12e714efd592e098df0 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 17:18:00 +0800 Subject: [PATCH 13/34] RQD-14: persist host NodeRepo for incremental make across CI jobs. Mount the agent checkout+out over the image node path so cold full rebuilds are not forced every run. Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 917839354416..7e1f4ad2cbdb 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -6,6 +6,10 @@ # Scratch dirs: mktemp under /tmp (BK agents are partially stateful). # Do not parallel-clone node into a BK artifact — different agents cannot share # disk; tar upload/download of the tree is pure waste. build_node_builder clones. +# +# NodeRepo: host checkout+out mounted over the image node path so make can +# incremental-rebuild across jobs on the same agent. Mounting only out/ is not +# enough — a fresh image COPY of sources resets mtimes and forces a full rebuild. steps: - trigger: "build-driver-linker" key: "build-driver-linker" @@ -50,6 +54,21 @@ steps: BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$1}') BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)" REPLAYIO_NODE_REF="$$BUILDKITE_COMMIT" ./scripts/docker/build_node_builder + NODE_REPO="/var/lib/buildkite-agent/cache/runtime-node-build/repo" + mkdir -p "$$NODE_REPO" + reclaim_node_repo() { + docker run --rm \ + -v "$$NODE_REPO:/repo" \ + --entrypoint chown \ + linux-node-build \ + -R "$$(id -u):$$(id -g)" /repo + } + reclaim_node_repo + if [ ! -d "$$NODE_REPO/.git" ]; then + git clone git@github.com:replayio/node.git "$$NODE_REPO" + fi + git -C "$$NODE_REPO" fetch --depth 1 origin "$$BUILDKITE_COMMIT" + git -C "$$NODE_REPO" checkout -f --detach FETCH_HEAD docker run --rm \ $$BUILDKITE_DOCKER_ENVS \ -e ACCESS_KEY_ID \ @@ -57,7 +76,9 @@ steps: -e AWS_ACCESS_KEY_ID="$$ACCESS_KEY_ID" \ -e AWS_SECRET_ACCESS_KEY="$$SECRET_ACCESS_KEY" \ -v "$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" \ + -v "$$NODE_REPO:/usr/build/tmp/replayio-node-repo" \ linux-node-build + reclaim_node_repo - label: "test-suite" key: "node-test-suite" From 6cf1b052814204a359674874d6cd6c08cff457f9 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 17:27:19 +0800 Subject: [PATCH 14/34] RQD-14: mark mounted NodeRepo safe.directory for root docker git. Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 7e1f4ad2cbdb..f91d9bedc545 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -69,12 +69,16 @@ steps: fi git -C "$$NODE_REPO" fetch --depth 1 origin "$$BUILDKITE_COMMIT" git -C "$$NODE_REPO" checkout -f --detach FETCH_HEAD + # Container runs as root; NodeRepo is agent-owned after reclaim → mark safe.directory. docker run --rm \ $$BUILDKITE_DOCKER_ENVS \ -e ACCESS_KEY_ID \ -e SECRET_ACCESS_KEY \ -e AWS_ACCESS_KEY_ID="$$ACCESS_KEY_ID" \ -e AWS_SECRET_ACCESS_KEY="$$SECRET_ACCESS_KEY" \ + -e GIT_CONFIG_COUNT=1 \ + -e GIT_CONFIG_KEY_0=safe.directory \ + -e GIT_CONFIG_VALUE_0=/usr/build/tmp/replayio-node-repo \ -v "$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" \ -v "$$NODE_REPO:/usr/build/tmp/replayio-node-repo" \ linux-node-build From 76dd59601b3f0dc390736e50ddc9e77185852ac5 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 17:35:44 +0800 Subject: [PATCH 15/34] RQD-14: pin backend safe.directory fix; drop ineffective GIT_CONFIG_*. Container git lacks GIT_CONFIG_*; safe.directory is set in run_node_build.sh. Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 6 ++---- REPLAY_BACKEND_REV | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index f91d9bedc545..b98c31a22c2c 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -69,16 +69,14 @@ steps: fi git -C "$$NODE_REPO" fetch --depth 1 origin "$$BUILDKITE_COMMIT" git -C "$$NODE_REPO" checkout -f --detach FETCH_HEAD - # Container runs as root; NodeRepo is agent-owned after reclaim → mark safe.directory. + # safe.directory is set inside the image (run_node_build.sh / buildNodeDirectly). + # Do not use GIT_CONFIG_* — Ubuntu 20.04 git lacks that feature. docker run --rm \ $$BUILDKITE_DOCKER_ENVS \ -e ACCESS_KEY_ID \ -e SECRET_ACCESS_KEY \ -e AWS_ACCESS_KEY_ID="$$ACCESS_KEY_ID" \ -e AWS_SECRET_ACCESS_KEY="$$SECRET_ACCESS_KEY" \ - -e GIT_CONFIG_COUNT=1 \ - -e GIT_CONFIG_KEY_0=safe.directory \ - -e GIT_CONFIG_VALUE_0=/usr/build/tmp/replayio-node-repo \ -v "$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" \ -v "$$NODE_REPO:/usr/build/tmp/replayio-node-repo" \ linux-node-build diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 801aaee98889..fed1b6825658 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -dfb9a7ac1bbcd85a7551f7945f370f40700f7f1b +ab8047ea0a52ddb807b234f3d700a55f079da78e From 6ffef7b4820ab301adfca0ea3fd5479fb9c340f0 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 19:26:47 +0800 Subject: [PATCH 16/34] RQD-14: stop injecting github-backend AWS keys into linux-node-build. prod/deploy-access-keys cannot s3:PutObject builds/; match build-node.yml and use the beefmaster agent identity inside docker. Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index b98c31a22c2c..f689c3d95982 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -28,16 +28,6 @@ steps: agents: - "deploy=true" - "size=large" - plugins: - - seek-oss/aws-sm#v2.3.1: - region: us-east-2 - env: - ACCESS_KEY_ID: - secret-id: "prod/deploy-access-keys" - json-key: ".aws_access_key_id" - SECRET_ACCESS_KEY: - secret-id: "prod/deploy-access-keys" - json-key: ".aws_secret_access_key" command: | set -euo pipefail export DOCKER_BUILDKIT=1 @@ -69,14 +59,12 @@ steps: fi git -C "$$NODE_REPO" fetch --depth 1 origin "$$BUILDKITE_COMMIT" git -C "$$NODE_REPO" checkout -f --detach FETCH_HEAD + # Match build-node.yml: do not inject prod/deploy-access-keys (github-backend) + # into the container — that IAM user cannot s3:PutObject builds/. beefmaster + # docker uses the agent identity (same as the manual build-node pipeline). # safe.directory is set inside the image (run_node_build.sh / buildNodeDirectly). - # Do not use GIT_CONFIG_* — Ubuntu 20.04 git lacks that feature. docker run --rm \ $$BUILDKITE_DOCKER_ENVS \ - -e ACCESS_KEY_ID \ - -e SECRET_ACCESS_KEY \ - -e AWS_ACCESS_KEY_ID="$$ACCESS_KEY_ID" \ - -e AWS_SECRET_ACCESS_KEY="$$SECRET_ACCESS_KEY" \ -v "$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" \ -v "$$NODE_REPO:/usr/build/tmp/replayio-node-repo" \ linux-node-build From f6fa956d1bdccc352b0a4ab17a5b8ab9f2ddbcd4 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 20:44:43 +0800 Subject: [PATCH 17/34] RQD-14: pin configure-skip; give tests HASURA_ADMIN_SECRET. Skip configure when out/Makefile exists for incremental NodeRepo make. Test steps need Hasura secret (ProcessorExit cascade without it). Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 16 +++------------- REPLAY_BACKEND_REV | 2 +- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index f689c3d95982..d0945dc34ef1 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -54,6 +54,7 @@ steps: -R "$$(id -u):$$(id -g)" /repo } reclaim_node_repo + trap 'reclaim_node_repo; rm -rf "$$WORKDIR"' EXIT if [ ! -d "$$NODE_REPO/.git" ]; then git clone git@github.com:replayio/node.git "$$NODE_REPO" fi @@ -68,7 +69,6 @@ steps: -v "$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" \ -v "$$NODE_REPO:/usr/build/tmp/replayio-node-repo" \ linux-node-build - reclaim_node_repo - label: "test-suite" key: "node-test-suite" @@ -80,12 +80,7 @@ steps: - seek-oss/aws-sm#v2.3.1: region: us-east-2 env: - ACCESS_KEY_ID: - secret-id: "prod/deploy-access-keys" - json-key: ".aws_access_key_id" - SECRET_ACCESS_KEY: - secret-id: "prod/deploy-access-keys" - json-key: ".aws_secret_access_key" + HASURA_ADMIN_SECRET: "prod/hasura-admin-secret" command: | set -euo pipefail PIN="__REPLAY_BACKEND_REV__" @@ -113,12 +108,7 @@ steps: - seek-oss/aws-sm#v2.3.1: region: us-east-2 env: - ACCESS_KEY_ID: - secret-id: "prod/deploy-access-keys" - json-key: ".aws_access_key_id" - SECRET_ACCESS_KEY: - secret-id: "prod/deploy-access-keys" - json-key: ".aws_secret_access_key" + HASURA_ADMIN_SECRET: "prod/hasura-admin-secret" command: | set -euo pipefail PIN="__REPLAY_BACKEND_REV__" diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index fed1b6825658..125e07d18fcb 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -ab8047ea0a52ddb807b234f3d700a55f079da78e +d87a9c0cc5771293a2d64ba88b59a0620e6cb4e6 From 3e6ac93209ead0b6c2daac57f2bbe8910ae9feeb Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 21:38:04 +0800 Subject: [PATCH 18/34] RQD-14: mtime-safe NodeRepo fetch; log disk at build start. Drop shallow fetch on the cached repo (it rewrote gyp mtimes and forced Makefile regen). Unshallow once if needed. df/du at job start for disk headroom. Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 16 +++- .github/actions/build-test-branch/action.yml | 6 -- .github/actions/build-test-branch/main.js | 79 -------------------- .github/actions/build-test/action.yml | 6 -- .github/actions/build-test/main.js | 56 -------------- .github/workflows/build-test-branch.yml | 36 --------- .github/workflows/build-test.yml | 32 -------- 7 files changed, 13 insertions(+), 218 deletions(-) delete mode 100644 .github/actions/build-test-branch/action.yml delete mode 100644 .github/actions/build-test-branch/main.js delete mode 100644 .github/actions/build-test/action.yml delete mode 100644 .github/actions/build-test/main.js delete mode 100644 .github/workflows/build-test-branch.yml delete mode 100644 .github/workflows/build-test.yml diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index d0945dc34ef1..c98c9f3f255c 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -10,6 +10,8 @@ # NodeRepo: host checkout+out mounted over the image node path so make can # incremental-rebuild across jobs on the same agent. Mounting only out/ is not # enough — a fresh image COPY of sources resets mtimes and forces a full rebuild. +# Do not `git fetch --depth 1` the cached NodeRepo — shallow fetch rewrites the +# tree and bumps gyp input mtimes → make "Regenerating Makefile" → near-full rebuild. steps: - trigger: "build-driver-linker" key: "build-driver-linker" @@ -31,6 +33,12 @@ steps: command: | set -euo pipefail export DOCKER_BUILDKIT=1 + echo "=== Disk space (build start) ===" + df -h + NODE_REPO="/var/lib/buildkite-agent/cache/runtime-node-build/repo" + if [ -d "$$NODE_REPO" ]; then + du -sh "$$NODE_REPO" "$$NODE_REPO/out" 2>/dev/null || du -sh "$$NODE_REPO" 2>/dev/null || true + fi PIN="__REPLAY_BACKEND_REV__" WORKDIR=$$(mktemp -d "/tmp/build-node-$${BUILDKITE_JOB_ID}.XXXXXX") trap 'rm -rf "$$WORKDIR"' EXIT @@ -44,7 +52,6 @@ steps: BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$1}') BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)" REPLAYIO_NODE_REF="$$BUILDKITE_COMMIT" ./scripts/docker/build_node_builder - NODE_REPO="/var/lib/buildkite-agent/cache/runtime-node-build/repo" mkdir -p "$$NODE_REPO" reclaim_node_repo() { docker run --rm \ @@ -57,9 +64,12 @@ steps: trap 'reclaim_node_repo; rm -rf "$$WORKDIR"' EXIT if [ ! -d "$$NODE_REPO/.git" ]; then git clone git@github.com:replayio/node.git "$$NODE_REPO" + elif [ -f "$$NODE_REPO/.git/shallow" ]; then + # Prior jobs used --depth 1; unshallow once so later checkouts keep mtimes. + git -C "$$NODE_REPO" fetch --unshallow origin || git -C "$$NODE_REPO" fetch --deepen=2147483647 origin fi - git -C "$$NODE_REPO" fetch --depth 1 origin "$$BUILDKITE_COMMIT" - git -C "$$NODE_REPO" checkout -f --detach FETCH_HEAD + git -C "$$NODE_REPO" fetch origin "$$BUILDKITE_COMMIT" + git -C "$$NODE_REPO" checkout -f --detach "$$BUILDKITE_COMMIT" # Match build-node.yml: do not inject prod/deploy-access-keys (github-backend) # into the container — that IAM user cannot s3:PutObject builds/. beefmaster # docker uses the agent identity (same as the manual build-node pipeline). diff --git a/.github/actions/build-test-branch/action.yml b/.github/actions/build-test-branch/action.yml deleted file mode 100644 index 76a5f7911976..000000000000 --- a/.github/actions/build-test-branch/action.yml +++ /dev/null @@ -1,6 +0,0 @@ -name: "Build/Test" -description: "Trigger build/test run" - -runs: - using: "node20" - main: "./main.js" diff --git a/.github/actions/build-test-branch/main.js b/.github/actions/build-test-branch/main.js deleted file mode 100644 index 0ace318ea981..000000000000 --- a/.github/actions/build-test-branch/main.js +++ /dev/null @@ -1,79 +0,0 @@ - -const { - getLatestRevision, - sendBuildTestRequest, - newTask, -} = require("../utils"); - -const branchName = process.env.GITHUB_REF_NAME; -console.log("BranchName", branchName); - -const nodeRevision = getLatestRevision(); - -const driverRevision = process.env.INPUT_DRIVER_REVISION; -console.log("DriverRevision", driverRevision); - -const clobberInput = process.env.INPUT_CLOBBER; -console.log("Clobber", clobberInput); -const clobber = clobberInput == "true"; - -const slotInput = process.env.INPUT_SLOT; -console.log("Slot", slotInput); -const slot = slotInput ? +slotInput : undefined; - -let requestName = `Node Build/Test Branch ${branchName} ${nodeRevision}`; -if (driverRevision) { - requestName += ` driver ${driverRevision}`; -} -if (slot) { - requestName += ` slot ${slot}`; -} - -sendBuildTestRequest({ - name: requestName, - tasks: [ - ...platformTasks("macOS"), - ...platformTasks("linux"), - ], -}); - -function platformTasks(platform) { - const buildTask = newTask( - `Build Node ${platform}`, - { - kind: "BuildRuntime", - runtime: "node", - revision: nodeRevision, - branch: branchName, - branchSlot: slot, - driverRevision, - clobber, - }, - platform - ); - - const testTask = newTask( - `Test Node ${platform}`, - { - kind: "RunTestSuite", - runtime: "node", - revision: nodeRevision, - driverRevision, - }, - platform, - [buildTask] - ); - - const jestTask = newTask( - `Run random Jest tests ${platform}`, - { - kind: "JestTests", - revision: nodeRevision, - driverRevision, - }, - platform, - [buildTask] - ); - - return [buildTask, testTask, jestTask]; -} diff --git a/.github/actions/build-test/action.yml b/.github/actions/build-test/action.yml deleted file mode 100644 index 76a5f7911976..000000000000 --- a/.github/actions/build-test/action.yml +++ /dev/null @@ -1,6 +0,0 @@ -name: "Build/Test" -description: "Trigger build/test run" - -runs: - using: "node20" - main: "./main.js" diff --git a/.github/actions/build-test/main.js b/.github/actions/build-test/main.js deleted file mode 100644 index eace5b00c7c1..000000000000 --- a/.github/actions/build-test/main.js +++ /dev/null @@ -1,56 +0,0 @@ - -const { - getLatestRevision, - sendBuildTestRequest, - newTask, -} = require("../utils"); - -const revision = getLatestRevision(); - -const clobberInput = process.env.INPUT_CLOBBER; -console.log("Clobber", clobberInput); -const clobber = clobberInput == "true"; - -sendBuildTestRequest({ - name: `Node Build/Test ${revision}`, - tasks: [ - ...platformTasks("macOS"), - ...platformTasks("linux"), - ], -}); - -function platformTasks(platform) { - const buildTask = newTask( - `Build Node ${platform}`, - { - kind: "BuildRuntime", - runtime: "node", - revision, - clobber, - }, - platform - ); - - const testTask = newTask( - `Test Node ${platform}`, - { - kind: "RunTestSuite", - runtime: "node", - revision, - }, - platform, - [buildTask] - ); - - const jestTask = newTask( - `Run random Jest tests ${platform}`, - { - kind: "JestTests", - revision, - }, - platform, - [buildTask] - ); - - return [buildTask, testTask, jestTask]; -} diff --git a/.github/workflows/build-test-branch.yml b/.github/workflows/build-test-branch.yml deleted file mode 100644 index cdb55f359451..000000000000 --- a/.github/workflows/build-test-branch.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Build/Test Branch -on: - workflow_dispatch: - inputs: - driver_revision: - description: "Driver revision to use, if not the latest" - required: false - clobber: - description: "Build node from scratch" - type: boolean - required: false - slot: - description: "Checked out repository slot to use" - required: false - -jobs: - build-test: - name: Trigger build/test run on current branch - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Connect to Tailscale - uses: tailscale/github-action@v3 - with: - authkey: ${{ secrets.TAILSCALE_API_KEY }} - tags: tag:node-fork-ci - - uses: actions/checkout@v2 - - uses: ./.github/actions/build-test-branch - env: - BUILD_TEST_AUTHORIZATION: ${{ secrets.BUILD_TEST_AUTHORIZATION }} - BUILD_TEST_HOSTNAME: a49855c191a944333918aea7ad31bc76-6a8a830d89921d8a.elb.us-east-2.amazonaws.com - BUILD_TEST_PORT: ${{ secrets.BUILD_TEST_PORT }} - BUILD_TEST_INSECURE: ${{ secrets.BUILD_TEST_INSECURE }} - INPUT_DRIVER_REVISION: ${{ github.event.inputs.driver_revision }} - INPUT_CLOBBER: ${{ github.event.inputs.clobber }} - INPUT_SLOT: ${{ github.event.inputs.slot }} diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml deleted file mode 100644 index aa7dc83f9f78..000000000000 --- a/.github/workflows/build-test.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Build/Test -on: - workflow_dispatch: - inputs: - clobber: - description: "Build node from scratch" - type: boolean - required: false - pull_request: - push: - branches: - - master - -jobs: - build-test: - name: Trigger build/test run - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Connect to Tailscale - uses: tailscale/github-action@v3 - with: - authkey: ${{ secrets.TAILSCALE_API_KEY }} - tags: tag:node-fork-ci - - uses: actions/checkout@v2 - - uses: ./.github/actions/build-test - env: - BUILD_TEST_AUTHORIZATION: ${{ secrets.BUILD_TEST_AUTHORIZATION }} - BUILD_TEST_HOSTNAME: a49855c191a944333918aea7ad31bc76-6a8a830d89921d8a.elb.us-east-2.amazonaws.com - BUILD_TEST_PORT: ${{ secrets.BUILD_TEST_PORT }} - BUILD_TEST_INSECURE: ${{ secrets.BUILD_TEST_INSECURE }} - INPUT_CLOBBER: ${{ github.event.inputs.clobber }} From ab0ae39811d98260ab07b6de405f637ede2704ab Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 21:38:14 +0800 Subject: [PATCH 19/34] RQD-14: restore accidentally deleted GHA build-test files. Prior commit had unrelated staged deletions; keep additive pipeline work only. Co-authored-by: Cursor --- .github/actions/build-test-branch/action.yml | 6 ++ .github/actions/build-test-branch/main.js | 79 ++++++++++++++++++++ .github/actions/build-test/action.yml | 6 ++ .github/actions/build-test/main.js | 56 ++++++++++++++ .github/workflows/build-test-branch.yml | 36 +++++++++ .github/workflows/build-test.yml | 32 ++++++++ 6 files changed, 215 insertions(+) create mode 100644 .github/actions/build-test-branch/action.yml create mode 100644 .github/actions/build-test-branch/main.js create mode 100644 .github/actions/build-test/action.yml create mode 100644 .github/actions/build-test/main.js create mode 100644 .github/workflows/build-test-branch.yml create mode 100644 .github/workflows/build-test.yml diff --git a/.github/actions/build-test-branch/action.yml b/.github/actions/build-test-branch/action.yml new file mode 100644 index 000000000000..76a5f7911976 --- /dev/null +++ b/.github/actions/build-test-branch/action.yml @@ -0,0 +1,6 @@ +name: "Build/Test" +description: "Trigger build/test run" + +runs: + using: "node20" + main: "./main.js" diff --git a/.github/actions/build-test-branch/main.js b/.github/actions/build-test-branch/main.js new file mode 100644 index 000000000000..0ace318ea981 --- /dev/null +++ b/.github/actions/build-test-branch/main.js @@ -0,0 +1,79 @@ + +const { + getLatestRevision, + sendBuildTestRequest, + newTask, +} = require("../utils"); + +const branchName = process.env.GITHUB_REF_NAME; +console.log("BranchName", branchName); + +const nodeRevision = getLatestRevision(); + +const driverRevision = process.env.INPUT_DRIVER_REVISION; +console.log("DriverRevision", driverRevision); + +const clobberInput = process.env.INPUT_CLOBBER; +console.log("Clobber", clobberInput); +const clobber = clobberInput == "true"; + +const slotInput = process.env.INPUT_SLOT; +console.log("Slot", slotInput); +const slot = slotInput ? +slotInput : undefined; + +let requestName = `Node Build/Test Branch ${branchName} ${nodeRevision}`; +if (driverRevision) { + requestName += ` driver ${driverRevision}`; +} +if (slot) { + requestName += ` slot ${slot}`; +} + +sendBuildTestRequest({ + name: requestName, + tasks: [ + ...platformTasks("macOS"), + ...platformTasks("linux"), + ], +}); + +function platformTasks(platform) { + const buildTask = newTask( + `Build Node ${platform}`, + { + kind: "BuildRuntime", + runtime: "node", + revision: nodeRevision, + branch: branchName, + branchSlot: slot, + driverRevision, + clobber, + }, + platform + ); + + const testTask = newTask( + `Test Node ${platform}`, + { + kind: "RunTestSuite", + runtime: "node", + revision: nodeRevision, + driverRevision, + }, + platform, + [buildTask] + ); + + const jestTask = newTask( + `Run random Jest tests ${platform}`, + { + kind: "JestTests", + revision: nodeRevision, + driverRevision, + }, + platform, + [buildTask] + ); + + return [buildTask, testTask, jestTask]; +} diff --git a/.github/actions/build-test/action.yml b/.github/actions/build-test/action.yml new file mode 100644 index 000000000000..76a5f7911976 --- /dev/null +++ b/.github/actions/build-test/action.yml @@ -0,0 +1,6 @@ +name: "Build/Test" +description: "Trigger build/test run" + +runs: + using: "node20" + main: "./main.js" diff --git a/.github/actions/build-test/main.js b/.github/actions/build-test/main.js new file mode 100644 index 000000000000..eace5b00c7c1 --- /dev/null +++ b/.github/actions/build-test/main.js @@ -0,0 +1,56 @@ + +const { + getLatestRevision, + sendBuildTestRequest, + newTask, +} = require("../utils"); + +const revision = getLatestRevision(); + +const clobberInput = process.env.INPUT_CLOBBER; +console.log("Clobber", clobberInput); +const clobber = clobberInput == "true"; + +sendBuildTestRequest({ + name: `Node Build/Test ${revision}`, + tasks: [ + ...platformTasks("macOS"), + ...platformTasks("linux"), + ], +}); + +function platformTasks(platform) { + const buildTask = newTask( + `Build Node ${platform}`, + { + kind: "BuildRuntime", + runtime: "node", + revision, + clobber, + }, + platform + ); + + const testTask = newTask( + `Test Node ${platform}`, + { + kind: "RunTestSuite", + runtime: "node", + revision, + }, + platform, + [buildTask] + ); + + const jestTask = newTask( + `Run random Jest tests ${platform}`, + { + kind: "JestTests", + revision, + }, + platform, + [buildTask] + ); + + return [buildTask, testTask, jestTask]; +} diff --git a/.github/workflows/build-test-branch.yml b/.github/workflows/build-test-branch.yml new file mode 100644 index 000000000000..cdb55f359451 --- /dev/null +++ b/.github/workflows/build-test-branch.yml @@ -0,0 +1,36 @@ +name: Build/Test Branch +on: + workflow_dispatch: + inputs: + driver_revision: + description: "Driver revision to use, if not the latest" + required: false + clobber: + description: "Build node from scratch" + type: boolean + required: false + slot: + description: "Checked out repository slot to use" + required: false + +jobs: + build-test: + name: Trigger build/test run on current branch + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Connect to Tailscale + uses: tailscale/github-action@v3 + with: + authkey: ${{ secrets.TAILSCALE_API_KEY }} + tags: tag:node-fork-ci + - uses: actions/checkout@v2 + - uses: ./.github/actions/build-test-branch + env: + BUILD_TEST_AUTHORIZATION: ${{ secrets.BUILD_TEST_AUTHORIZATION }} + BUILD_TEST_HOSTNAME: a49855c191a944333918aea7ad31bc76-6a8a830d89921d8a.elb.us-east-2.amazonaws.com + BUILD_TEST_PORT: ${{ secrets.BUILD_TEST_PORT }} + BUILD_TEST_INSECURE: ${{ secrets.BUILD_TEST_INSECURE }} + INPUT_DRIVER_REVISION: ${{ github.event.inputs.driver_revision }} + INPUT_CLOBBER: ${{ github.event.inputs.clobber }} + INPUT_SLOT: ${{ github.event.inputs.slot }} diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 000000000000..aa7dc83f9f78 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,32 @@ +name: Build/Test +on: + workflow_dispatch: + inputs: + clobber: + description: "Build node from scratch" + type: boolean + required: false + pull_request: + push: + branches: + - master + +jobs: + build-test: + name: Trigger build/test run + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Connect to Tailscale + uses: tailscale/github-action@v3 + with: + authkey: ${{ secrets.TAILSCALE_API_KEY }} + tags: tag:node-fork-ci + - uses: actions/checkout@v2 + - uses: ./.github/actions/build-test + env: + BUILD_TEST_AUTHORIZATION: ${{ secrets.BUILD_TEST_AUTHORIZATION }} + BUILD_TEST_HOSTNAME: a49855c191a944333918aea7ad31bc76-6a8a830d89921d8a.elb.us-east-2.amazonaws.com + BUILD_TEST_PORT: ${{ secrets.BUILD_TEST_PORT }} + BUILD_TEST_INSECURE: ${{ secrets.BUILD_TEST_INSECURE }} + INPUT_CLOBBER: ${{ github.event.inputs.clobber }} From b807ac7c3082ad5b9efeab2607925c6fbe812790 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 21:48:35 +0800 Subject: [PATCH 20/34] RQD-14: stop chown-ctime from forcing full NodeRepo checkout. reclaim_node_repo chown -R dirties every path via ctime; checkout -f then rewrote all 32k sources and triggered Regenerating Makefile. Ignore ctime and refresh the index after reclaim. Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index c98c9f3f255c..42dbdaf4cc93 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -12,6 +12,9 @@ # enough — a fresh image COPY of sources resets mtimes and forces a full rebuild. # Do not `git fetch --depth 1` the cached NodeRepo — shallow fetch rewrites the # tree and bumps gyp input mtimes → make "Regenerating Makefile" → near-full rebuild. +# reclaim_node_repo chown -R bumps ctime on every file; with default git stat +# checks that dirties the whole tree and `checkout -f` rewrites all sources. +# core.trustctime=false + update-index --refresh after chown keeps mtimes. steps: - trigger: "build-driver-linker" key: "build-driver-linker" @@ -68,7 +71,14 @@ steps: # Prior jobs used --depth 1; unshallow once so later checkouts keep mtimes. git -C "$$NODE_REPO" fetch --unshallow origin || git -C "$$NODE_REPO" fetch --deepen=2147483647 origin fi + # chown -R (reclaim) bumps ctime; without this, checkout -f rewrites all files. + git -C "$$NODE_REPO" config core.trustctime false + git -C "$$NODE_REPO" update-index --refresh || true git -C "$$NODE_REPO" fetch origin "$$BUILDKITE_COMMIT" + echo "=== NodeRepo sync (HEAD -> $$BUILDKITE_COMMIT) ===" + echo "HEAD=$$(git -C "$$NODE_REPO" rev-parse HEAD 2>/dev/null || echo none)" + echo "dirty=$$(git -C "$$NODE_REPO" status --porcelain 2>/dev/null | wc -l)" + echo "diff_files=$$(git -C "$$NODE_REPO" diff --name-only HEAD "$$BUILDKITE_COMMIT" 2>/dev/null | wc -l)" git -C "$$NODE_REPO" checkout -f --detach "$$BUILDKITE_COMMIT" # Match build-node.yml: do not inject prod/deploy-access-keys (github-backend) # into the container — that IAM user cannot s3:PutObject builds/. beefmaster From 3afdf095e805adb2c223e706af769188ff2ebca2 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 23:51:55 +0800 Subject: [PATCH 21/34] RQD-14: pin backend after local/test crash-upload skip. Co-authored-by: Cursor --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 125e07d18fcb..e16a0b534c40 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -d87a9c0cc5771293a2d64ba88b59a0620e6cb4e6 +8289a556bb2933d40b60d4226d27812f43c34f82 From 6c77f59900060cb9eadbd08507f8ea1ec984169b Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 23:53:05 +0800 Subject: [PATCH 22/34] RQD-14: pin backend after reverting crash-upload skip. Co-authored-by: Cursor --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index e16a0b534c40..10f0f328d4c6 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -8289a556bb2933d40b60d4226d27812f43c34f82 +c2fa6c44cf5497cf1e83babda8b7a8a53b09e229 From 9442f1d0e1117fe11105043c18c6f1c47fa4162e Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Thu, 30 Jul 2026 23:59:15 +0800 Subject: [PATCH 23/34] RQD-14: pin backend for __fread_chk binding. Co-authored-by: Cursor --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 10f0f328d4c6..0034159f46df 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -c2fa6c44cf5497cf1e83babda8b7a8a53b09e229 +78366fed56f7f43c00d17606df2ff83be42f7703 From 6ef9428369994cfdb510aac58f90b1a05da6acfe Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 00:41:22 +0800 Subject: [PATCH 24/34] RQD-14: pin backend after deploy-agent driver builds. Co-authored-by: Cursor --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 0034159f46df..dfa1039ac002 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -78366fed56f7f43c00d17606df2ff83be42f7703 +97130a07511fdd74d48eb645eb38afc331bbf1b5 From 4d8bf14db1a8fc738918269da8d689b596e0a6c4 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 01:16:30 +0800 Subject: [PATCH 25/34] RQD-14: seed immer Jest repo for test-jest; pin backend. --- .buildkite/runtime-node-build-and-test.yml | 4 ++++ REPLAY_BACKEND_REV | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 42dbdaf4cc93..8ec6fd07b7a2 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -142,6 +142,10 @@ steps: git -C "$$BACKEND_DIR" checkout FETCH_HEAD buildkite-agent artifact download build_id "$$WORKDIR/" BUILD_ID="$$(cat "$$WORKDIR/build_id")" + JEST_DIR="$$WORKDIR/jest/immer" + mkdir -p "$$WORKDIR/jest" + git clone --depth 1 https://github.com/immerjs/immer.git "$$JEST_DIR" + npm --prefix "$$JEST_DIR" install --prefer-offline --progress=false cd "$$BACKEND_DIR" npm ci --prefer-offline --progress=false npx tsx src/build/buildNode.ts test-jest --build-id "$$BUILD_ID" diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index dfa1039ac002..c57fb60ad69d 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -97130a07511fdd74d48eb645eb38afc331bbf1b5 +4816260c496e3653106b2b14c6d4d48a121662df From 156e4258e6951d78ccdf1102f5db2d947e3ec565 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 01:22:09 +0800 Subject: [PATCH 26/34] RQD-14: pin backend for napi_create_array callback. --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index c57fb60ad69d..3569ea10c463 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -4816260c496e3653106b2b14c6d4d48a121662df +721d6444dc51e9afce7c90c10152038ac8f9fae9 From b984b5fa09da9a58c1ad314b5afffb29c9cad094 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 02:31:09 +0800 Subject: [PATCH 27/34] RQD-14: pin backend for unimplemented NAPI callback specs. --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 3569ea10c463..56782a18244c 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -721d6444dc51e9afce7c90c10152038ac8f9fae9 +b8e3af9be6447540c1a49b4bf40bd6d87765f9ce From 39c4df1f46fead3665e95aca29a1c3a298a610e8 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 02:51:17 +0800 Subject: [PATCH 28/34] RQD-14: pin backend for dl_iterate_phdr NoOp binding. --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 56782a18244c..653b54bb4160 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -b8e3af9be6447540c1a49b4bf40bd6d87765f9ce +2c90bc6b4883c43ba4cb03d9b5f155f38e890200 From 78de73593d121c81cf877c417ddad110ab703ca1 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 03:08:11 +0800 Subject: [PATCH 29/34] RQD-14: immer npm install --legacy-peer-deps for test-jest. --- .buildkite/runtime-node-build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 8ec6fd07b7a2..15a841cd8523 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -145,7 +145,7 @@ steps: JEST_DIR="$$WORKDIR/jest/immer" mkdir -p "$$WORKDIR/jest" git clone --depth 1 https://github.com/immerjs/immer.git "$$JEST_DIR" - npm --prefix "$$JEST_DIR" install --prefer-offline --progress=false + npm --prefix "$$JEST_DIR" install --legacy-peer-deps --prefer-offline --progress=false cd "$$BACKEND_DIR" npm ci --prefer-offline --progress=false npx tsx src/build/buildNode.ts test-jest --build-id "$$BUILD_ID" From af0343f27c7caa22e9a4038e6ff90ee4723ade51 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 03:16:09 +0800 Subject: [PATCH 30/34] RQD-14: drop prefer-offline for immer npm install. BK agent cache returns ETARGET for @types/node@24.3.1 which exists on the registry. --- .buildkite/runtime-node-build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 15a841cd8523..1c8b3d663ed3 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -145,7 +145,7 @@ steps: JEST_DIR="$$WORKDIR/jest/immer" mkdir -p "$$WORKDIR/jest" git clone --depth 1 https://github.com/immerjs/immer.git "$$JEST_DIR" - npm --prefix "$$JEST_DIR" install --legacy-peer-deps --prefer-offline --progress=false + npm --prefix "$$JEST_DIR" install --legacy-peer-deps --progress=false cd "$$BACKEND_DIR" npm ci --prefer-offline --progress=false npx tsx src/build/buildNode.ts test-jest --build-id "$$BUILD_ID" From 08b79c3c6bdc8c417efb55f69884fb20321f62ec Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 04:14:07 +0800 Subject: [PATCH 31/34] RQD-16: pin immer v9.0.12 and yarn-install for test-jest seed. Floating main is vitest-era and never produces node_modules/.bin/jest. Co-authored-by: Cursor --- .buildkite/runtime-node-build-and-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/runtime-node-build-and-test.yml b/.buildkite/runtime-node-build-and-test.yml index 1c8b3d663ed3..f9d9b2291f28 100644 --- a/.buildkite/runtime-node-build-and-test.yml +++ b/.buildkite/runtime-node-build-and-test.yml @@ -142,10 +142,12 @@ steps: git -C "$$BACKEND_DIR" checkout FETCH_HEAD buildkite-agent artifact download build_id "$$WORKDIR/" BUILD_ID="$$(cat "$$WORKDIR/build_id")" + # Jest-era pin (harness last touch ~2022-02); immer main is vitest and has no .bin/jest. JEST_DIR="$$WORKDIR/jest/immer" mkdir -p "$$WORKDIR/jest" - git clone --depth 1 https://github.com/immerjs/immer.git "$$JEST_DIR" - npm --prefix "$$JEST_DIR" install --legacy-peer-deps --progress=false + git clone --depth 1 --branch v9.0.12 https://github.com/immerjs/immer.git "$$JEST_DIR" + yarn --cwd "$$JEST_DIR" install --frozen-lockfile --non-interactive + test -x "$$JEST_DIR/node_modules/.bin/jest" cd "$$BACKEND_DIR" npm ci --prefer-offline --progress=false npx tsx src/build/buildNode.ts test-jest --build-id "$$BUILD_ID" From 039f4883b3d76ed57222980cae95dfc46d982c64 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 04:15:46 +0800 Subject: [PATCH 32/34] RQD-14: pin REPLAY_BACKEND_REV to backend master after #13123. --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 653b54bb4160..7839267d8b7f 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -2c90bc6b4883c43ba4cb03d9b5f155f38e890200 +aced6456f036a725dfc7bb9336b370318ba48a43 From bbadd0ff552862f5173b26cb860083ce61a79a20 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 04:25:55 +0800 Subject: [PATCH 33/34] RQD-14: re-pin REPLAY_BACKEND_REV to current backend master. --- REPLAY_BACKEND_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPLAY_BACKEND_REV b/REPLAY_BACKEND_REV index 7839267d8b7f..56fe7cf66217 100644 --- a/REPLAY_BACKEND_REV +++ b/REPLAY_BACKEND_REV @@ -1 +1 @@ -aced6456f036a725dfc7bb9336b370318ba48a43 +2ec2e9b74d6fa84364667c4371ccf28e367e83e8 From a74216e1c06ef972e968dfb54b360c5d57907fc8 Mon Sep 17 00:00:00 2001 From: "D. Seifert" Date: Fri, 31 Jul 2026 04:28:02 +0800 Subject: [PATCH 34/34] RQD-14: disable legacy GHA Build/Test on PR/push. Replaced by Buildkite runtime-node-build-and-test; keep workflow_dispatch only. --- .github/workflows/build-test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index aa7dc83f9f78..c4645216d542 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,3 +1,5 @@ +# Disabled: replaced by Buildkite `runtime-node-build-and-test` (RQD-14). +# Legacy ELB Build/Test trigger retained for manual dispatch only. name: Build/Test on: workflow_dispatch: @@ -6,10 +8,6 @@ on: description: "Build node from scratch" type: boolean required: false - pull_request: - push: - branches: - - master jobs: build-test: