RQD-14: Buildkite Migration + test fixes + -dev buildId suffix

.buildkite/runtime-node-build-and-test.yml

@@ -0,0 +1,153 @@
+# NodeBKWiring build/test steps (PipelineYamlHome).
+# Emitted by replay_build_scripts/build-pipeline.py (bakes pin SHA from
+# node REPLAY_BACKEND_REV into the PIN_TOKEN placeholders).
+# ${BUILDKITE_*} remain for Buildkite interpolation at PipelineUpload.
+#
+# Scratch dirs: mktemp under /tmp (BK agents are partially stateful).
+# Do not parallel-clone node into a BK artifact — different agents cannot share
+# disk; tar upload/download of the tree is pure waste. build_node_builder clones.
+#
+# NodeRepo: host checkout+out mounted over the image node path so make can
+# incremental-rebuild across jobs on the same agent. Mounting only out/ is not
+# enough — a fresh image COPY of sources resets mtimes and forces a full rebuild.
+# Do not `git fetch --depth 1` the cached NodeRepo — shallow fetch rewrites the
+# tree and bumps gyp input mtimes → make "Regenerating Makefile" → near-full rebuild.
+# reclaim_node_repo chown -R bumps ctime on every file; with default git stat
+# checks that dirties the whole tree and `checkout -f` rewrites all sources.
+# core.trustctime=false + update-index --refresh after chown keeps mtimes.
+steps:
+  - trigger: "build-driver-linker"
+    key: "build-driver-linker"
+    build:
+      commit: "__REPLAY_BACKEND_REV__"
+      message: "Triggered from node: ${BUILDKITE_MESSAGE}"
+
+  # Plain command — do not use replay-deploy-buildkite-plugin (that emits
+  # "Deploy to <branch> environment" / preprod steps; wrong for PR CI).
+  - label: ":bust_in_silhouette: Build node"
+    key: "build-node-linux"
+    # BK default is 60m; cold node make -j4 exceeds that (see build #12).
+    timeout_in_minutes: 120
+    depends_on:
+      - "build-driver-linker"
+    agents:
+      - "deploy=true"
+      - "size=large"
+    command: |
+      set -euo pipefail
+      export DOCKER_BUILDKIT=1
+      echo "=== Disk space (build start) ==="
+      df -h
+      NODE_REPO="/var/lib/buildkite-agent/cache/runtime-node-build/repo"
+      if [ -d "$$NODE_REPO" ]; then
+        du -sh "$$NODE_REPO" "$$NODE_REPO/out" 2>/dev/null || du -sh "$$NODE_REPO" 2>/dev/null || true
+      fi
+      PIN="__REPLAY_BACKEND_REV__"
+      WORKDIR=$$(mktemp -d "/tmp/build-node-$${BUILDKITE_JOB_ID}.XXXXXX")
+      trap 'rm -rf "$$WORKDIR"' EXIT
+      BACKEND_DIR="$$WORKDIR/backend"
+      mkdir -p "$$BACKEND_DIR"
+      git -C "$$BACKEND_DIR" init
+      git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git
+      git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN"
+      git -C "$$BACKEND_DIR" checkout FETCH_HEAD
+      cd "$$BACKEND_DIR"
+      BUILDKITE_DOCKER_ENVS=$$(env | awk -F= '/^BUILDKITE/{printf "-e %s ", $$1}')
+      BUILDKITE_AGENT_BIN="$$(command -v buildkite-agent)"
+      REPLAYIO_NODE_REF="$$BUILDKITE_COMMIT" ./scripts/docker/build_node_builder
+      mkdir -p "$$NODE_REPO"
+      reclaim_node_repo() {
+        docker run --rm \
+          -v "$$NODE_REPO:/repo" \
+          --entrypoint chown \
+          linux-node-build \
+          -R "$$(id -u):$$(id -g)" /repo
+      }
+      reclaim_node_repo
+      trap 'reclaim_node_repo; rm -rf "$$WORKDIR"' EXIT
+      if [ ! -d "$$NODE_REPO/.git" ]; then
+        git clone git@github.com:replayio/node.git "$$NODE_REPO"
+      elif [ -f "$$NODE_REPO/.git/shallow" ]; then
+        # Prior jobs used --depth 1; unshallow once so later checkouts keep mtimes.
+        git -C "$$NODE_REPO" fetch --unshallow origin || git -C "$$NODE_REPO" fetch --deepen=2147483647 origin
+      fi
+      # chown -R (reclaim) bumps ctime; without this, checkout -f rewrites all files.
+      git -C "$$NODE_REPO" config core.trustctime false
+      git -C "$$NODE_REPO" update-index --refresh || true
+      git -C "$$NODE_REPO" fetch origin "$$BUILDKITE_COMMIT"
+      echo "=== NodeRepo sync (HEAD -> $$BUILDKITE_COMMIT) ==="
+      echo "HEAD=$$(git -C "$$NODE_REPO" rev-parse HEAD 2>/dev/null || echo none)"
+      echo "dirty=$$(git -C "$$NODE_REPO" status --porcelain 2>/dev/null | wc -l)"
+      echo "diff_files=$$(git -C "$$NODE_REPO" diff --name-only HEAD "$$BUILDKITE_COMMIT" 2>/dev/null | wc -l)"
+      git -C "$$NODE_REPO" checkout -f --detach "$$BUILDKITE_COMMIT"
+      # Match build-node.yml: do not inject prod/deploy-access-keys (github-backend)
+      # into the container — that IAM user cannot s3:PutObject builds/. beefmaster
+      # docker uses the agent identity (same as the manual build-node pipeline).
+      # safe.directory is set inside the image (run_node_build.sh / buildNodeDirectly).
+      docker run --rm \
+        $$BUILDKITE_DOCKER_ENVS \
+        -v "$$BUILDKITE_AGENT_BIN:/usr/local/bin/buildkite-agent:ro" \
+        -v "$$NODE_REPO:/usr/build/tmp/replayio-node-repo" \
+        linux-node-build
+
+  - label: "test-suite"
+    key: "node-test-suite"
+    depends_on:
+      - "build-node-linux"
+    agents:
+      - "deploy=true"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          region: us-east-2
+          env:
+            HASURA_ADMIN_SECRET: "prod/hasura-admin-secret"
+    command: |
+      set -euo pipefail
+      PIN="__REPLAY_BACKEND_REV__"
+      WORKDIR=$$(mktemp -d "/tmp/node-test-suite-$${BUILDKITE_JOB_ID}.XXXXXX")
+      trap 'rm -rf "$$WORKDIR"' EXIT
+      BACKEND_DIR="$$WORKDIR/backend"
+      mkdir -p "$$BACKEND_DIR"
+      git -C "$$BACKEND_DIR" init
+      git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git
+      git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN"
+      git -C "$$BACKEND_DIR" checkout FETCH_HEAD
+      buildkite-agent artifact download build_id "$$WORKDIR/"
+      BUILD_ID="$$(cat "$$WORKDIR/build_id")"
+      cd "$$BACKEND_DIR"
+      npm ci --prefer-offline --progress=false
+      npx tsx src/build/buildNode.ts test-suite --build-id "$$BUILD_ID"
+
+  - label: "test-jest"
+    key: "node-test-jest"
+    depends_on:
+      - "build-node-linux"
+    agents:
+      - "deploy=true"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          region: us-east-2
+          env:
+            HASURA_ADMIN_SECRET: "prod/hasura-admin-secret"
+    command: |
+      set -euo pipefail
+      PIN="__REPLAY_BACKEND_REV__"
+      WORKDIR=$$(mktemp -d "/tmp/node-test-jest-$${BUILDKITE_JOB_ID}.XXXXXX")
+      trap 'rm -rf "$$WORKDIR"' EXIT
+      BACKEND_DIR="$$WORKDIR/backend"
+      mkdir -p "$$BACKEND_DIR"
+      git -C "$$BACKEND_DIR" init
+      git -C "$$BACKEND_DIR" remote add origin git@github.com:replayio/backend.git
+      git -C "$$BACKEND_DIR" fetch --depth 1 origin "$$PIN"
+      git -C "$$BACKEND_DIR" checkout FETCH_HEAD
+      buildkite-agent artifact download build_id "$$WORKDIR/"
+      BUILD_ID="$$(cat "$$WORKDIR/build_id")"
+      # Jest-era pin (harness last touch ~2022-02); immer main is vitest and has no .bin/jest.
+      JEST_DIR="$$WORKDIR/jest/immer"
+      mkdir -p "$$WORKDIR/jest"
+      git clone --depth 1 --branch v9.0.12 https://github.com/immerjs/immer.git "$$JEST_DIR"
+      yarn --cwd "$$JEST_DIR" install --frozen-lockfile --non-interactive
+      test -x "$$JEST_DIR/node_modules/.bin/jest"
+      cd "$$BACKEND_DIR"
+      npm ci --prefer-offline --progress=false
+      npx tsx src/build/buildNode.ts test-jest --build-id "$$BUILD_ID"

.github/workflows/build-test.yml

@@ -1,3 +1,5 @@
+# Disabled: replaced by Buildkite `runtime-node-build-and-test` (RQD-14).
+# Legacy ELB Build/Test trigger retained for manual dispatch only.
 name: Build/Test
 on:
   workflow_dispatch:
@@ -6,10 +8,6 @@ on:
         description: "Build node from scratch"
         type: boolean
         required: false
-  pull_request:
-  push:
-    branches:
-      - master
 
 jobs:
   build-test:

.gitignore

@@ -17,6 +17,7 @@
 !.flake8
 !.gitattributes
 !.github
+!.buildkite
 !.gitignore
 !.gitkeep
 !.mailmap

REPLAY_BACKEND_REV

@@ -1 +1 @@
-d887c4838d6a4e305f874edd8cf3d422a068d9a2
+2ec2e9b74d6fa84364667c4371ccf28e367e83e8

build.js

@@ -25,8 +25,9 @@ if (localDriverDir) {
   let driverArchive = `${currentPlatform()}-recordreplay.tgz`;
   let downloadDriverRevision = process.env.DRIVER_REVISION ? process.env.DRIVER_REVISION : fs.readFileSync("REPLAY_BACKEND_REV", "utf8");
   let downloadArchive = `${currentPlatform()}-recordreplay-${downloadDriverRevision.trim().substring(0, 12)}.tgz`;
+  let downloadUrl = `https://static.replay.io/downloads/${downloadArchive}`;
   const driverArchivePath = path.join(OutDir, driverArchive);
-  spawnChecked("curl", [`https://static.replay.io/downloads/${downloadArchive}`, "-o", driverArchivePath], { stdio: "inherit" });
+  downloadDriverArchive(downloadUrl, driverArchivePath);
   spawnChecked("tar", ["xf", driverArchivePath, "-C", OutDir]);
   fs.unlinkSync(driverArchivePath);
 
@@ -82,6 +83,24 @@ spawnChecked("make", [`-j${numCPUs}`, "-C", OutDir, "BUILDTYPE=Release"], {
   },
 });
 
+function downloadDriverArchive(downloadUrl, driverArchivePath) {
+  curl(downloadUrl, driverArchivePath);
+}
+
+function curl(url, outputPath) {
+  const prettyCmd = ["curl", "--fail", url, "-o", outputPath].join(" ");
+  console.error(prettyCmd);
+
+  const rv = spawnSync("curl", ["--fail", url, "-o", outputPath], {
+    stdio: "inherit",
+  });
+
+  if (rv.status != 0 || rv.error) {
+    console.error(rv.error);
+    throw new Error(`Target driver/linker was not found: ${url}`);
+  }
+}
+
 function spawnChecked(cmd, args, options) {
   const prettyCmd = [cmd].concat(args).join(" ");
   console.error(prettyCmd);
@@ -142,5 +161,11 @@ function computeBuildId() {
   // Use the later of the two dates in the build ID.
   const date = +runtimeDate >= +driverDate ? runtimeDate : driverDate;
 
-  return `${currentPlatform()}-node-${date}-${runtimeRevision}-${driverRevision}`;
+  // Chromium twin: upload_build_artifacts.mjs buildIdExtension / backend utils.ts.
+  const buildIdExtension =
+    process.env.BUILDKITE_BRANCH !== process.env.BUILDKITE_PIPELINE_DEFAULT_BRANCH
+      ? "-dev"
+      : process.env.LOCAL_DEVELOPER_BUILD_EXTENSION || "";
+
+  return `${currentPlatform()}-node-${date}-${runtimeRevision}-${driverRevision}${buildIdExtension}`;
 }

replay_build_scripts/build-pipeline.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""PipelineUpload for runtime-node-build-and-test (Chromium twin of build-pipeline.py).
+
+Bakes REPLAY_BACKEND_REV into .buildkite/runtime-node-build-and-test.yml and
+prints the graph for `buildkite-agent pipeline upload`.
+
+DriverBuildCheck: if linux-recordreplay-<rev12>.tgz is already on S3, replace the
+build-driver-linker trigger with a noop (same key) so depends_on stays valid.
+
+In CI (skip-checkout), fetches pin + YAML from public raw.githubusercontent.com
+using BUILDKITE_COMMIT. Locally, reads from the node checkout.
+
+Do not curl private replayio/backend for pipeline YAML.
+"""
+
+import os
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+PIN_TOKEN = "__REPLAY_BACKEND_REV__"
+REPO = "replayio/node"
+
+
+def read_commit_hash(text: str) -> str:
+    return text.strip().split()[0]
+
+
+def fetch_raw(path: str) -> str:
+    commit = os.environ["BUILDKITE_COMMIT"]
+    url = f"https://raw.githubusercontent.com/{REPO}/{commit}/{path}"
+    with urllib.request.urlopen(url) as resp:
+        return resp.read().decode()
+
+
+def read_text(rel_path: str) -> str:
+    local = Path(__file__).resolve().parent.parent / rel_path
+    if local.is_file():
+        return local.read_text()
+    return fetch_raw(rel_path)
+
+
+def driver_archive_present(driver_revision: str) -> bool:
+    """DriverBuildCheck: True if linux driver archive for this rev is already on S3."""
+    url = f"https://static.replay.io/downloads/linux-recordreplay-{driver_revision}.tgz"
+    req = urllib.request.Request(url, method="HEAD")
+    try:
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            return 200 <= resp.status < 300
+    except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError):
+        return False
+
+
+def apply_driver_build_check(yaml_text: str, pin: str) -> str:
+    """Replace build-driver-linker trigger with noop when archive already on S3."""
+    driver_revision = pin[:12]
+    if not driver_archive_present(driver_revision):
+        return yaml_text
+
+    trigger = f"""  - trigger: "build-driver-linker"
+    key: "build-driver-linker"
+    build:
+      commit: "{pin}"
+      message: "Triggered from node: ${{BUILDKITE_MESSAGE}}"
+"""
+    noop = f"""  - label: "DriverBuildCheck (archive present)"
+    key: "build-driver-linker"
+    agents:
+      - "deploy=true"
+    plugins:
+      - thedyrt/skip-checkout#v0.1.1: ~
+    command: "echo DriverBuildCheck linux-recordreplay-{driver_revision}.tgz already on S3"
+"""
+    if trigger not in yaml_text:
+        raise RuntimeError("DriverBuildCheck: expected build-driver-linker trigger block missing")
+    return yaml_text.replace(trigger, noop, 1)
+
+
+def main() -> None:
+    pin = read_commit_hash(read_text("REPLAY_BACKEND_REV"))
+    yaml_text = read_text(".buildkite/runtime-node-build-and-test.yml")
+    # Bake pin so upload does not depend on REPLAY_BACKEND_REV in the agent env.
+    # Leave ${BUILDKITE_*} for Buildkite interpolation at upload time.
+    yaml_text = yaml_text.replace(PIN_TOKEN, pin)
+    print(apply_driver_build_check(yaml_text, pin))
+
+
+if __name__ == "__main__":
+    main()

src/crypto/crypto_tls.cc

@@ -613,6 +613,7 @@ void TLSWrap::EncOut() {
   size_t count = arraysize(data);
   write_size_ = NodeBIO::FromBIO(enc_out_)->PeekMultiple(data, size, &count);
   CHECK(write_size_ != 0 && count != 0);
+  v8::recordreplay::Assert("TLSWrap::EncOut %zu %zu", count, write_size_);
 
   uv_buf_t buf[arraysize(data)];
   uv_buf_t* bufs = buf;

src/stream_base-inl.h

@@ -165,6 +165,8 @@ StreamWriteResult StreamBase::Write(
     size_t count,
     uv_stream_t* send_handle,
     v8::Local<v8::Object> req_wrap_obj) {
+  v8::recordreplay::Assert("StreamBase::Write %zu", count);
+
   Environment* env = stream_env();
   int err;