From 0e5925d52288f46f62bccb20495707d93552eee5 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Tue, 29 Sep 2026 19:43:12 +0000 Subject: [PATCH 1/9] fix(ci): reduce redundant builds and bound dependency acquisition --- .github/actions/collect-ci-summary/action.yml | 16 - .github/actions/run-mobile-e2e/action.yml | 111 ++ .github/actions/setup-android/action.yml | 43 +- .github/actions/setup-deno/action.yml | 4 +- .github/actions/setup-maestro/action.yml | 15 - .github/actions/setup-moon/action.yml | 9 +- .../actions/setup-moon/install-pinned-node.sh | 7 +- .../setup-moon/install-pinned-node.test.sh | 6 +- .../setup-moon/install-pinned-toolchain.sh | 11 +- .../install-pinned-toolchain.test.sh | 7 +- .github/actions/setup-node-runtime/action.yml | 4 +- .../actions/setup-npm-publisher/action.yml | 4 +- .../actions/setup-npm-publisher/install.sh | 11 +- .../setup-npm-publisher/install.test.sh | 6 +- .github/actions/setup-rust-tools/action.yml | 18 +- .github/actions/setup-rust/action.yml | 5 + .github/actions/setup-swift/action.yml | 8 +- .../actions/setup-wasix-builder/action.yml | 34 + .github/actions/setup-wasmer-llvm/install.sh | 9 +- .../actions/setup-wasmer-llvm/install.test.sh | 11 +- .github/moon.yml | 14 +- .github/scripts/moon-task-capabilities.mts | 16 +- .../scripts/moon-task-capabilities.test.mts | 2 +- .github/scripts/release-candidate-lib.mts | 93 +- .../resolve-planned-moon-execution.mts | 22 +- .../resolve-planned-moon-execution.test.mts | 17 +- .../resolve-planned-moon-execution.test.sh | 54 +- .github/scripts/run-moon-targets.sh | 4 + .github/scripts/run-planned-moon-job.sh | 8 +- .github/scripts/verify-release-candidate.mts | 51 +- .github/scripts/write-release-candidate.mts | 25 +- .github/workflows/broker-runtime.yml | 7 +- .github/workflows/ci.yml | 787 ++++++----- .../workflows/extension-artifacts-native.yml | 92 +- .../workflows/liboliphaunt-native-desktop.yml | 7 +- .github/workflows/mobile-e2e.yml | 100 +- .../workflows/mobile-extension-packages.yml | 8 +- .github/workflows/release.yml | 25 + .github/workflows/wasix-host.yml | 183 +++ src/benchmarks/perf/moon.yml | 1 + src/benchmarks/wasix/README.md | 4 +- .../CI_RELEASE_PROCESS_AUDIT_2026-09-29.md | 1153 +++++++++++++++++ src/docs/maintainers/assets.md | 4 +- src/docs/maintainers/release.md | 17 +- src/docs/maintainers/testing.md | 137 +- src/examples/moon.yml | 6 - .../maestro/installed-smoke.yaml | 15 - src/extensions/artifacts/native/moon.yml | 4 +- .../native/tools/package-release-assets.sh | 24 +- .../tools/package-release-assets.test.sh | 43 + src/extensions/artifacts/wasix/moon.yml | 2 +- src/extensions/evidence/matrix.toml | 78 +- .../generated/docs/extension-evidence.json | 234 ++-- src/extensions/tests/native/moon.yml | 1 + ...tive-extension-lifecycle-receipts.test.mts | 153 ++- ...fy-native-extension-lifecycle-receipts.mts | 32 +- .../tools/collect-wasix-evidence.sh | 2 +- src/extensions/tools/extension-evidence.mts | 2 +- src/extensions/tools/extension-model.test.mts | 4 + src/native/mobile-bindings/moon.yml | 10 +- .../bin/build-postgres18-android-arm64.sh | 1 + .../bin/build-postgres18-ios-device.sh | 1 + .../bin/build-postgres18-ios-simulator.sh | 1 + .../runtime/bin/build-postgres18-linux.sh | 1 + .../runtime/bin/build-postgres18-macos.sh | 1 + .../runtime/bin/build-postgres18-windows.sh | 1 + .../bin/check-postgres18-ios-simulator.sh | 1 + src/native/runtime/moon.yml | 1 + src/native/runtime/tools/runtime-preflight.sh | 15 +- src/native/sdks/kotlin/moon.yml | 25 +- .../sdks/kotlin/tools/test-native-bindings.sh | 12 +- .../react-native/tools/check-package.test.mts | 50 + .../react-native/tools/expo-android-runner.sh | 31 +- .../react-native/tools/expo-ios-runner.sh | 10 +- .../tools/expo-runner-android-device.sh | 132 +- .../tools/expo-runner-android-device.test.sh | 246 ++-- .../react-native/tools/expo-runner-common.sh | 42 +- .../tools/expo-runner-ios-installed-app.sh | 214 +-- .../expo-runner-ios-installed-app.test.sh | 46 +- .../tools/expo-runner-reporting.mts | 21 - .../tools/expo-runner-reporting.sh | 15 +- .../sdks/react-native/tools/mobile-e2e.sh | 2 - .../tools/stage-release-artifacts.mts | 4 +- src/native/sdks/rust/moon.yml | 25 +- src/native/sdks/swift/moon.yml | 25 +- .../sdks/swift/tools/prepare-bindings.sh | 1 - src/native/sdks/swift/tools/swift.sh | 1 + src/native/sdks/tests/README.md | 2 +- src/native/sdks/tests/with-runtime.sh | 49 + src/native/sdks/tests/with-runtime.test.sh | 55 + src/third-party/postgres/fetch-source.sh | 11 +- src/third-party/postgres/fetch-source.test.sh | 14 +- src/third-party/tools/fetch-sources.sh | 54 +- src/third-party/tools/moon.yml | 7 + src/third-party/tools/source-fetch-core.mts | 3 +- .../tools/source-fetch-core.test.mts | 6 + .../tools/source-fetch-core.test.sh | 41 +- .../tools/source-fetch-transport.test.sh | 8 +- src/wasix/browser-host/build-provenance.mts | 5 + src/wasix/browser-host/build-sdk.sh | 62 +- src/wasix/browser-host/moon.yml | 3 + src/wasix/postgres-tools/moon.yml | 4 +- .../postgres-tools/ts/tests/smoke-host.sh | 1 - .../postmaster/bin/package-release-assets.sh | 2 +- src/wasix/postmaster/bin/prepare-baseline.sh | 1 + src/wasix/postmaster/lib/common.sh | 15 +- src/wasix/postmaster/lib/common.test.sh | 25 +- src/wasix/postmaster/moon.yml | 50 +- src/wasix/postmaster/wasmer/tests.sh | 46 +- .../runtime/assets/build/docker/Dockerfile | 9 +- .../build/docker/Dockerfile.dockerignore | 12 + .../docker/install-pinned-apt-packages.sh | 13 +- .../install-pinned-apt-packages.test.sh | 18 +- .../build/docker/install-pinned-wasixcc.sh | 12 +- .../docker/install-pinned-wasixcc.test.sh | 9 +- .../assets/build/docker_contrib_extensions.sh | 2 +- .../runtime/assets/build/docker_initdb.sh | 2 +- .../runtime/assets/build/docker_oliphaunt.sh | 2 +- .../runtime/assets/build/docker_pgdump.sh | 2 +- .../assets/build/docker_pgxs_extensions.sh | 2 +- src/wasix/runtime/assets/build/docker_psql.sh | 2 +- .../assets/build/docker_runtime_support.sh | 2 +- .../assets/build/prepare_postgres_source.sh | 1 + .../runtime/assets/build/wasix_third_party.sh | 2 +- src/wasix/runtime/moon.yml | 20 + .../runtime/tools/build-compiler-output.sh | 27 +- .../tools/build-runtime-portable.test.sh | 20 + src/wasix/sdks/rust/moon.yml | 9 +- src/wasix/sdks/rust/tools/test-resources.sh | 12 +- src/wasix/sdks/ts/moon.yml | 16 +- .../ts/tools/integration/smoke-browser.sh | 1 - .../sdks/ts/tools/integration/smoke-node.sh | 1 - tools/ci/capture-ci-test-observations.sh | 12 +- tools/ci/capture-ci-test-observations.test.sh | 18 + tools/ci/check-workflows.sh | 2 + tools/ci/ci-plan-node-products.test.mts | 137 +- tools/ci/ci-plan-test-inputs.mts | 11 +- .../ci-plan-wasix-postmaster-release.test.mts | 15 +- tools/ci/ci-plan.sh | 19 + tools/ci/ci-release-scope.test.sh | 54 +- tools/ci/ci_plan.mts | 83 +- tools/ci/moon.yml | 2 +- tools/ci/start-android-emulator-ci.sh | 7 +- tools/ci/workflow-caches.test.mts | 155 +++ tools/ci/workflow-moon-transfers.test.mts | 140 +- tools/dev/acquisition.sh | 94 ++ tools/dev/acquisition.test.sh | 96 ++ tools/dev/android-sdk.toml | 2 + tools/dev/extract-maestro.mts | 27 - tools/dev/install-pinned-js-runtime.sh | 10 +- tools/dev/install-pinned-js-runtime.test.sh | 4 + tools/dev/install-pinned-maintainer-tool.sh | 9 +- tools/dev/install-pinned-winflexbison.sh | 7 +- tools/dev/install-pinned-winflexbison.test.sh | 6 +- tools/dev/maestro.toml | 15 - tools/dev/maintainer-tool-install.test.sh | 4 + tools/dev/moon.yml | 10 +- tools/dev/setup-android-sdk.sh | 91 +- tools/dev/setup-android-sdk.test.sh | 45 +- tools/dev/setup-maestro.sh | 194 --- tools/dev/setup-maestro.test.sh | 328 ----- tools/release/public-consumer-smoke.test.mts | 2 +- tools/release/public-consumer-smoke.test.sh | 17 +- tools/release/release_plan.mts | 3 + 164 files changed, 4752 insertions(+), 2232 deletions(-) delete mode 100644 .github/actions/collect-ci-summary/action.yml create mode 100644 .github/actions/run-mobile-e2e/action.yml delete mode 100644 .github/actions/setup-maestro/action.yml create mode 100644 .github/actions/setup-wasix-builder/action.yml create mode 100644 .github/workflows/wasix-host.yml create mode 100644 src/docs/internal/CI_RELEASE_PROCESS_AUDIT_2026-09-29.md delete mode 100644 src/examples/native/react-native-expo/maestro/installed-smoke.yaml create mode 100644 src/extensions/artifacts/native/tools/package-release-assets.test.sh create mode 100644 src/native/sdks/tests/with-runtime.sh create mode 100644 src/native/sdks/tests/with-runtime.test.sh create mode 100644 src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore create mode 100644 tools/ci/capture-ci-test-observations.test.sh create mode 100644 tools/ci/workflow-caches.test.mts create mode 100644 tools/dev/acquisition.sh create mode 100644 tools/dev/acquisition.test.sh delete mode 100644 tools/dev/extract-maestro.mts delete mode 100644 tools/dev/maestro.toml delete mode 100755 tools/dev/setup-maestro.sh delete mode 100755 tools/dev/setup-maestro.test.sh diff --git a/.github/actions/collect-ci-summary/action.yml b/.github/actions/collect-ci-summary/action.yml deleted file mode 100644 index 0fcacf360..000000000 --- a/.github/actions/collect-ci-summary/action.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Collect CI summary -description: Append a small Moon/release summary to the GitHub step summary. - -runs: - using: composite - steps: - - name: Write CI summary - shell: bash - run: | - { - echo "## Oliphaunt CI" - echo - echo "- Moon projects: \`moon query projects\`" - echo "- Moon tasks: \`moon query tasks\`" - echo "- Release plan: \`bash tools/release/release-plan.sh --from-product-tags --head-ref \`" - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/run-mobile-e2e/action.yml b/.github/actions/run-mobile-e2e/action.yml new file mode 100644 index 000000000..8798723bd --- /dev/null +++ b/.github/actions/run-mobile-e2e/action.yml @@ -0,0 +1,111 @@ +name: Run installed mobile app +description: Shared platform setup, exact-launch SDK smoke, and reports for CI and replay. +inputs: + platform: + required: true + description: android or ios + source-sha: + required: true + description: Exact source SHA of the downloaded app +runs: + using: composite + steps: + - name: Set up Node and Bun + uses: ./.github/actions/setup-node-bun + + - name: Reclaim Android emulator disk + if: ${{ inputs.platform == 'android' }} + shell: bash + run: bash .github/scripts/reclaim-android-mobile-build-disk.sh + + - name: Set up Android + if: ${{ inputs.platform == 'android' }} + uses: ./.github/actions/setup-android + with: + gradle-cache: "false" + native-tools: "false" + + - name: List Android app artifact + if: ${{ inputs.platform == 'android' }} + shell: bash + run: find target/mobile-build/react-native/android -maxdepth 2 -type f -print + + - name: Provision runner KVM access + if: ${{ inputs.platform == 'android' }} + shell: bash + run: | + test -c /dev/kvm + if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then + sudo chmod a+rw /dev/kvm + fi + + - name: Start Android emulator + if: ${{ inputs.platform == 'android' }} + env: + OLIPHAUNT_ANDROID_EMULATOR_API: "35" + OLIPHAUNT_ANDROID_EMULATOR_DISK_HEADROOM_MB: "2048" + OLIPHAUNT_ANDROID_EMULATOR_PARTITION_SIZE_MB: "6144" + shell: bash + run: tools/ci/start-android-emulator-ci.sh + + - name: Run Android installed-app E2E + if: ${{ inputs.platform == 'android' }} + env: + CI_HEAD_SHA: ${{ inputs.source-sha }} + OLIPHAUNT_EXPO_ANDROID_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/android + OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE: release + OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE: "0" + shell: bash + run: bash src/native/sdks/react-native/tools/mobile-e2e.sh android + + - name: Upload Android E2E reports + if: ${{ always() && inputs.platform == 'android' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: react-native-mobile-android-e2e-reports + path: | + target/mobile/react-native/android-e2e/reports + target/mobile/react-native/android-e2e/logs + ${{ runner.temp }}/oliphaunt-android-emulator.log + if-no-files-found: ignore + - name: Set up Apple + if: ${{ inputs.platform == 'ios' }} + uses: ./.github/actions/setup-apple + with: + install-build-dependencies: "false" + + - name: Verify and extract iOS app artifact + if: ${{ inputs.platform == 'ios' }} + shell: bash + run: | + rm -rf target/mobile-build/react-native/ios + bash src/native/sdks/react-native/tools/ios-app-transport.sh verify-extract \ + --transport-dir target/mobile-build/react-native/ios-transport \ + --output-dir target/mobile-build/react-native/ios + + - name: List iOS app artifact + if: ${{ inputs.platform == 'ios' }} + shell: bash + run: find target/mobile-build/react-native/ios -maxdepth 2 -print + + - name: Run iOS installed-app E2E + if: ${{ inputs.platform == 'ios' }} + env: + CI_HEAD_SHA: ${{ inputs.source-sha }} + OLIPHAUNT_EXPO_IOS_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/ios + OLIPHAUNT_EXPO_IOS_CONFIGURATION: Release + OLIPHAUNT_EXPO_IOS_SDK: iphonesimulator + OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE: "0" + shell: bash + run: bash src/native/sdks/react-native/tools/mobile-e2e.sh ios + + - name: Upload iOS E2E reports + if: ${{ always() && inputs.platform == 'ios' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: react-native-mobile-ios-e2e-reports + path: | + target/mobile/react-native/ios-e2e/reports + target/mobile/react-native/ios-e2e/logs + target/mobile/react-native/ios-e2e/*.log + if-no-files-found: ignore diff --git a/.github/actions/setup-android/action.yml b/.github/actions/setup-android/action.yml index 1eb37ab36..8d85d3133 100644 --- a/.github/actions/setup-android/action.yml +++ b/.github/actions/setup-android/action.yml @@ -2,18 +2,12 @@ name: Set up Android description: Set up Java and expose Android SDK paths for Gradle/Expo jobs. inputs: - ndk-version: - description: Android NDK side-by-side version required by native SDK builds. - required: false - default: "27.0.12077973" - cmake-version: - description: Android CMake version required by native SDK builds. - required: false - default: "3.22.1" - compile-sdk: - description: Android platform API level used by SDK checks. - required: false - default: "36" + native-tools: + description: Install NDK and CMake for native compilation. + default: "true" + expo: + description: Also provision the Expo app NDK before Gradle runs. + default: "false" native-ccache: description: Whether to install and configure ccache for native Android C/C++ builds. required: false @@ -109,12 +103,21 @@ runs: ccache --set-config=max_size=2G ccache --zero-stats + - name: Set up macOS acquisition timer + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + . tools/dev/acquisition.sh + if ! oliphaunt_acquisition_timeout >/dev/null 2>&1; then + HOMEBREW_NO_AUTO_UPDATE=1 brew install coreutils + fi + oliphaunt_acquisition_timeout >/dev/null + - name: Configure Android SDK shell: bash env: - NDK_VERSION: ${{ inputs.ndk-version }} - CMAKE_VERSION: ${{ inputs.cmake-version }} - COMPILE_SDK: ${{ inputs.compile-sdk }} + NATIVE_TOOLS: ${{ inputs.native-tools }} + EXPO: ${{ inputs.expo }} run: | # zizmor: ignore[github-env] Android SDK paths are runner-owned or HOME-scoped and validated before export. set -euo pipefail if [[ -z "${ANDROID_HOME:-}" ]]; then @@ -128,16 +131,12 @@ runs: export ANDROID_HOME="$HOME/android-sdk" fi fi - tools/dev/setup-android-sdk.sh \ - --sdk-root "$ANDROID_HOME" \ - --ndk-version "$NDK_VERSION" \ - --cmake-version "$CMAKE_VERSION" \ - --compile-sdk "$COMPILE_SDK" + args=(--sdk-root "$ANDROID_HOME" --native-tools "$NATIVE_TOOLS") + case "$EXPO" in true) args+=(--expo) ;; false) ;; *) exit 1 ;; esac + tools/dev/setup-android-sdk.sh "${args[@]}" echo "ANDROID_HOME=${ANDROID_HOME}" >> "$GITHUB_ENV" echo "ANDROID_SDK_ROOT=${ANDROID_HOME}" >> "$GITHUB_ENV" - echo "ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/${NDK_VERSION}" >> "$GITHUB_ENV" echo "${ANDROID_HOME}/cmdline-tools/latest/bin" >> "$GITHUB_PATH" echo "${ANDROID_HOME}/platform-tools" >> "$GITHUB_PATH" echo "ANDROID_HOME=${ANDROID_HOME}" - echo "ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/${NDK_VERSION}" diff --git a/.github/actions/setup-deno/action.yml b/.github/actions/setup-deno/action.yml index dd2097672..57a0fb52a 100644 --- a/.github/actions/setup-deno/action.yml +++ b/.github/actions/setup-deno/action.yml @@ -3,9 +3,9 @@ description: Install the pinned Deno toolchain for TypeScript runtime checks. inputs: deno-version: - description: Deno version. + description: Optional assertion against the repository Deno pin. required: false - default: "v2.8.1" + default: "" outputs: execution-envelope: diff --git a/.github/actions/setup-maestro/action.yml b/.github/actions/setup-maestro/action.yml deleted file mode 100644 index 332bd595c..000000000 --- a/.github/actions/setup-maestro/action.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Set up Maestro -description: Install the pinned open-source Maestro CLI for local emulator/simulator E2E. - -runs: - using: composite - steps: - - name: Set up Java - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a - with: - distribution: temurin - java-version: "17" - - - name: Install Maestro CLI - shell: bash - run: tools/dev/setup-maestro.sh diff --git a/.github/actions/setup-moon/action.yml b/.github/actions/setup-moon/action.yml index 923b8993d..d8f3aafc2 100644 --- a/.github/actions/setup-moon/action.yml +++ b/.github/actions/setup-moon/action.yml @@ -2,6 +2,10 @@ name: Set up Moon description: Install verified Node.js, Moon, and Bun binaries and optionally hydrate JavaScript workspace dependencies. inputs: + task-cache: + description: Restore/save Moon task outputs; disable for planning and uncached finalizers. + required: false + default: "true" install-workspace: description: Install Bun workspace dependencies for JavaScript-family tasks. required: false @@ -17,7 +21,7 @@ runs: path: | ${{ runner.temp }}/oliphaunt-moon-toolchain ${{ runner.temp }}/oliphaunt-pinned-tools - key: verified-moon-toolchain-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.moon/toolchains.yml', '.github/actions/setup-moon/action.yml', '.github/actions/setup-node-bun/action.yml', '.github/actions/setup-moon/install-pinned-toolchain.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/dev/moon-cli.toml', 'tools/dev/moon-plugins.toml', 'tools/dev/proto.toml', 'tools/dev/bun.toml', 'tools/dev/install-pinned-js-runtime.sh', 'tools/dev/extract-pinned-zip.sh', 'tools/packaging/portable-archive.mts', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh') }} + key: verified-moon-toolchain-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.moon/toolchains.yml', '.github/actions/setup-moon/action.yml', '.github/actions/setup-node-bun/action.yml', '.github/actions/setup-moon/install-pinned-toolchain.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/dev/moon-cli.toml', 'tools/dev/moon-plugins.toml', 'tools/dev/proto.toml', 'tools/dev/bun.toml', 'tools/dev/install-pinned-js-runtime.sh', 'tools/dev/extract-pinned-zip.sh', 'tools/packaging/portable-archive.mts', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} - name: Set up exact Node.js and Bun uses: ./.github/actions/setup-node-bun @@ -80,7 +84,7 @@ runs: path: | ${{ runner.temp }}/oliphaunt-moon-toolchain ${{ runner.temp }}/oliphaunt-pinned-tools - key: verified-moon-toolchain-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.moon/toolchains.yml', '.github/actions/setup-moon/action.yml', '.github/actions/setup-node-bun/action.yml', '.github/actions/setup-moon/install-pinned-toolchain.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/dev/moon-cli.toml', 'tools/dev/moon-plugins.toml', 'tools/dev/proto.toml', 'tools/dev/bun.toml', 'tools/dev/install-pinned-js-runtime.sh', 'tools/dev/extract-pinned-zip.sh', 'tools/packaging/portable-archive.mts', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh') }} + key: verified-moon-toolchain-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.moon/toolchains.yml', '.github/actions/setup-moon/action.yml', '.github/actions/setup-node-bun/action.yml', '.github/actions/setup-moon/install-pinned-toolchain.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/dev/moon-cli.toml', 'tools/dev/moon-plugins.toml', 'tools/dev/proto.toml', 'tools/dev/bun.toml', 'tools/dev/install-pinned-js-runtime.sh', 'tools/dev/extract-pinned-zip.sh', 'tools/packaging/portable-archive.mts', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} - name: Install workspace dependencies if: ${{ inputs.install-workspace == 'true' }} @@ -88,6 +92,7 @@ runs: run: bun install --frozen-lockfile - name: Restore Moon task outputs + if: ${{ inputs.task-cache == 'true' }} uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: # casOutputsCache uses these directories; hashes/ is diagnostic metadata. diff --git a/.github/actions/setup-moon/install-pinned-node.sh b/.github/actions/setup-moon/install-pinned-node.sh index 7b9be6984..ac04fa343 100755 --- a/.github/actions/setup-moon/install-pinned-node.sh +++ b/.github/actions/setup-moon/install-pinned-node.sh @@ -33,6 +33,8 @@ done # shellcheck source=tools/dev/curl-platform-flags.sh . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "Node.js bootstrap" 300 manifest_value() { @@ -204,15 +206,14 @@ if [ "$archive_valid" != "1" ]; then curl_args=( --fail --location --silent --show-error --proto '=https' --proto-redir '=https' --tlsv1.2 - --retry 5 --retry-all-errors --retry-connrefused --retry-delay 2 --retry-max-time 300 - --connect-timeout 20 --max-time 300 --speed-limit 1024 --speed-time 30 + --connect-timeout 20 --speed-limit 1024 --speed-time 30 --remove-on-error --max-filesize "$archive_bytes" --output "$partial" ) if [ -n "$curl_tls_flag" ]; then curl_args+=("$curl_tls_flag") fi curl_args+=("$url") - if ! "$curl_command" "${curl_args[@]}"; then + if ! oliphaunt_acquisition_curl 300 6 2 "$curl_command" "${curl_args[@]}"; then rm -f "$partial" fail "could not download pinned Node.js archive $url" fi diff --git a/.github/actions/setup-moon/install-pinned-node.test.sh b/.github/actions/setup-moon/install-pinned-node.test.sh index cc226c37d..ce579367c 100755 --- a/.github/actions/setup-moon/install-pinned-node.test.sh +++ b/.github/actions/setup-moon/install-pinned-node.test.sh @@ -4,6 +4,10 @@ set -euo pipefail root="$(git rev-parse --show-toplevel)" installer="$root/.github/actions/setup-moon/install-pinned-node.sh" work="$(mktemp -d)" +mkdir -p "$work/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$work/no-delay/sleep" +chmod +x "$work/no-delay/sleep" +export PATH="$work/no-delay:$PATH" trap 'rm -rf "$work"' EXIT mkdir -p "$work/payload/node-v22.22.3-linux-x64/bin" "$work/bin" @@ -42,7 +46,7 @@ set -euo pipefail output="" last="" joined=" $* " -for required in "--fail" "--location" "--proto =https" "--proto-redir =https" "--tlsv1.2" "--retry-all-errors" "--retry-connrefused" "--remove-on-error" "--max-filesize"; do +for required in "--fail" "--location" "--proto =https" "--proto-redir =https" "--tlsv1.2" "--retry 0" "--remove-on-error" "--max-filesize"; do [[ "$joined" == *" $required "* ]] || { echo "missing hardened curl argument: $required" >&2 exit 91 diff --git a/.github/actions/setup-moon/install-pinned-toolchain.sh b/.github/actions/setup-moon/install-pinned-toolchain.sh index 47b1bc883..9a8fd2a10 100755 --- a/.github/actions/setup-moon/install-pinned-toolchain.sh +++ b/.github/actions/setup-moon/install-pinned-toolchain.sh @@ -42,6 +42,8 @@ done # shellcheck source=tools/dev/curl-platform-flags.sh . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "Moon toolchain and plugins" 900 command -v bun >/dev/null 2>&1 || fail "Bun is required; run setup-node-bun first" @@ -257,8 +259,7 @@ curl_tls_flag="$(oliphaunt_curl_platform_tls_flag)" curl_common=( --fail --location --silent --show-error --proto '=https' --proto-redir '=https' --tlsv1.2 - --retry 6 --retry-all-errors --retry-connrefused --retry-max-time 300 - --connect-timeout 20 --max-time 300 --speed-limit 1024 --speed-time 30 + --connect-timeout 20 --speed-limit 1024 --speed-time 30 --remove-on-error ) if [ -n "$curl_tls_flag" ]; then @@ -287,7 +288,7 @@ download_verified() { args+=(--header "Authorization: Bearer $bearer") fi args+=("$url") - if "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then + if oliphaunt_acquisition_curl 300 7 2 "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then : else rc=$? @@ -317,7 +318,7 @@ registry_token() { --output "$response" "https://ghcr.io/token?scope=repository:$repository:pull" ) - if ! "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then + if ! oliphaunt_acquisition_curl 300 7 2 "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then rm -f "$response" fail "could not obtain a bounded read-only GHCR token for $repository" fi @@ -356,7 +357,7 @@ download_oci_manifest() { --output "$partial" "https://ghcr.io/v2/$repository/manifests/sha256:$digest" ) - if ! "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then + if ! oliphaunt_acquisition_curl 300 7 2 "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then rm -f "$partial" "$headers" fail "could not fetch pinned OCI manifest $repository@sha256:$digest" fi diff --git a/.github/actions/setup-moon/install-pinned-toolchain.test.sh b/.github/actions/setup-moon/install-pinned-toolchain.test.sh index eb1f3656b..f814b2b67 100755 --- a/.github/actions/setup-moon/install-pinned-toolchain.test.sh +++ b/.github/actions/setup-moon/install-pinned-toolchain.test.sh @@ -5,6 +5,10 @@ root="$(git rev-parse --show-toplevel)" installer="$root/.github/actions/setup-moon/install-pinned-toolchain.sh" extractor="$root/.github/actions/setup-moon/toolchain-archive.mts" tmp="$(mktemp -d)" +mkdir -p "$tmp/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$tmp/no-delay/sleep" +chmod +x "$tmp/no-delay/sleep" +export PATH="$tmp/no-delay:$PATH" trap 'rm -rf "$tmp"' EXIT fail() { @@ -27,6 +31,7 @@ mkdir -p \ "$fixture/content" \ "$fixture/blobs" cp "$root/tools/dev/curl-platform-flags.sh" "$fixture/tools/dev/curl-platform-flags.sh" +cp "$root/tools/dev/acquisition.sh" "$fixture/tools/dev/acquisition.sh" moon_version="9.8.7" proto_version="7.6.5" @@ -201,7 +206,7 @@ final="$(bash "$installer")" [ "$(find "$final/plugins" -mindepth 1 -maxdepth 1 | wc -l | tr -d '[:space:]')" = "4" ] || fail "wrong plugin count" [ "$(wc -l <"$FAKE_CURL_LOG" | tr -d '[:space:]')" = "13" ] || fail "unexpected first-install request count" while IFS= read -r call; do - for flag in --ssl-revoke-best-effort --tlsv1.2 --retry-all-errors --retry-connrefused --max-filesize --max-time --speed-limit; do + for flag in --ssl-revoke-best-effort --tlsv1.2 --retry --max-filesize --max-time --speed-limit; do [[ "$call" == *"$flag"* ]] || fail "curl request omitted $flag" done done <"$FAKE_CURL_LOG" diff --git a/.github/actions/setup-node-runtime/action.yml b/.github/actions/setup-node-runtime/action.yml index 52c36fc9d..004b1c9a5 100644 --- a/.github/actions/setup-node-runtime/action.yml +++ b/.github/actions/setup-node-runtime/action.yml @@ -14,7 +14,7 @@ runs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ runner.temp }}/oliphaunt-node-runtime - key: verified-node-runtime-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.github/actions/setup-node-runtime/action.yml', '.github/actions/setup-moon/install-pinned-node.sh', 'tools/dev/node-runtime.toml', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh') }} + key: verified-node-runtime-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.github/actions/setup-node-runtime/action.yml', '.github/actions/setup-moon/install-pinned-node.sh', 'tools/dev/node-runtime.toml', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} - name: Install verified Node.js runtime id: install @@ -41,4 +41,4 @@ runs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ runner.temp }}/oliphaunt-node-runtime - key: verified-node-runtime-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.github/actions/setup-node-runtime/action.yml', '.github/actions/setup-moon/install-pinned-node.sh', 'tools/dev/node-runtime.toml', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh') }} + key: verified-node-runtime-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.github/actions/setup-node-runtime/action.yml', '.github/actions/setup-moon/install-pinned-node.sh', 'tools/dev/node-runtime.toml', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} diff --git a/.github/actions/setup-npm-publisher/action.yml b/.github/actions/setup-npm-publisher/action.yml index a2168f53b..015a7036c 100644 --- a/.github/actions/setup-npm-publisher/action.yml +++ b/.github/actions/setup-npm-publisher/action.yml @@ -25,7 +25,7 @@ runs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ runner.temp }}/oliphaunt-npm-publisher - key: verified-npm-publisher-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-npm-publisher/action.yml', '.github/actions/setup-npm-publisher/install.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/release/npm-publisher.toml', 'tools/packaging/portable-archive.mts', 'tools/dev/curl-platform-flags.sh') }} + key: verified-npm-publisher-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-npm-publisher/action.yml', '.github/actions/setup-npm-publisher/install.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/release/npm-publisher.toml', 'tools/packaging/portable-archive.mts', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} - name: Install exact npm publisher id: install @@ -98,4 +98,4 @@ runs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ runner.temp }}/oliphaunt-npm-publisher - key: verified-npm-publisher-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-npm-publisher/action.yml', '.github/actions/setup-npm-publisher/install.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/release/npm-publisher.toml', 'tools/packaging/portable-archive.mts', 'tools/dev/curl-platform-flags.sh') }} + key: verified-npm-publisher-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-npm-publisher/action.yml', '.github/actions/setup-npm-publisher/install.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/release/npm-publisher.toml', 'tools/packaging/portable-archive.mts', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} diff --git a/.github/actions/setup-npm-publisher/install.sh b/.github/actions/setup-npm-publisher/install.sh index 52432f18c..afefd51b3 100755 --- a/.github/actions/setup-npm-publisher/install.sh +++ b/.github/actions/setup-npm-publisher/install.sh @@ -29,6 +29,8 @@ for path in "$manifest" "$extractor" "$curl_platform_flags"; do done # shellcheck source=tools/dev/curl-platform-flags.sh . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "npm publisher bootstrap" 300 command -v node >/dev/null 2>&1 || fail "the verified Node.js runtime must be on PATH" @@ -196,12 +198,12 @@ if ! { [ -f "$archive" ] && [ ! -L "$archive" ] && partial="$(mktemp "$archive_root/.download.XXXXXX")" tls_flag="$(oliphaunt_curl_platform_tls_flag)" curl_args=(--fail --location --silent --show-error --proto '=https' --proto-redir '=https' - --tlsv1.2 --retry 5 --retry-all-errors --retry-connrefused --retry-delay 2 - --retry-max-time 300 --connect-timeout 20 --max-time 300 --speed-limit 1024 + --tlsv1.2 + --connect-timeout 20 --speed-limit 1024 --speed-time 30 --remove-on-error --max-filesize "$archive_bytes" --output "$partial") [ -z "$tls_flag" ] || curl_args+=("$tls_flag") curl_args+=("$url") - if ! "$curl_command" "${curl_args[@]}"; then + if ! oliphaunt_acquisition_curl 300 6 2 "$curl_command" "${curl_args[@]}"; then rm -f "$partial"; fail "could not download pinned npm publisher archive" fi [ "$(wc -c <"$partial" | tr -d '[:space:]')" = "$archive_bytes" ] && @@ -212,7 +214,6 @@ if ! { [ -f "$archive" ] && [ ! -L "$archive" ] && chmod 0444 "$partial" mv "$partial" "$archive" fi - stage="$(mktemp -d "$install_parent/.verified.stage.XXXXXX")" backup="" old_moved=0 @@ -229,7 +230,6 @@ trap cleanup EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM - extract_args=(extract --archive "$archive" --format "$format" --prefix "$prefix" --entry-count "$entry_count" --expected-bytes "$archive_bytes" --expanded-bytes "$expanded_bytes" --destination "$stage/npm" @@ -248,7 +248,6 @@ chmod 0444 "$stage/bin/npm.cmd" "$stage/bin/npx.cmd" printf '%s\n' "$receipt_text" >"$stage/receipt" chmod 0444 "$stage/receipt" cache_valid "$stage" || fail "staged npm publisher failed integrity or version validation" - if [ -e "$final" ] || [ -L "$final" ]; then backup="$(mktemp -d "$install_parent/.verified.backup.XXXXXX")"; rmdir "$backup" mv "$final" "$backup"; old_moved=1 diff --git a/.github/actions/setup-npm-publisher/install.test.sh b/.github/actions/setup-npm-publisher/install.test.sh index 4f2b07ca6..416cec097 100755 --- a/.github/actions/setup-npm-publisher/install.test.sh +++ b/.github/actions/setup-npm-publisher/install.test.sh @@ -5,6 +5,10 @@ root="$(git rev-parse --show-toplevel)" installer="$root/.github/actions/setup-npm-publisher/install.sh" extractor="$root/.github/actions/setup-moon/toolchain-archive.mts" work="$(mktemp -d)" +mkdir -p "$work/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$work/no-delay/sleep" +chmod +x "$work/no-delay/sleep" +export PATH="$work/no-delay:$PATH" trap 'rm -rf "$work"' EXIT mkdir -p "$work/payload/package/bin" "$work/bin" "$work/blockers" bash "$root/tools/dev/bun.sh" build "$root/.github/actions/setup-npm-publisher/testdata/package-manager-fixture.mts" \ @@ -62,7 +66,7 @@ cat >"$work/bin/curl" <<'EOF' #!/usr/bin/env bash set -euo pipefail joined=" $* " -for required in "--fail" "--location" "--proto =https" "--proto-redir =https" "--tlsv1.2" "--retry-all-errors" "--retry-connrefused" "--remove-on-error" "--max-filesize" "--ssl-revoke-best-effort"; do +for required in "--fail" "--location" "--proto =https" "--proto-redir =https" "--tlsv1.2" "--retry 0" "--remove-on-error" "--max-filesize" "--ssl-revoke-best-effort"; do [[ "$joined" == *" $required "* ]] || exit 91 done output="" diff --git a/.github/actions/setup-rust-tools/action.yml b/.github/actions/setup-rust-tools/action.yml index 3938ad3b2..321bb3cc5 100644 --- a/.github/actions/setup-rust-tools/action.yml +++ b/.github/actions/setup-rust-tools/action.yml @@ -5,7 +5,7 @@ inputs: toolchain: description: Rust toolchain version. required: false - default: "1.93.1" + default: "" components: description: Comma-separated Rust components. required: false @@ -30,10 +30,24 @@ inputs: runs: using: composite steps: + - name: Resolve Rust toolchain + id: toolchain + shell: bash + env: + TOOLCHAIN_INPUT: ${{ inputs.toolchain }} + run: | + set -euo pipefail + version="$TOOLCHAIN_INPUT" + if [[ -z "$version" ]]; then + version="$(sed -n 's/^channel = "\([^"]*\)"/\1/p' rust-toolchain.toml)" + fi + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo 'expected a pinned Rust version' >&2; exit 1; } + echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: - toolchain: ${{ inputs.toolchain }} + toolchain: ${{ steps.toolchain.outputs.version }} components: ${{ inputs.components }} - name: Cache Cargo output diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index e175487e5..2c1463777 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -18,6 +18,10 @@ inputs: description: Whether to enable the Cargo cache. required: false default: "true" + cache-workspaces: + description: Workspace and target-directory mappings for the Cargo cache. + required: false + default: ". -> target" runs: using: composite @@ -27,5 +31,6 @@ runs: with: components: ${{ inputs.components }} cache: ${{ inputs.cache }} + cache-workspaces: ${{ inputs.cache-workspaces }} cache-save-if: ${{ inputs.cache-save-if }} tools: ${{ inputs.tools }} diff --git a/.github/actions/setup-swift/action.yml b/.github/actions/setup-swift/action.yml index e608e31e9..8b13e6dbf 100644 --- a/.github/actions/setup-swift/action.yml +++ b/.github/actions/setup-swift/action.yml @@ -10,7 +10,13 @@ runs: - name: Import Swift signing keys shell: bash # Pin Swift's upstream key bundle instead of relying on the website endpoint. - run: curl --proto '=https' --proto-redir '=https' --tlsv1.2 --fail --silent --show-error --location --retry 3 --connect-timeout 15 --max-time 120 https://raw.githubusercontent.com/swiftlang/swift-org-website/894b4ef6cfd942971e433497feb0458af641ec1e/keys/all-keys.asc | gpg --batch --import - + run: | + . tools/dev/acquisition.sh + oliphaunt_acquisition_start 'Swift signing keys' 120 + keys=$(mktemp) + trap 'rm -f "$keys"' EXIT + oliphaunt_acquisition_curl 120 4 2 curl --proto '=https' --proto-redir '=https' --tlsv1.2 --fail --silent --show-error --location --connect-timeout 15 --output "$keys" https://raw.githubusercontent.com/swiftlang/swift-org-website/894b4ef6cfd942971e433497feb0458af641ec1e/keys/all-keys.asc + gpg --batch --import "$keys" - name: Install Swift uses: swift-actions/setup-swift@d8e84bc3a450686a95474d7d6fa4a3301498debc # v3, Swiftly 1.1.0 with: diff --git a/.github/actions/setup-wasix-builder/action.yml b/.github/actions/setup-wasix-builder/action.yml new file mode 100644 index 000000000..afd6678e1 --- /dev/null +++ b/.github/actions/setup-wasix-builder/action.yml @@ -0,0 +1,34 @@ +name: Set up WASIX builder +description: Load the pinned WASIX Docker builder using the shared build cache. + +inputs: + cache-save-if: + description: Whether this run may save the builder cache. + required: true + +runs: + using: composite + steps: + - name: Identify builder recipe + id: recipe + shell: bash + run: | + source src/wasix/postmaster/lib/common.sh + recipe_sha256="$(fresh_wasix_builder_recipe_sha256)" + printf 'sha256=%s\n' "$recipe_sha256" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd + + - name: Load WASIX builder + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f + with: + context: . + file: src/wasix/runtime/assets/build/docker/Dockerfile + tags: | + oliphaunt-wasix-wasix-build:ci + oliphaunt-wasix-wasix-build:local + labels: dev.oliphaunt.wasix-builder.recipe-sha256=${{ steps.recipe.outputs.sha256 }} + load: true + cache-from: type=gha,scope=wasix-builder + cache-to: ${{ inputs.cache-save-if == 'true' && 'type=gha,mode=max,scope=wasix-builder,ignore-error=true' || '' }} diff --git a/.github/actions/setup-wasmer-llvm/install.sh b/.github/actions/setup-wasmer-llvm/install.sh index 86574ff93..a7656f5a8 100755 --- a/.github/actions/setup-wasmer-llvm/install.sh +++ b/.github/actions/setup-wasmer-llvm/install.sh @@ -19,6 +19,8 @@ if [ ! -f "$curl_platform_flags" ] || [ -L "$curl_platform_flags" ]; then fi # shellcheck source=tools/dev/curl-platform-flags.sh . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "Wasmer LLVM" 1800 if [[ ! "$LLVM_URL" =~ ^https://[^[:space:]]+$ ]]; then echo "Wasmer LLVM URL must be a single HTTPS URL" >&2 @@ -201,12 +203,7 @@ curl_args=( --location --fail --show-error - --retry 4 - --retry-all-errors - --retry-delay 10 - --retry-max-time 3600 --connect-timeout 30 - --max-time 1800 --max-filesize "$LLVM_BYTES" --proto '=https' --proto-redir '=https' @@ -217,7 +214,7 @@ if [ -n "$curl_platform_tls_flag" ]; then curl_args+=("$curl_platform_tls_flag") fi curl_args+=(--output "$archive" "$LLVM_URL") -curl "${curl_args[@]}" +oliphaunt_acquisition_curl 1800 5 10 curl "${curl_args[@]}" actual_bytes="$(wc -c < "$archive" | tr -d '[:space:]')" if [ "$actual_bytes" != "$LLVM_BYTES" ]; then diff --git a/.github/actions/setup-wasmer-llvm/install.test.sh b/.github/actions/setup-wasmer-llvm/install.test.sh index cab124eed..22b303065 100755 --- a/.github/actions/setup-wasmer-llvm/install.test.sh +++ b/.github/actions/setup-wasmer-llvm/install.test.sh @@ -20,6 +20,10 @@ sha256_file() { } work_root="$(mktemp -d)" +mkdir -p "$work_root/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$work_root/no-delay/sleep" +chmod +x "$work_root/no-delay/sleep" +export PATH="$work_root/no-delay:$PATH" cleanup() { rm -rf "$work_root" } @@ -223,12 +227,9 @@ grep -F "$success_final/bin" "$success_runner/github-path" >/dev/null || fail "G for flag in \ '--location' \ '--fail' \ - '--retry 4' \ - '--retry-all-errors' \ - '--retry-delay 10' \ - '--retry-max-time 3600' \ + '--retry 0' \ '--connect-timeout 30' \ - '--max-time 1800' \ + '--max-time' \ "--max-filesize $valid_bytes" \ '--proto =https' \ '--proto-redir =https' \ diff --git a/.github/moon.yml b/.github/moon.yml index 4208183b8..932c55d33 100644 --- a/.github/moon.yml +++ b/.github/moon.yml @@ -29,6 +29,7 @@ tasks: - "actions/setup-wasmer-llvm/**/*" - "/tools/packaging/portable-archive.mts" - "/tools/dev/curl-platform-flags.sh" + - "/tools/dev/acquisition.sh" - "/tools/packaging/testdata/tar-fixture.mts" options: cache: true @@ -47,12 +48,21 @@ tasks: - "/tools/ci/with-projects.sh" - "/tools/dev/bun.sh" - "/tools/dev/curl-platform-flags.sh" + - "/tools/dev/acquisition.sh" - "/tools/ci/workflow-security.mts" - "/tools/ci/workflow-security.test.mts" + - "/tools/ci/workflow-caches.test.mts" + - "/src/wasix/postmaster/lib/common.sh" + - "/src/wasix/postmaster/executor/Cargo.toml" + - "/src/wasix/postmaster/executor/Cargo.lock" + - "/src/wasix/postmaster/executor/src/**" + - "/src/wasix/postmaster/sources/*.toml" + - "/src/third-party/postgres/source.toml" + - "/src/wasix/runtime/assets/build/docker/**" - "/tools/ci/ci-plan-node-products.test.mts" - "/tools/ci/ci-plan-wasix-postmaster-release.test.mts" - "/tools/ci/ci-plan-test-*.mts" - - "/tools/ci/capture-ci-test-observations.sh" + - "/tools/ci/capture-ci-test-observations*.sh" - "/tools/ci/ci-release-scope.test.*" - "/tools/ci/workflow-moon-transfers.test.mts" - "/tools/ci/check-workflows.sh" @@ -61,6 +71,8 @@ tasks: - "/tools/ci/ci-plan.sh" - "/src/native/sdks/ts/tools/published-consumer.mts" - "/src/native/sdks/ts/package.json" + - "/src/native/sdks/swift/tools/ios-carrier-manifest.mts" + - "/src/native/runtime/VERSION" - "/tools/release/check_registry_publication.mts" - "/tools/release/public-consumer-smoke.mts" - "/tools/release/release-graph.mts" diff --git a/.github/scripts/moon-task-capabilities.mts b/.github/scripts/moon-task-capabilities.mts index ca69f49c5..993710831 100644 --- a/.github/scripts/moon-task-capabilities.mts +++ b/.github/scripts/moon-task-capabilities.mts @@ -133,10 +133,22 @@ function compareTargets(left, right) { return left.target < right.target ? -1 : left.target > right.target ? 1 : 0; } -function groupRow(targets) { +function groupRow(targets, index) { const first = targets[0]; return { - label: targets.map(({ label }) => label).join(' + '), + label: `${ + [ + first.requires_apple && 'Apple', + first.requires_android_sdk && 'Android', + first.requires_wasmer_llvm && 'WASIX', + first.requires_swift && 'Swift', + first.requires_rust && 'Rust', + first.requires_maintainer_tools && 'Tooling', + first.requires_workspace && 'JavaScript', + ] + .filter(Boolean) + .join(' / ') || 'Source' + } ${index + 1}`, target_count: targets.length, requires_rust: first.requires_rust, requires_maintainer_tools: first.requires_maintainer_tools, diff --git a/.github/scripts/moon-task-capabilities.test.mts b/.github/scripts/moon-task-capabilities.test.mts index 44e5430ab..1892c935e 100644 --- a/.github/scripts/moon-task-capabilities.test.mts +++ b/.github/scripts/moon-task-capabilities.test.mts @@ -71,7 +71,7 @@ describe('Moon task capabilities', () => { groups.map(({ target_count }) => target_count), [1, 1, 1, 4, 4, 1, 2, 1], ); - assert.equal(groups[3].label, 'Plain / 0 + Plain / 1 + Plain / 2 + Plain / 3'); + assert.equal(groups[3].label, 'Source 4'); assert.equal(groups.filter(({ requires_rust }) => requires_rust).length, 1); assert.equal(groups.filter(({ requires_android_sdk }) => requires_android_sdk).length, 1); assert.equal(groups.filter(({ requires_apple }) => requires_apple).length, 1); diff --git a/.github/scripts/release-candidate-lib.mts b/.github/scripts/release-candidate-lib.mts index d256992a0..172fb54c6 100644 --- a/.github/scripts/release-candidate-lib.mts +++ b/.github/scripts/release-candidate-lib.mts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; -import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; import path from 'node:path'; +import { verifyReceipts } from '../../src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts'; function compareText(left, right) { return left < right ? -1 : left > right ? 1 : 0; @@ -150,6 +151,17 @@ export function affectedPlanBinding(planPath, wasixReleaseRegressionRequired) { projects, extensionPackageProducts, wasixReleaseRegressionRequired, + ...(jobs.includes('native-extension-lifecycle') + ? { + nativeExtensionLifecycle: { + extensions: sortedUniqueStrings( + plan.native_extension_lifecycle_sql_names, + 'native lifecycle extensions', + ), + shardCount: plan.native_extension_lifecycle_shard_count, + }, + } + : {}), ...(qualification === undefined ? {} : { qualification }), }; } @@ -300,6 +312,41 @@ export function wasixEvidenceBinding( }; } +export function nativeEvidenceBinding( + evidenceRoot, + { repository, runId, runAttempt, sha, tree, selection }, +) { + const root = path.resolve(evidenceRoot); + const file = 'output/aggregate-receipt.json'; + const { bytes, value: evidence } = strictJson(path.join(root, file), 'native lifecycle evidence'); + same(evidence.github?.repository, repository, 'native evidence GitHub repository'); + same(evidence.github?.workflow, 'CI', 'native evidence GitHub workflow'); + same(evidence.github?.job, 'native-extension-lifecycle-aggregate', 'native evidence GitHub job'); + same(evidence.github?.runId, Number(runId), 'native evidence GitHub runId'); + positiveInteger(evidence.github?.runAttempt, 'native evidence GitHub runAttempt'); + assert( + evidence.github.runAttempt <= runAttempt, + 'native evidence attempt is newer than candidate', + ); + const verified = verifyReceipts({ + receipts: path.join(root, 'input'), + 'candidate-sha': sha, + 'candidate-tree': tree, + 'expected-extensions-csv': selection.extensions.join(','), + 'expected-shard-count': String(selection.shardCount), + repository, + 'run-id': String(runId), + 'run-attempt': String(evidence.github.runAttempt), + }); + assertBindingMatches(evidence, verified, 'native aggregate receipt'); + return { + artifact: 'native-extension-lifecycle-evidence', + file, + digest: sha256(bytes), + github: evidence.github, + }; +} + export function assertCandidateBindingShape(candidate) { assert( candidate?.schemaVersion === 2, @@ -437,9 +484,47 @@ export function assertCandidateBindingShape(candidate) { requirements.wasixReleaseRegression === candidate.affectedPlan.wasixReleaseRegressionRequired, 'release candidate WASIX evidence requirement is inconsistent with affected plan', ); - const expectedArtifacts = requirements.wasixReleaseRegression - ? ['wasix-release-regression-evidence'] - : []; + const nativeRequired = jobs.includes('native-extension-lifecycle'); + assert( + (requirements.nativeExtensionLifecycle ?? false) === nativeRequired, + 'release candidate native evidence requirement is inconsistent with selected jobs', + ); + if (nativeRequired) { + const selection = candidate.affectedPlan.nativeExtensionLifecycle; + assert( + selection && + sortedUniqueStrings(selection.extensions, 'native lifecycle extensions').length > 0, + 'release candidate native lifecycle selection is missing', + ); + assert( + Number.isSafeInteger(selection.shardCount) && + selection.shardCount > 0 && + selection.shardCount <= selection.extensions.length, + 'release candidate native lifecycle shard count is invalid', + ); + const evidence = candidate.evidence?.nativeExtensionLifecycle; + assert( + /^sha256:[0-9a-f]{64}$/.test(evidence?.digest), + 'release candidate native evidence digest is missing or invalid', + ); + same(evidence.github?.repository, candidate.repository, 'native evidence repository'); + same(evidence.github?.runId, Number(candidate.runId), 'native evidence runId'); + positiveInteger(evidence.github?.runAttempt, 'native evidence runAttempt'); + assert( + evidence.github.runAttempt <= candidate.runAttempt, + 'native evidence attempt is newer than candidate', + ); + } else { + assert( + candidate.evidence?.nativeExtensionLifecycle == null && + candidate.affectedPlan.nativeExtensionLifecycle == null, + 'release candidate carries native evidence that its plan did not require', + ); + } + const expectedArtifacts = [ + ...(nativeRequired ? ['native-extension-lifecycle-evidence'] : []), + ...(requirements.wasixReleaseRegression ? ['wasix-release-regression-evidence'] : []), + ]; assert( JSON.stringify(requirements.artifacts) === JSON.stringify(expectedArtifacts), 'release candidate evidence artifact requirements are inconsistent', diff --git a/.github/scripts/resolve-planned-moon-execution.mts b/.github/scripts/resolve-planned-moon-execution.mts index 263770171..27ef381b2 100644 --- a/.github/scripts/resolve-planned-moon-execution.mts +++ b/.github/scripts/resolve-planned-moon-execution.mts @@ -91,6 +91,23 @@ function parseTransferred() { return value; } +export function executionBatches(targets, tasks) { + const levels = new Map(); + const batches = []; + // resolveExecution already orders prerequisites before consumers. Only batch + // independent tasks; --upstream none does not enforce their dependency order. + for (const target of targets) { + const level = Math.max( + 0, + ...dependencyTargets(tasks.get(target), tasks).map((dep) => (levels.get(dep) ?? -1) + 1), + ); + levels.set(target, level); + batches[level] ??= []; + batches[level].push(target); + } + return batches; +} + function taskMap() { const graph = JSON.parse(readFileSync(process.env.OLIPHAUNT_MOON_TASK_GRAPH_FILE, 'utf8')); return new Map(Object.values(graph.data ?? {}).map((task) => [task.target, task])); @@ -126,7 +143,10 @@ if (import.meta.main) { for (const target of targets) collect(target); const execution = resolveExecution(targets, transferred, tasks); for (const target of execution.localDependencies) console.log(`local\t${target}`); - for (const target of execution.targets) console.log(`target\t${target}`); + const batches = execution.transferred.length + ? executionBatches(execution.targets, tasks) + : [execution.targets]; + for (const batch of batches) console.log(`target\t${batch.join(' ')}`); for (const target of execution.transferred) console.log(`transferred\t${target}`); } catch (error) { fail(error instanceof Error ? error.message : String(error)); diff --git a/.github/scripts/resolve-planned-moon-execution.test.mts b/.github/scripts/resolve-planned-moon-execution.test.mts index 8c1e5bee8..67c000619 100644 --- a/.github/scripts/resolve-planned-moon-execution.test.mts +++ b/.github/scripts/resolve-planned-moon-execution.test.mts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import { resolveExecution } from './resolve-planned-moon-execution.mts'; +import { executionBatches, resolveExecution } from './resolve-planned-moon-execution.mts'; const tasks = new Map([ [ @@ -26,6 +26,21 @@ const tasks = new Map([ ], ]); +test('batches independent artifact consumers without starting their dependents early', () => { + const graph = new Map([ + ['import:bytes', { deps: [] }], + ['a:build', { deps: ['import:bytes'] }], + ['b:build', { deps: ['import:bytes'] }], + ['c:test', { deps: ['a:build', 'b:build'] }], + ['d:test', { deps: ['a:build'] }], + ]); + const execution = resolveExecution(['c:test', 'd:test'], ['import:bytes'], graph); + assert.deepEqual(executionBatches(execution.targets, graph), [ + ['a:build', 'b:build'], + ['c:test', 'd:test'], + ]); +}); + test('leaves ordinary Moon execution intact when no dependency was transferred', () => { assert.deepEqual(resolveExecution(['release:package'], [], tasks), { localDependencies: [], diff --git a/.github/scripts/resolve-planned-moon-execution.test.sh b/.github/scripts/resolve-planned-moon-execution.test.sh index 806b3b28c..0936fc2c0 100644 --- a/.github/scripts/resolve-planned-moon-execution.test.sh +++ b/.github/scripts/resolve-planned-moon-execution.test.sh @@ -8,7 +8,7 @@ bash tools/dev/bun.sh test ./.github/scripts/resolve-planned-moon-execution.test resolver=.github/scripts/resolve-planned-moon-execution.mts export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"wasix-ts-sdk-package":["oliphaunt-wasix-ts:package","oliphaunt-wasix-ts:test-consumer","oliphaunt-wasix-ts:test-browser"]}' export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["liboliphaunt-wasix:runtime-portable","oliphaunt-wasix-napi:build-release-assets","extension-artifacts-wasix:compiler-output","database-resources:build-wasix-standard","database-resources:build-wasix-icu","database-resources:package-icu"]' -bash tools/dev/bun.sh "$resolver" wasix-ts-sdk-package >"$scratch/output" +bash tools/dev/bun.sh "$resolver" wasix-ts-sdk-package | awk -F '\t' '{n=split($2, targets, " "); for(i=1;i<=n;i++) print $1 "\t" targets[i]}' >"$scratch/output" for task in package test-consumer test-browser; do grep -Fx "$(printf 'target\toliphaunt-wasix-ts:%s' "$task")" "$scratch/output"; done grep -Fx $'target\tdatabase-resources:package-wasix' "$scratch/output" for variant in standard icu; do @@ -29,22 +29,68 @@ for platform in android ios; do printf 'transferred\tliboliphaunt-native:build-runtime-%s\n' "$target" >>"$scratch/expected" done export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON="${transfers%,}]" - bash tools/dev/bun.sh "$resolver" "$job" >"$scratch/output" + bash tools/dev/bun.sh "$resolver" "$job" | awk -F '\t' '{n=split($2, targets, " "); for(i=1;i<=n;i++) print $1 "\t" targets[i]}' >"$scratch/output" cmp "$scratch/expected" "$scratch/output" done export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"react-native-sdk-package":["oliphaunt-react-native:test-consumer","oliphaunt-react-native:package"]}' export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["liboliphaunt-native:finalize-runtime-ios-abi"]' -bash tools/dev/bun.sh "$resolver" react-native-sdk-package >"$scratch/output" +bash tools/dev/bun.sh "$resolver" react-native-sdk-package | awk -F '\t' '{n=split($2, targets, " "); for(i=1;i<=n;i++) print $1 "\t" targets[i]}' >"$scratch/output" grep $'^target\t' "$scratch/output" >"$scratch/targets" printf 'target\toliphaunt-react-native:package\ntarget\toliphaunt-react-native:test-consumer\n' >"$scratch/expected" cmp "$scratch/expected" "$scratch/targets" if grep -E $'^local\t(oliphaunt-react-native:package|liboliphaunt-native:finalize-runtime-ios-abi)$' "$scratch/output"; then exit 1; fi unset OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"liboliphaunt-native-android":["liboliphaunt-native:package-runtime-android-arm64-v8a","liboliphaunt-native:package-runtime-android-x86_64"]}' -bash tools/dev/bun.sh "$resolver" liboliphaunt-native-android liboliphaunt-native:package-runtime-android-x86_64 >"$scratch/output" +bash tools/dev/bun.sh "$resolver" liboliphaunt-native-android liboliphaunt-native:package-runtime-android-x86_64 | awk -F '\t' '{n=split($2, targets, " "); for(i=1;i<=n;i++) print $1 "\t" targets[i]}' >"$scratch/output" printf 'target\tliboliphaunt-native:package-runtime-android-x86_64\n' >"$scratch/expected" cmp "$scratch/expected" "$scratch/output" if bash tools/dev/bun.sh "$resolver" liboliphaunt-native-android liboliphaunt-native:package-runtime-ios-xcframework >"$scratch/output" 2>"$scratch/error"; then echo 'accepted a target outside the job plan' >&2; exit 1 fi grep -q 'is not planned' "$scratch/error" + + +# Prove the scheduling boundary with the pinned Moon binary: siblings overlap, +# their consumer waits for both, and a transferred producer must never execute. +mkdir -p "$scratch/parallel/.moon" "$scratch/parallel/.github/scripts" +cp .github/scripts/{run-planned-moon-job.sh,run-moon-targets.sh,resolve-planned-moon-execution.mts,select-planned-moon-targets.mts} "$scratch/parallel/.github/scripts/" +cat > "$scratch/parallel/.moon/workspace.yml" <<'YAML' +projects: + fixture: . +YAML +cat > "$scratch/parallel/moon.yml" <<'YAML' +id: fixture +language: unknown +tasks: + downloaded: + script: exit 99 + a: + command: bash sibling.sh a b + deps: [downloaded] + b: + command: bash sibling.sh b a + deps: [downloaded] + joined: + script: test -f a.done && test -f b.done && touch joined.done + deps: [a, b] +YAML +cat > "$scratch/parallel/sibling.sh" <<'SH' +#!/usr/bin/env bash +set -eu +touch "$1.started" +for attempt in {1..100}; do + [ ! -e "$2.started" ] || break + sleep 0.1 +done +test -f "$2.started" +touch "$1.done" +SH +( + cd "$scratch/parallel" + git init -q + git -c user.name=fixture -c user.email=fixture@example.invalid commit -q --allow-empty -m fixture + export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"parallel":["fixture:joined"]}' + export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["fixture:downloaded"]' + MOON_CONCURRENCY=2 bash .github/scripts/run-planned-moon-job.sh parallel + test -f joined.done +) diff --git a/.github/scripts/run-moon-targets.sh b/.github/scripts/run-moon-targets.sh index f3501fa00..b6a691e50 100755 --- a/.github/scripts/run-moon-targets.sh +++ b/.github/scripts/run-moon-targets.sh @@ -10,6 +10,10 @@ if [ "${1:-}" = --matrix ]; then groups="$(bun .github/scripts/select-moon-target-groups.mts)" while IFS=$'\t' read -r upstream targets; do read -r -a target_args <<<"$targets" + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + printf '\nSelected Moon tasks:\n\n' >> "$GITHUB_STEP_SUMMARY" + printf -- '- `%s`\n' "${target_args[@]}" >> "$GITHUB_STEP_SUMMARY" + fi "$moon_bin" run --upstream "$upstream" "${target_args[@]}" done <<<"$groups" else diff --git a/.github/scripts/run-planned-moon-job.sh b/.github/scripts/run-planned-moon-job.sh index efd05cbc8..e55343298 100755 --- a/.github/scripts/run-planned-moon-job.sh +++ b/.github/scripts/run-planned-moon-job.sh @@ -52,14 +52,18 @@ if [[ "${#transferred_dependencies[@]}" -gt 0 ]]; then if [[ "${#local_dependencies[@]}" -gt 0 ]]; then .github/scripts/run-moon-targets.sh "${local_dependencies[@]}" fi - for target in "${targets[@]}"; do + for batch in "${targets[@]}"; do # Omitting transferred producers also omits their source hashes in Moon. # Execute intermediate prerequisites and roots against the downloaded bytes. - MOON_CACHE=off .github/scripts/run-moon-targets.sh --upstream none "$target" + read -r -a batch_targets <<<"$batch" + MOON_CACHE=off .github/scripts/run-moon-targets.sh --upstream none "${batch_targets[@]}" done exit 0 fi +# Without transfers the resolver emits one batch and Moon owns the whole DAG. +read -r -a targets <<<"${targets[0]}" + if [[ "${#moon_args[@]}" -gt 0 ]]; then exec .github/scripts/run-moon-targets.sh "${moon_args[@]}" "${targets[@]}" fi diff --git a/.github/scripts/verify-release-candidate.mts b/.github/scripts/verify-release-candidate.mts index 9a9ba41fd..dbf9e7b93 100644 --- a/.github/scripts/verify-release-candidate.mts +++ b/.github/scripts/verify-release-candidate.mts @@ -1,15 +1,20 @@ #!/usr/bin/env bun import { readFileSync } from 'node:fs'; import process from 'node:process'; +import { + extensionArtifactProductsForReleaseProducts, + extensionSqlNamesForProducts, +} from '../../tools/release/release-artifact-targets.mts'; import { affectedPlanBinding, assertBindingMatches, assertCandidateBindingShape, + assertQualificationProductCoverage, candidateQualificationMode, FULL_PAYLOAD_QUALIFICATION_MODE, + nativeEvidenceBinding, PRODUCT_QUALIFICATION_MODE, - assertQualificationProductCoverage, wasixEvidenceBinding, } from './release-candidate-lib.mts'; @@ -36,6 +41,8 @@ function parseArgs(argv) { '--plan', '--qualification-mode', '--products-json', + '--native-evidence-required', + '--native-evidence-root', '--wasix-evidence-required', '--wasix-evidence-root', ].includes(name) @@ -52,7 +59,8 @@ function parseArgs(argv) { fail( 'usage: verify-release-candidate.mts --plan ' + '--wasix-evidence-required true|false [--qualification-mode full-payload] ' + - '[--wasix-evidence-root ]', + '[--wasix-evidence-root ] [--native-evidence-required true|false] ' + + '[--native-evidence-root ]', ); } const required = values.get('wasix-evidence-required'); @@ -62,6 +70,11 @@ function parseArgs(argv) { if (required === 'true' && !values.has('wasix-evidence-root')) { fail('--wasix-evidence-root is required when WASIX evidence is required'); } + const nativeRequired = values.get('native-evidence-required') ?? 'false'; + if (!['true', 'false'].includes(nativeRequired)) + fail('--native-evidence-required must be true or false'); + if (nativeRequired === 'true' && !values.has('native-evidence-root')) + fail('--native-evidence-root is required when native evidence is required'); const qualificationMode = values.get('qualification-mode') ?? FULL_PAYLOAD_QUALIFICATION_MODE; if ( ![FULL_PAYLOAD_QUALIFICATION_MODE, PRODUCT_QUALIFICATION_MODE, 'release'].includes( @@ -79,6 +92,8 @@ function parseArgs(argv) { candidatePath, planPath: values.get('plan'), wasixEvidenceRequired: required === 'true', + nativeEvidenceRequired: nativeRequired === 'true', + nativeEvidenceRoot: values.get('native-evidence-root'), wasixEvidenceRoot: values.get('wasix-evidence-root'), qualificationMode, products, @@ -195,4 +210,36 @@ if (args.wasixEvidenceRequired) { } } +if (args.nativeEvidenceRequired && !candidate.evidenceRequirements.nativeExtensionLifecycle) + fail( + 'selected release products require native evidence, but the qualified CI plan did not require it', + ); +if (args.nativeEvidenceRequired) { + try { + if (args.products) { + const requiredExtensions = extensionSqlNamesForProducts( + extensionArtifactProductsForReleaseProducts(args.products, { family: 'native' }), + ); + for (const extension of requiredExtensions) { + if (!expectedPlan.nativeExtensionLifecycle.extensions.includes(extension)) + fail(`native evidence is missing published extension ${extension}`); + } + } + assertBindingMatches( + candidate.evidence.nativeExtensionLifecycle, + nativeEvidenceBinding(args.nativeEvidenceRoot, { + repository: expected.repository, + runId: expected.runId, + runAttempt: candidate.runAttempt, + sha: expected.sha, + tree: expectedTree, + selection: expectedPlan.nativeExtensionLifecycle, + }), + 'release candidate native evidence', + ); + } catch (error) { + fail(error.message); + } +} + console.log(`verified qualified CI run ${candidate.runId} for ${candidate.sha}`); diff --git a/.github/scripts/write-release-candidate.mts b/.github/scripts/write-release-candidate.mts index 1db3643e5..0a606e618 100644 --- a/.github/scripts/write-release-candidate.mts +++ b/.github/scripts/write-release-candidate.mts @@ -8,6 +8,7 @@ import { assertCandidateBindingShape, candidateQualificationMode, FULL_PAYLOAD_QUALIFICATION_MODE, + nativeEvidenceBinding, PRODUCT_QUALIFICATION_MODE, wasixEvidenceBinding, } from './release-candidate-lib.mts'; @@ -81,6 +82,23 @@ if (wasixRequired) { } } +const nativeRequired = affectedPlan.nativeExtensionLifecycle !== undefined; +let nativeEvidence = null; +if (nativeRequired) { + try { + nativeEvidence = nativeEvidenceBinding(requiredEnv('NATIVE_EVIDENCE_ROOT'), { + repository: requiredEnv('GITHUB_REPOSITORY'), + runId: requiredEnv('GITHUB_RUN_ID'), + runAttempt, + sha: checkedOutSha, + tree, + selection: affectedPlan.nativeExtensionLifecycle, + }); + } catch (error) { + fail(error.message); + } +} + const candidate = { schemaVersion: 2, repository: requiredEnv('GITHUB_REPOSITORY'), @@ -96,10 +114,15 @@ const candidate = { affectedPlan, evidenceRequirements: { wasixReleaseRegression: wasixRequired, - artifacts: wasixRequired ? ['wasix-release-regression-evidence'] : [], + nativeExtensionLifecycle: nativeRequired, + artifacts: [ + ...(nativeRequired ? ['native-extension-lifecycle-evidence'] : []), + ...(wasixRequired ? ['wasix-release-regression-evidence'] : []), + ], }, evidence: { wasixReleaseRegression: wasixEvidence, + nativeExtensionLifecycle: nativeEvidence, }, }; diff --git a/.github/workflows/broker-runtime.yml b/.github/workflows/broker-runtime.yml index 5bdd279b4..4883f0527 100644 --- a/.github/workflows/broker-runtime.yml +++ b/.github/workflows/broker-runtime.yml @@ -2,6 +2,10 @@ name: broker-runtime on: workflow_call: inputs: + cache-save-if: + description: Resolved cache-save policy from the calling CI workflow. + required: true + type: boolean matrix: required: true type: string @@ -14,7 +18,6 @@ env: OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ inputs.job-targets }} CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - HEAVY_CACHE_SAVE_IF: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} defaults: run: shell: bash @@ -39,6 +42,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} - name: Set up MSVC if: ${{ runner.os == 'Windows' }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34e5d8cd5..6475cbad5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,8 +70,6 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - NPM_VERSION: 11.18.0 - DENO_VERSION: v2.8.1 ACTIONLINT_VERSION: 1.7.12 ASSET_PROFILE: release WASMER_LLVM_VERSION: "22.1" @@ -98,6 +96,12 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: + liboliphaunt_wasix_aot_runtime_matrix_linux: ${{ steps.plan.outputs.liboliphaunt_wasix_aot_runtime_matrix_linux }} + liboliphaunt_wasix_aot_runtime_matrix_other: ${{ steps.plan.outputs.liboliphaunt_wasix_aot_runtime_matrix_other }} + wasix_napi_targets: ${{ steps.plan.outputs.wasix_napi_targets }} + reuse_ios_carrier: ${{ steps.plan.outputs.reuse_ios_carrier }} + cache_save_if: ${{ env.HEAVY_CACHE_SAVE_IF }} + llvm_version: ${{ env.WASMER_LLVM_VERSION }} published_dependencies: ${{ steps.plan.outputs.published_dependencies }} mobile_extension_package_native_targets_ios_csv: ${{ steps.plan.outputs.mobile_extension_package_native_targets_ios_csv }} mobile_extension_package_native_targets_android_csv: ${{ steps.plan.outputs.mobile_extension_package_native_targets_android_csv }} @@ -155,7 +159,7 @@ jobs: qualification_mode: ${{ steps.plan.outputs.qualification_mode }} qualification_base_sha: ${{ steps.plan.outputs.qualification_base_sha }} qualification_head_sha: ${{ steps.plan.outputs.qualification_head_sha }} - wasix_release_regression_required: ${{ contains(fromJson(steps.plan.outputs.jobs), 'liboliphaunt-wasix-runtime') && (github.event_name != 'workflow_dispatch' || inputs.wasm_target == 'all' || inputs.wasm_target == 'linux-x64-gnu') }} + wasix_release_regression_required: ${{ contains(fromJson(steps.plan.outputs.jobs), 'wasix-release-regression') && (github.event_name != 'workflow_dispatch' || inputs.wasm_target == 'all' || inputs.wasm_target == 'linux-x64-gnu') }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -167,8 +171,17 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon with: + task-cache: "false" install-workspace: "false" + - name: Restore frozen iOS carrier metadata + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/ci/ios-carrier + key: ios-carrier-v1-${{ hashFiles('src/native/runtime/VERSION', 'src/native/sdks/swift/tools/ios-carrier-manifest.mts', 'tools/packaging/**', 'tools/release/**') }}-${{ github.event.pull_request.head.sha || github.sha }} + restore-keys: | + ios-carrier-v1-${{ hashFiles('src/native/runtime/VERSION', 'src/native/sdks/swift/tools/ios-carrier-manifest.mts', 'tools/packaging/**', 'tools/release/**') }}- + - name: Plan artifact builder jobs id: plan env: @@ -192,6 +205,15 @@ jobs: MOON_HEAD: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }} run: bash .github/scripts/write-affected-moon-target-matrices.sh + - name: Transfer verified unchanged iOS carrier metadata + if: ${{ steps.plan.outputs.reuse_ios_carrier == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: frozen-ios-carrier + path: target/ci/ios-carrier + if-no-files-found: error + retention-days: 30 + - name: Upload build plan uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: @@ -218,6 +240,7 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon with: + task-cache: "false" install-workspace: "false" - name: Check release intent @@ -266,6 +289,8 @@ jobs: - name: Set up Rust if: ${{ matrix.requires_rust }} uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Install Android Rust targets if: ${{ matrix.requires_rust && matrix.requires_android_sdk }} @@ -284,11 +309,15 @@ jobs: - name: Set up Android if: ${{ matrix.requires_android_sdk }} uses: ./.github/actions/setup-android + with: + native-tools: ${{ matrix.requires_rust && 'true' || 'false' }} + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Wasmer LLVM if: ${{ matrix.requires_wasmer_llvm }} uses: ./.github/actions/setup-wasmer-llvm with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} url: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_URL || env.WASMER_LLVM_LINUX_X64_URL }} sha256: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_SHA256 || env.WASMER_LLVM_LINUX_X64_SHA256 }} bytes: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_BYTES || env.WASMER_LLVM_LINUX_X64_BYTES }} @@ -326,6 +355,8 @@ jobs: - name: Set up Rust if: ${{ needs.affected.outputs.policy_requires_rust == 'true' }} uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Swift if: ${{ needs.affected.outputs.policy_requires_swift == 'true' }} @@ -334,11 +365,14 @@ jobs: - name: Set up Android if: ${{ needs.affected.outputs.policy_requires_android_sdk == 'true' }} uses: ./.github/actions/setup-android + with: + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Wasmer LLVM if: ${{ needs.affected.outputs.policy_requires_wasmer_llvm == 'true' }} uses: ./.github/actions/setup-wasmer-llvm with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} url: ${{ env.WASMER_LLVM_LINUX_X64_URL }} sha256: ${{ env.WASMER_LLVM_LINUX_X64_SHA256 }} bytes: ${{ env.WASMER_LLVM_LINUX_X64_BYTES }} @@ -408,6 +442,7 @@ jobs: if: ${{ matrix.requires_rust }} uses: ./.github/actions/setup-rust with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} tools: cargo-nextest@0.9.137 - name: Set up Swift @@ -423,11 +458,15 @@ jobs: - name: Set up Android if: ${{ matrix.requires_android_sdk }} uses: ./.github/actions/setup-android + with: + native-tools: ${{ matrix.requires_rust && 'true' || 'false' }} + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Wasmer LLVM if: ${{ matrix.requires_wasmer_llvm }} uses: ./.github/actions/setup-wasmer-llvm with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} url: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_URL || env.WASMER_LLVM_LINUX_X64_URL }} sha256: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_SHA256 || env.WASMER_LLVM_LINUX_X64_SHA256 }} bytes: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_BYTES || env.WASMER_LLVM_LINUX_X64_BYTES }} @@ -470,39 +509,59 @@ jobs: run: bun .github/scripts/check-ci-gate.mts selected extension-artifacts-native-linux: + name: Native Extensions (Linux) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_linux).include[0] != null }} uses: ./.github/workflows/extension-artifacts-native.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_linux }} job-targets: ${{ needs.affected.outputs.job_targets }} - extension-artifacts-native-android: + extension-artifacts-native-android-static: + name: Native Extensions (Android Compilation) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_android).include[0] != null }} uses: ./.github/workflows/extension-artifacts-native.yml with: + phase: android-static + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} + matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_android }} + job-targets: ${{ needs.affected.outputs.job_targets }} + + extension-artifacts-native-android: + name: Native Extensions (Android Packaging) + needs: [affected, extension-artifacts-native-android-static, extension-artifacts-native-linux] + if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_android).include[0] != null }} + uses: ./.github/workflows/extension-artifacts-native.yml + with: + phase: android-package + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_android }} job-targets: ${{ needs.affected.outputs.job_targets }} extension-artifacts-native-ios: + name: Native Extensions (iOS) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_ios).include[0] != null }} uses: ./.github/workflows/extension-artifacts-native.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_ios }} job-targets: ${{ needs.affected.outputs.job_targets }} extension-artifacts-native-other: + name: Native Extensions (Desktop) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_other).include[0] != null }} uses: ./.github/workflows/extension-artifacts-native.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_other }} job-targets: ${{ needs.affected.outputs.job_targets }} extension-artifacts-native: - name: Builds / extension-artifacts-native + name: Builds / Native Extensions needs: [affected, extension-artifacts-native-linux, extension-artifacts-native-android, extension-artifacts-native-ios, extension-artifacts-native-other] if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'extension-artifacts-native') }} runs-on: ubuntu-24.04 @@ -517,7 +576,7 @@ jobs: esac extension-artifacts-wasix: - name: Builds / WASIX Extension Artifacts (${{ matrix.target }}) + name: Builds / WASIX Extensions (${{ matrix.target }}) needs: - affected - liboliphaunt-wasix-runtime @@ -540,6 +599,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download portable WASIX runtime outputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -568,7 +629,7 @@ jobs: if-no-files-found: error extension-packages: - name: Builds / Extension Packages + name: Packages / Extensions needs: - affected - extension-artifacts-native @@ -590,6 +651,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download native exact-extension artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -626,20 +689,24 @@ jobs: if-no-files-found: error mobile-extension-packages-android: + name: Extension Packages (Android) needs: [affected, extension-artifacts-native-android] if: ${{ needs.affected.outputs.mobile_extension_package_native_targets_android_csv != '' }} uses: ./.github/workflows/mobile-extension-packages.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} family: android targets: ${{ needs.affected.outputs.mobile_extension_package_native_targets_android_csv }} products: ${{ needs.affected.outputs.extension_package_products_csv }} job-targets: ${{ needs.affected.outputs.job_targets }} mobile-extension-packages-ios: + name: Extension Packages (iOS) needs: [affected, extension-artifacts-native-ios] if: ${{ needs.affected.outputs.mobile_extension_package_native_targets_ios_csv != '' }} uses: ./.github/workflows/mobile-extension-packages.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} family: ios targets: ${{ needs.affected.outputs.mobile_extension_package_native_targets_ios_csv }} products: ${{ needs.affected.outputs.extension_package_products_csv }} @@ -660,23 +727,27 @@ jobs: esac liboliphaunt-native-desktop-linux: + name: Native Runtime (Linux) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.liboliphaunt_native_desktop_runtime_matrix_linux).include[0] != null }} uses: ./.github/workflows/liboliphaunt-native-desktop.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.liboliphaunt_native_desktop_runtime_matrix_linux }} job-targets: ${{ needs.affected.outputs.job_targets }} liboliphaunt-native-desktop-other: + name: Native Runtime (Desktop) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.liboliphaunt_native_desktop_runtime_matrix_other).include[0] != null }} uses: ./.github/workflows/liboliphaunt-native-desktop.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.liboliphaunt_native_desktop_runtime_matrix_other }} job-targets: ${{ needs.affected.outputs.job_targets }} liboliphaunt-native-desktop: - name: Builds / liboliphaunt-native-desktop + name: Builds / Native Desktop Runtime needs: [affected, liboliphaunt-native-desktop-linux, liboliphaunt-native-desktop-other] if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-native-desktop') }} runs-on: ubuntu-24.04 @@ -715,6 +786,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Android uses: ./.github/actions/setup-android @@ -807,6 +880,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Prepare native build paths env: @@ -866,7 +941,7 @@ jobs: if-no-files-found: ignore liboliphaunt-native-android-abi: - name: Builds / Native Runtime ABI Compatibility (Android) + name: Builds / Android ABI + Seeds needs: - affected - liboliphaunt-native-android @@ -883,6 +958,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download Android arm64 ABI receipts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -938,7 +1015,7 @@ jobs: if-no-files-found: error liboliphaunt-native-ios-abi: - name: Builds / Native Runtime ABI Compatibility (iOS) + name: Builds / iOS ABI + Seeds needs: - affected - liboliphaunt-native-ios @@ -955,6 +1032,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download iOS ABI receipts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -998,7 +1077,7 @@ jobs: if-no-files-found: error liboliphaunt-native-release-assets: - name: Builds / Native Runtime Release Assets + name: Packages / Native Runtime needs: - affected - liboliphaunt-native-android @@ -1019,9 +1098,13 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download liboliphaunt target release assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1070,7 +1153,7 @@ jobs: if-no-files-found: error rust-sdk-package: - name: Builds / Rust SDK Package + name: Packages / Rust SDK needs: - affected - checks @@ -1091,6 +1174,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Build Rust SDK package artifacts run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh rust-sdk-package @@ -1152,23 +1237,27 @@ jobs: if-no-files-found: error broker-runtime-linux: + name: Broker Runtime (Linux) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.broker_runtime_matrix_linux).include[0] != null }} uses: ./.github/workflows/broker-runtime.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.broker_runtime_matrix_linux }} job-targets: ${{ needs.affected.outputs.job_targets }} broker-runtime-other: + name: Broker Runtime (Desktop) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.broker_runtime_matrix_other).include[0] != null }} uses: ./.github/workflows/broker-runtime.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.broker_runtime_matrix_other }} job-targets: ${{ needs.affected.outputs.job_targets }} broker-runtime: - name: Builds / broker-runtime + name: Builds / Broker Runtime needs: [affected, broker-runtime-linux, broker-runtime-other] if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'broker-runtime') }} runs-on: ubuntu-24.04 @@ -1207,6 +1296,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up MSVC if: ${{ runner.os == 'Windows' }} @@ -1230,7 +1321,7 @@ jobs: if-no-files-found: error broker-release-assets: - name: Builds / Broker Release Assets + name: Packages / Broker Runtime needs: - affected - broker-runtime @@ -1247,6 +1338,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download broker target release assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1262,7 +1355,7 @@ jobs: run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh broker-release-assets node-direct-release-assets: - name: Builds / Node.js Direct Release Assets + name: Packages / Node.js Direct needs: - affected - node-direct @@ -1279,6 +1372,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download Node direct target release assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1301,115 +1396,22 @@ jobs: run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh node-direct-release-assets wasix-napi: - name: Builds / WASIX Node-API (${{ matrix.target }}) - needs: - - affected - - extension-artifacts-wasix - - liboliphaunt-wasix-aot - - liboliphaunt-wasix-runtime - if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.extension-artifacts-wasix.result == 'success' && needs.liboliphaunt-wasix-aot.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && contains(fromJson(needs.affected.outputs.jobs), 'wasix-napi') }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.affected.outputs.wasix_napi_runtime_matrix) }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 + name: Builds / WASIX Node-API + needs: [affected, wasix-host-linux, wasix-host-other] + if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'wasix-napi') }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha || github.sha }} - persist-credentials: false - - - name: Set up Moon - uses: ./.github/actions/setup-moon - - - name: Set up Rust - uses: ./.github/actions/setup-rust - - - name: Set up MSVC - if: ${{ runner.os == 'Windows' }} - uses: ./.github/actions/setup-msvc - - - name: Download same-run portable WASIX outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: liboliphaunt-wasix-runtime-portable - path: . - - - name: Download same-run WASIX exact-extension outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - pattern: liboliphaunt-wasix-extension-artifacts-* - path: target/extensions/wasix/release-assets - merge-multiple: true - - - name: Download same-run target extension AOT outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: liboliphaunt-wasix-extension-aot-${{ matrix.target }} - path: target/extensions/wasix/aot-artifacts - - - name: Download same-run target core and tool AOT outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: liboliphaunt-wasix-runtime-aot-${{ matrix.target }} - path: target/oliphaunt-wasix/napi-aot-download - - - name: Restore exact target core and tool AOT layout - shell: bash + - name: Check selected hosts env: - EXPECTED_TARGET_TRIPLE: ${{ matrix.target_triple }} + RESULTS: ${{ join(needs.*.result, ',') }} run: | - artifact_dir=target/oliphaunt-wasix/napi-aot-download - marker="$artifact_dir/target-triple.txt" - raw_target_dir="$artifact_dir/files" - if [[ ! -f "$marker" || ! -d "$raw_target_dir" ]]; then - echo "invalid WASIX N-API AOT artifact envelope in $artifact_dir" >&2 - exit 1 - fi - target="$(tr -d '\r\n' < "$marker")" - if [[ "$target" != "${EXPECTED_TARGET_TRIPLE:?EXPECTED_TARGET_TRIPLE is required}" ]]; then - echo "WASIX N-API AOT artifact targets $target, expected $EXPECTED_TARGET_TRIPLE" >&2 - exit 1 - fi - destination="target/oliphaunt-wasix/aot/$target" - mkdir -p "$destination" - cp -R "$raw_target_dir/." "$destination/" - - - name: Set up macOS smoke timeout - if: ${{ runner.os == 'macOS' }} - shell: bash - run: brew list coreutils >/dev/null 2>&1 || HOMEBREW_NO_AUTO_UPDATE=1 brew install coreutils - - - name: Build WASIX Node-API release assets - shell: bash - env: - OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD: "1" - OLIPHAUNT_ICU_DATA_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/wasix-build/work/icu-wasix/share/icu - OLIPHAUNT_WASM_GENERATED_AOT_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/aot - OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT: ${{ github.workspace }}/target/extension-artifacts - OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/assets - OLIPHAUNT_WASIX_NAPI_ARTIFACT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["extension-artifacts-wasix:build-target", "extension-artifacts-wasix:build-aot", "liboliphaunt-wasix:runtime-aot"]' - run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-napi - - - name: Upload WASIX Node-API release assets - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: oliphaunt-wasix-napi-release-assets-${{ matrix.target }} - path: target/oliphaunt-wasix-napi/release-assets - if-no-files-found: error - - - name: Upload WASIX Node-API optional npm package - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: oliphaunt-wasix-napi-npm-package-${{ matrix.target }} - path: target/oliphaunt-wasix-napi/npm-packages/*.tgz - if-no-files-found: error + case ",$RESULTS," in + *,failure,*|*,cancelled,*) exit 1 ;; + esac wasix-napi-release-assets: - name: Builds / WASIX Node-API Release Assets + name: Packages / WASIX Node-API needs: - affected - wasix-napi @@ -1426,6 +1428,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download WASIX Node-API target release assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1464,6 +1468,8 @@ jobs: uses: ./.github/actions/setup-moon - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Install Apple Rust targets run: rustup target add aarch64-apple-ios aarch64-apple-ios-sim aarch64-apple-darwin - name: Select Apple framework toolchain @@ -1482,7 +1488,7 @@ jobs: if-no-files-found: error swift-sdk-package: - name: Builds / Swift SDK Package + name: Packages / Swift SDK needs: - affected - liboliphaunt-native-ios-abi @@ -1527,7 +1533,7 @@ jobs: if-no-files-found: error kotlin-sdk-package: - name: Builds / Kotlin SDK Package + name: Packages / Kotlin SDK needs: - affected - checks @@ -1548,6 +1554,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Install Android Rust targets run: rustup target add aarch64-linux-android x86_64-linux-android @@ -1568,7 +1576,7 @@ jobs: if-no-files-found: error kotlin-maven-staging: - name: Builds / Kotlin SDK Maven Staging + name: Packages / Kotlin Maven needs: - affected - kotlin-sdk-package @@ -1585,6 +1593,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download exact same-run Kotlin SDK package artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1599,11 +1609,13 @@ jobs: run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh kotlin-maven-staging react-native-sdk-package: - name: Builds / React Native SDK Package + name: Packages / React Native SDK needs: - affected + - checks + - tests - liboliphaunt-native-ios-abi - if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'react-native-sdk-package') }} + if: ${{ !cancelled() && !failure() && needs.checks.result == 'success' && needs.tests.result == 'success' && (needs.affected.outputs.reuse_ios_carrier == 'true' || needs.liboliphaunt-native-ios-abi.result == 'success') && contains(fromJson(needs.affected.outputs.jobs), 'react-native-sdk-package') }} runs-on: ubuntu-24.04 timeout-minutes: 90 steps: @@ -1620,13 +1632,22 @@ jobs: install-workspace: "true" - name: Download Apple liboliphaunt release assets + if: ${{ needs.affected.outputs.reuse_ios_carrier != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: liboliphaunt-native-abi-compatible-release-assets-ios-datum64 path: target/liboliphaunt/abi-compatible-release-assets/ios-datum64 + - name: Download frozen iOS carrier metadata + if: ${{ needs.affected.outputs.reuse_ios_carrier == 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: frozen-ios-carrier + path: target/ci/ios-carrier + - name: Build React Native SDK package artifacts env: + OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER: ${{ needs.affected.outputs.reuse_ios_carrier == 'true' && 'target/ci/ios-carrier/manifest.json' || '' }} OLIPHAUNT_REACT_NATIVE_IOS_RELEASE_ASSET_DIR: ${{ github.workspace }}/target/liboliphaunt/abi-compatible-release-assets/ios-datum64 OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-native:finalize-runtime-ios-abi"]' run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh react-native-sdk-package @@ -1638,8 +1659,24 @@ jobs: path: target/sdk-artifacts/oliphaunt-react-native if-no-files-found: error + - name: Freeze current iOS carrier metadata + if: ${{ env.HEAVY_CACHE_SAVE_IF == 'true' }} + env: + SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p target/ci/ios-carrier + cp target/sdk-artifacts/oliphaunt-react-native/ios-carriers/oliphaunt-react-native-ios-carriers.json target/ci/ios-carrier/manifest.json + printf '%s\n' "$SOURCE_SHA" > target/ci/ios-carrier/source-sha + + - name: Save frozen iOS carrier metadata + if: ${{ env.HEAVY_CACHE_SAVE_IF == 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/ci/ios-carrier + key: ios-carrier-v1-${{ hashFiles('src/native/runtime/VERSION', 'src/native/sdks/swift/tools/ios-carrier-manifest.mts', 'tools/packaging/**', 'tools/release/**') }}-${{ github.event.pull_request.head.sha || github.sha }} + js-sdk-package: - name: Builds / JavaScript SDK Package + name: Packages / JavaScript SDK + ICU permissions: contents: read actions: read @@ -1667,8 +1704,6 @@ jobs: - name: Set up Deno uses: ./.github/actions/setup-deno - with: - deno-version: ${{ env.DENO_VERSION }} - name: Build TypeScript SDK package artifacts run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh js-sdk-package @@ -1728,11 +1763,11 @@ jobs: bun .github/scripts/moon-producer-receipt.mts oliphaunt-query-ts:package oliphaunt-query-ts-sdk-package-artifacts src/query/ts native-consumers: - name: Builds / Native Product Consumers - needs: [affected, js-sdk-package, rust-sdk-package, liboliphaunt-native-desktop, broker-runtime, node-direct] + name: Tests / Native Consumers + needs: [affected, js-sdk-package, rust-sdk-package, liboliphaunt-native-desktop-linux, broker-runtime-linux, node-direct] if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && contains(fromJson(needs.affected.outputs.jobs), 'native-consumers') }} runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 45 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -1746,11 +1781,21 @@ jobs: install-workspace: "true" - name: Set up Deno uses: ./.github/actions/setup-deno - with: - deno-version: ${{ env.DENO_VERSION }} - name: Set up Rust - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-integration') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-swift:test-native') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-kotlin:test-native-bindings') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-mobile-bindings:test-native') }} uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + tools: nextest + - name: Set up Swift for native SDK tests + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-swift:test-native') }} + uses: ./.github/actions/setup-swift + - name: Set up Android SDK for Kotlin host tests + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-kotlin:test-native-bindings') }} + uses: ./.github/actions/setup-android + with: + native-tools: "false" + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download Rust SDK package for installed consumer if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1788,13 +1833,13 @@ jobs: name: oliphaunt-query-ts-sdk-package-artifacts path: target/sdk-artifacts/oliphaunt-query-ts - name: Download Linux runtime archive - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-js:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-js:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-integration') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-swift:test-native') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-kotlin:test-native-bindings') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-mobile-bindings:test-native') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: liboliphaunt-native-release-assets-linux-x64-gnu path: target/liboliphaunt/desktop-release-assets/linux-x64-gnu - name: Download Linux broker archive - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-js:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-js:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-integration') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: oliphaunt-broker-release-assets-linux-x64-gnu @@ -1809,7 +1854,7 @@ jobs: if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} run: chmod 0755 target/oliphaunt-rust/release-consumer/oliphaunt-rust-release-consumer - name: Download Linux tools archive - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-integration') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: postgres-tools-native-release-assets-linux-x64-gnu @@ -1826,15 +1871,78 @@ jobs: OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["oliphaunt-rust:test-consumer", "postgres-tools-native:package-assets", "oliphaunt-js:package", "oliphaunt-query-ts:package", "liboliphaunt-native:package-runtime-desktop-target", "oliphaunt-broker:build-release-assets", "oliphaunt-node-direct:build-release-assets"]' run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh native-consumers + wasix-ts-package: + name: Packages / WASIX TypeScript SDK + needs: [affected, checks, tests] + if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'wasix-ts-package') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Moon + uses: ./.github/actions/setup-moon + with: + install-workspace: "true" + + - name: Set up Rust and wasm-pack + uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + tools: wasm-pack@0.15.0 + + - name: Restore browser host compiler cache + id: browser-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/oliphaunt-wasix-ts/host/cargo + key: wasix-browser-${{ runner.os }}-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('src/wasix/browser-host/**') }} + restore-keys: | + wasix-browser-${{ runner.os }}-${{ hashFiles('rust-toolchain.toml') }}- + + - name: Package WASIX TypeScript SDK + run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-ts-package + + - name: Save browser host compiler cache + if: ${{ env.HEAVY_CACHE_SAVE_IF == 'true' && steps.browser-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/oliphaunt-wasix-ts/host/cargo + key: ${{ steps.browser-cache.outputs.cache-primary-key }} + + - name: Upload WASIX TypeScript SDK package artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: oliphaunt-wasix-ts-sdk-package-artifacts + path: target/sdk-artifacts/oliphaunt-wasix-ts + if-no-files-found: error + + - name: Upload WASIX TypeScript consumer inputs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: oliphaunt-wasix-ts-consumer-inputs + path: | + target/sdk-artifacts/oliphaunt-wasix-ts + target/oliphaunt-wasix-ts/package + target/oliphaunt-wasix-ts/host/wasmer-sdk + src/wasix/sdks/ts/lib + if-no-files-found: error + retention-days: 30 + wasix-ts-sdk-package: - name: Builds / WASIX TypeScript SDK + name: Tests / WASIX TypeScript Consumers needs: - affected + - wasix-ts-package - liboliphaunt-wasix-runtime - - wasix-napi - js-sdk-package - - liboliphaunt-wasix-aot - if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && (!contains(fromJson(needs.affected.outputs.jobs), 'wasix-napi') || needs.wasix-napi.result == 'success') && (!contains(fromJson(needs.affected.outputs.jobs), 'js-sdk-package') || needs.js-sdk-package.result == 'success') && (!contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-aot') || needs.liboliphaunt-wasix-aot.result == 'success') && contains(fromJson(needs.affected.outputs.jobs), 'wasix-ts-sdk-package') }} + - wasix-host-linux + if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.wasix-ts-package.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && (!contains(fromJson(needs.affected.outputs.jobs), 'wasix-napi') || needs.wasix-host-linux.result == 'success') && (!contains(fromJson(needs.affected.outputs.jobs), 'js-sdk-package') || needs.js-sdk-package.result == 'success') && (!contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-aot') || needs.wasix-host-linux.result == 'success') && contains(fromJson(needs.affected.outputs.jobs), 'wasix-ts-sdk-package') }} runs-on: ubuntu-24.04 timeout-minutes: 120 steps: @@ -1850,15 +1958,14 @@ jobs: with: install-workspace: "true" - - name: Set up Rust and wasm-pack - uses: ./.github/actions/setup-rust + - name: Download WASIX TypeScript consumer inputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: - tools: wasm-pack@0.15.0 + name: oliphaunt-wasix-ts-consumer-inputs + path: . - name: Set up Deno uses: ./.github/actions/setup-deno - with: - deno-version: ${{ env.DENO_VERSION }} - name: Download same-run portable WASIX outputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1915,21 +2022,14 @@ jobs: name: database-resources-icu-npm path: target/release/npm-packages/oliphaunt-icu - - name: Build, test, and package the WASIX TypeScript SDK + - name: Test WASIX TypeScript consumers env: - OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-portable", "extension-artifacts-wasix:compiler-output", "database-resources:build-wasix-standard", "database-resources:build-wasix-icu", "database-resources:package-icu", "oliphaunt-wasix-napi:build-release-assets", "oliphaunt-wasix-tools-ts:package", "postgres-tools-wasix:package-portable", "postgres-tools-wasix:package-aot"]' + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["oliphaunt-wasix-ts:build", "wasix-browser-host:build", "oliphaunt-wasix-ts:package", "liboliphaunt-wasix:runtime-portable", "extension-artifacts-wasix:compiler-output", "database-resources:build-wasix-standard", "database-resources:build-wasix-icu", "database-resources:package-icu", "oliphaunt-wasix-napi:build-release-assets", "oliphaunt-wasix-tools-ts:package", "postgres-tools-wasix:package-portable", "postgres-tools-wasix:package-aot"]' run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-ts-sdk-package - - name: Upload WASIX TypeScript SDK package artifacts - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['wasix-ts-sdk-package'], 'oliphaunt-wasix-ts:package') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: oliphaunt-wasix-ts-sdk-package-artifacts - path: target/sdk-artifacts/oliphaunt-wasix-ts - if-no-files-found: error wasix-rust-package: - name: Builds / WASIX Rust Binding Package + name: Packages / WASIX Rust Binding needs: - affected - checks @@ -1952,6 +2052,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Build Rust WASIX binding package artifacts run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-rust-package @@ -1973,7 +2075,7 @@ jobs: if-no-files-found: error liboliphaunt-wasix-runtime: - name: Builds / liboliphaunt WASIX Runtime + name: Builds / WASIX Portable Runtime needs: - affected - checks @@ -2023,33 +2125,14 @@ jobs: target/oliphaunt-wasix/wasix-build/work/geos-wasix target/oliphaunt-wasix/wasix-build/work/geos-wasix-build target/oliphaunt-wasix/wasix-build/build - key: wasix-build-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ env.WASMER_LLVM_VERSION }}-${{ env.WASMER_LLVM_LINUX_X64_BYTES }}-${{ env.WASMER_LLVM_LINUX_X64_SHA256 }}-${{ github.event.pull_request.head.sha || github.sha }} + key: wasix-build-v2-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ hashFiles('tools/dev/acquisition.sh', 'src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore', 'src/wasix/runtime/toolchain.toml', 'src/wasix/runtime/assets/build/docker/Dockerfile', 'src/wasix/runtime/assets/build/docker/*.sh', '!src/wasix/runtime/assets/build/docker/*.test.sh') }}-${{ github.event.pull_request.head.sha || github.sha }} restore-keys: | - wasix-build-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ env.WASMER_LLVM_VERSION }}-${{ env.WASMER_LLVM_LINUX_X64_BYTES }}-${{ env.WASMER_LLVM_LINUX_X64_SHA256 }}-${{ github.event.pull_request.base.sha || github.event.before || github.sha }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Build WASIX builder image and save cache - if: ${{ env.HEAVY_CACHE_SAVE_IF == 'true' }} - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f - with: - context: src/wasix/runtime/assets/build/docker - file: src/wasix/runtime/assets/build/docker/Dockerfile - tags: oliphaunt-wasix-wasix-build:ci - load: true - cache-from: type=gha,scope=wasix-builder - cache-to: type=gha,mode=max,scope=wasix-builder,ignore-error=true + wasix-build-v2-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ hashFiles('tools/dev/acquisition.sh', 'src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore', 'src/wasix/runtime/toolchain.toml', 'src/wasix/runtime/assets/build/docker/Dockerfile', 'src/wasix/runtime/assets/build/docker/*.sh', '!src/wasix/runtime/assets/build/docker/*.test.sh') }}- - - name: Build WASIX builder image - if: ${{ env.HEAVY_CACHE_SAVE_IF != 'true' }} - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f + - name: Set up WASIX builder + uses: ./.github/actions/setup-wasix-builder with: - context: src/wasix/runtime/assets/build/docker - file: src/wasix/runtime/assets/build/docker/Dockerfile - tags: oliphaunt-wasix-wasix-build:ci - load: true - cache-from: type=gha,scope=wasix-builder + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Install Wasmer LLVM 22.1 for WASIX cluster-seed generation uses: ./.github/actions/setup-wasmer-llvm @@ -2093,7 +2176,7 @@ jobs: target/oliphaunt-wasix/wasix-build/work/geos-wasix target/oliphaunt-wasix/wasix-build/work/geos-wasix-build target/oliphaunt-wasix/wasix-build/build - key: wasix-build-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ env.WASMER_LLVM_VERSION }}-${{ env.WASMER_LLVM_LINUX_X64_BYTES }}-${{ env.WASMER_LLVM_LINUX_X64_SHA256 }}-${{ github.event.pull_request.head.sha || github.sha }} + key: wasix-build-v2-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ hashFiles('tools/dev/acquisition.sh', 'src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore', 'src/wasix/runtime/toolchain.toml', 'src/wasix/runtime/assets/build/docker/Dockerfile', 'src/wasix/runtime/assets/build/docker/*.sh', '!src/wasix/runtime/assets/build/docker/*.test.sh') }}-${{ github.event.pull_request.head.sha || github.sha }} - name: Upload portable WASIX tools compiler outputs if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-runtime'], 'postgres-tools-wasix:compiler-output') }} @@ -2139,121 +2222,47 @@ jobs: src/wasix/runtime/assets/generated/** if-no-files-found: error + wasix-host-linux: + name: WASIX (Linux) + needs: [affected, liboliphaunt-wasix-runtime, extension-artifacts-wasix] + if: ${{ !cancelled() && !failure() && needs.liboliphaunt-wasix-runtime.result == 'success' && (!contains(fromJson(needs.affected.outputs.jobs), 'extension-artifacts-wasix') || needs.extension-artifacts-wasix.result == 'success') && fromJson(needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix_linux).include[0] != null }} + uses: ./.github/workflows/wasix-host.yml + with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} + matrix: ${{ needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix_linux }} + job-targets: ${{ needs.affected.outputs.job_targets }} + napi-targets: ${{ needs.affected.outputs.wasix_napi_targets }} + llvm-version: ${{ needs.affected.outputs.llvm_version }} + + wasix-host-other: + name: WASIX (Desktop) + needs: [affected, liboliphaunt-wasix-runtime, extension-artifacts-wasix] + if: ${{ !cancelled() && !failure() && needs.liboliphaunt-wasix-runtime.result == 'success' && (!contains(fromJson(needs.affected.outputs.jobs), 'extension-artifacts-wasix') || needs.extension-artifacts-wasix.result == 'success') && fromJson(needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix_other).include[0] != null }} + uses: ./.github/workflows/wasix-host.yml + with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} + matrix: ${{ needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix_other }} + job-targets: ${{ needs.affected.outputs.job_targets }} + napi-targets: ${{ needs.affected.outputs.wasix_napi_targets }} + llvm-version: ${{ needs.affected.outputs.llvm_version }} + liboliphaunt-wasix-aot: - name: Builds / liboliphaunt WASIX AOT (${{ matrix.target_id }}) - needs: - - affected - - liboliphaunt-wasix-runtime - if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-aot') }} - runs-on: ${{ matrix.os }} - timeout-minutes: 180 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix || '{"include":[]}') }} + name: Builds / WASIX AOT + needs: [affected, wasix-host-linux, wasix-host-other] + if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-aot') }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha || github.sha }} - persist-credentials: false - - - name: Set up Moon - uses: ./.github/actions/setup-moon - - - name: Set up Rust - uses: ./.github/actions/setup-rust - - - name: Set up MSVC - if: ${{ runner.os == 'Windows' }} - uses: ./.github/actions/setup-msvc - - - name: Download portable WASIX build outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: liboliphaunt-wasix-runtime-portable - path: . - - - name: Download portable WASIX tools compiler outputs - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:build-aot') }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: postgres-tools-wasix-compiler-output - path: target/postgres-tools/wasix/assets - - - name: Download portable WASIX extension compiler outputs - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'extension-artifacts-wasix:build-aot') }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: extensions-wasix-compiler-output - path: target/extensions/wasix/assets - - - name: Install Wasmer LLVM 22.1 for AOT generation - uses: ./.github/actions/setup-wasmer-llvm - with: - url: ${{ matrix.llvm_url }} - sha256: ${{ matrix.llvm_sha256 }} - bytes: ${{ matrix.llvm_bytes }} - version: ${{ env.WASMER_LLVM_VERSION }} - - - name: Build, validate, and smoke target AOT artifacts + - name: Check selected hosts env: - AOT_TARGET: ${{ matrix.target }} - OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-portable", "postgres-tools-wasix:compiler-output", "extension-artifacts-wasix:compiler-output"]' - run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh liboliphaunt-wasix-aot - - - name: Upload target PostgreSQL WASIX tools - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:package-aot') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: postgres-tools-wasix-release-assets-${{ matrix.target_id }} - path: target/postgres-tools/wasix/release-assets/*-aot-${{ matrix.target_id }}.tar.gz - if-no-files-found: error - - - name: Stage target AOT artifact envelope - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'liboliphaunt-wasix:runtime-aot') }} - env: - AOT_TARGET: ${{ matrix.target }} + RESULTS: ${{ join(needs.*.result, ',') }} run: | - target="${AOT_TARGET:?AOT_TARGET is required}" - source="target/oliphaunt-wasix/aot/$target" - if [ ! -d "$source" ]; then - echo "missing AOT output directory: $source" >&2 - exit 1 - fi - upload="target/oliphaunt-wasix/aot-upload" - rm -rf "$upload" - mkdir -p "$upload/files" - cp -R "$source/." "$upload/files/" - printf '%s\n' "$target" >"$upload/target-triple.txt" - - - name: Upload target artifacts - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'liboliphaunt-wasix:runtime-aot') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: liboliphaunt-wasix-runtime-aot-${{ matrix.target_id }} - path: | - target/oliphaunt-wasix/aot-upload/** - if-no-files-found: error - - - name: Upload target WASIX tools compiler outputs - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:build-aot') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: postgres-tools-wasix-aot-${{ matrix.target_id }} - path: target/postgres-tools/wasix/aot - if-no-files-found: error - - - name: Upload target extension AOT artifacts - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'extension-artifacts-wasix:build-aot') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: liboliphaunt-wasix-extension-aot-${{ matrix.target_id }} - path: target/extensions/wasix/aot-artifacts - if-no-files-found: error + case ",$RESULTS," in + *,failure,*|*,cancelled,*) exit 1 ;; + esac liboliphaunt-wasix-release-assets: - name: Builds / liboliphaunt WASIX Release Assets + name: Packages / WASIX Runtime needs: - affected - liboliphaunt-wasix-runtime @@ -2271,9 +2280,13 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download portable WASIX runtime outputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -2320,7 +2333,7 @@ jobs: if-no-files-found: error wasix-postmaster-portable: - name: Builds / WASIX Postmaster / Portable + qualification + name: Builds / WASIX Postmaster (Portable + Tests) needs: - affected - checks @@ -2339,8 +2352,21 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + - name: Set up WASIX builder + uses: ./.github/actions/setup-wasix-builder + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + # Runtime Cargo commands use the executor workspace's patched dependency config. + cache-workspaces: | + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/wasmer/target + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/postmaster-executor-target + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/postmaster-compiler-target + src/wasix/postmaster/tools/sealed-export-closure -> ../../../../../target/oliphaunt-wasix-postmaster/runtime/sealed-export-closure-target - name: Install Wasmer LLVM 22.1 uses: ./.github/actions/setup-wasmer-llvm @@ -2395,7 +2421,7 @@ jobs: retention-days: 3 wasix-postmaster-target: - name: Builds / WASIX Postmaster / ${{ matrix.target_id }} + qualification + name: Builds / WASIX Postmaster (${{ matrix.target_id }} + Tests) needs: - affected - wasix-postmaster-portable @@ -2418,6 +2444,15 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + cache: ${{ matrix.target_id != 'linux-x64-gnu' }} + # Runtime Cargo commands use the executor workspace's patched dependency config. + cache-workspaces: | + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/wasmer/target + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/postmaster-executor-target + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/postmaster-compiler-target + src/wasix/postmaster/tools/sealed-export-closure -> ../../../../../target/oliphaunt-wasix-postmaster/runtime/sealed-export-closure-target - name: Install portable shell utilities if: ${{ runner.os == 'macOS' }} @@ -2433,6 +2468,7 @@ jobs: - name: Install Wasmer LLVM 22.1 uses: ./.github/actions/setup-wasmer-llvm with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} url: ${{ matrix.llvm_url }} sha256: ${{ matrix.llvm_sha256 }} bytes: ${{ matrix.llvm_bytes }} @@ -2479,7 +2515,7 @@ jobs: if-no-files-found: error wasix-postmaster: - name: Builds / WASIX Postmaster / Aggregate release assets + name: Packages / WASIX Postmaster needs: - affected - wasix-postmaster-target @@ -2547,8 +2583,6 @@ jobs: - name: Set up Deno for packed native tools smoke if: ${{ matrix.shard == 0 }} uses: ./.github/actions/setup-deno - with: - deno-version: ${{ env.DENO_VERSION }} - name: Download same-run Linux native runtime uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -2600,6 +2634,7 @@ jobs: name: native-extension-lifecycle-evidence-${{ matrix.shard }} path: target/native-extension-lifecycle/evidence if-no-files-found: error + overwrite: true retention-days: 30 native-extension-lifecycle-aggregate: @@ -2621,6 +2656,7 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon with: + task-cache: "false" install-workspace: "false" - name: Download all same-run native lifecycle shard receipts @@ -2650,6 +2686,9 @@ jobs: --candidate-tree "$candidate_tree" \ --expected-extensions-csv "$OLIPHAUNT_NATIVE_EXTENSION_PROOF_SQL_NAMES" \ --expected-shard-count "$OLIPHAUNT_NATIVE_EXTENSION_PROOF_SHARD_COUNT" \ + --repository "$GITHUB_REPOSITORY" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ --output target/native-extension-lifecycle/aggregate/output/aggregate-receipt.json - name: Upload aggregate native extension lifecycle evidence @@ -2660,7 +2699,8 @@ jobs: name: native-extension-lifecycle-evidence path: target/native-extension-lifecycle/aggregate if-no-files-found: error - retention-days: 30 + overwrite: true + retention-days: 90 wasix-release-regression: name: E2E / WASIX Release Regression @@ -2668,8 +2708,9 @@ jobs: - affected - extension-artifacts-wasix - liboliphaunt-wasix-runtime - - liboliphaunt-wasix-aot - if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.extension-artifacts-wasix.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && needs.liboliphaunt-wasix-aot.result == 'success' && needs.affected.outputs.wasix_release_regression_required == 'true' }} + - wasix-host-linux + - js-sdk-package + if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.extension-artifacts-wasix.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && needs.wasix-host-linux.result == 'success' && needs.affected.outputs.wasix_release_regression_required == 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 240 steps: @@ -2688,6 +2729,7 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} components: rustfmt - name: Download same-run portable WASIX outputs @@ -2751,12 +2793,28 @@ jobs: mkdir -p "$destination" rsync -a "$raw_target_dir/" "$destination/" + - name: Download same-run WASIX cluster seeds + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['wasix-release-regression'], 'oliphaunt-wasix-rust:test-integration') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: database-resources-wasix-seeds-portable + path: target/database-resources/release-assets + + - name: Download same-run ICU data + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['wasix-release-regression'], 'oliphaunt-wasix-rust:test-integration') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: database-resources-icu-data-portable + path: target/database-resources/release-assets + - name: Stage exact-extension WASIX evidence inputs run: tools/dev/bun.sh src/extensions/artifacts/packages/tools/build-extension-ci-artifacts.mts --all --family wasix --require-wasix - name: Collect WASIX extension evidence id: regression env: + OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ needs.affected.outputs.job_targets }} + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-aot", "extension-artifacts-wasix:build-target", "extension-artifacts-wasix:build-aot", "postgres-tools-wasix:build-aot", "database-resources:build-wasix-standard", "database-resources:build-wasix-icu", "database-resources:package-icu"]' CI_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT: ${{ github.workspace }}/target/extension-artifacts run: | @@ -2827,10 +2885,14 @@ jobs: - name: Set up Android uses: ./.github/actions/setup-android with: + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + expo: "true" native-ccache: "true" - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Reclaim Android mobile build disk run: bash .github/scripts/reclaim-android-mobile-build-disk.sh @@ -2970,6 +3032,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download iOS liboliphaunt target uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -3209,6 +3273,7 @@ jobs: - kotlin-maven-staging - react-native-sdk-package - js-sdk-package + - wasix-ts-package - wasix-ts-sdk-package - native-consumers - wasix-rust-package @@ -3216,7 +3281,6 @@ jobs: - liboliphaunt-wasix-aot - liboliphaunt-wasix-release-assets - wasix-postmaster - - wasix-release-regression - mobile-build-android - mobile-build-ios runs-on: ubuntu-24.04 @@ -3240,13 +3304,6 @@ jobs: GATE_LABEL: selected build jobs run: bun .github/scripts/check-ci-gate.mts selected - - name: Check WASIX release regression - id: wasix_regression_gate - env: - NEEDS_JSON: ${{ toJson(needs) }} - SELECTED_JOBS_JSON: ${{ needs.affected.outputs.wasix_release_regression_required == 'true' && '["wasix-release-regression"]' || '[]' }} - GATE_LABEL: WASIX release regression - run: bun .github/scripts/check-ci-gate.mts selected mobile-e2e-android: name: E2E / Android App @@ -3264,66 +3321,17 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - name: Set up Node and Bun - id: setup_android_e2e_node - uses: ./.github/actions/setup-node-bun - - - name: Reclaim Android emulator disk - id: reclaim_android_emulator_disk - run: bash .github/scripts/reclaim-android-mobile-build-disk.sh - - - name: Set up Android - id: setup_android_e2e - uses: ./.github/actions/setup-android - with: - gradle-cache: "false" - - - name: Set up Maestro - uses: ./.github/actions/setup-maestro - - name: Download Android app artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: react-native-mobile-android-app-android-x86_64 path: target/mobile-build/react-native/android - - name: List Android app artifact - run: find target/mobile-build/react-native/android -maxdepth 2 -type f -print - - - name: Provision runner KVM access - run: | - test -c /dev/kvm - if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then - sudo chmod a+rw /dev/kvm - fi - - - name: Start Android emulator - id: start_android_emulator - env: - OLIPHAUNT_ANDROID_EMULATOR_API: "35" - OLIPHAUNT_ANDROID_EMULATOR_DISK_HEADROOM_MB: "2048" - OLIPHAUNT_ANDROID_EMULATOR_PARTITION_SIZE_MB: "6144" - run: tools/ci/start-android-emulator-ci.sh - - - name: Run Android installed-app E2E - env: - CI_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - OLIPHAUNT_EXPO_ANDROID_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/android - OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE: release - OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE: "0" - OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER: maestro - run: bash src/native/sdks/react-native/tools/mobile-e2e.sh android - - - name: Upload Android E2E reports - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + - name: Test installed app + uses: ./.github/actions/run-mobile-e2e with: - name: react-native-mobile-android-e2e-reports - path: | - target/mobile/react-native/android-e2e/reports - target/mobile/react-native/android-e2e/logs - ${{ runner.temp }}/oliphaunt-android-emulator.log - if-no-files-found: ignore + platform: android + source-sha: ${{ github.event.pull_request.head.sha || github.sha }} mobile-e2e-ios: name: E2E / iOS App @@ -3341,59 +3349,24 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - name: Set up Node and Bun - uses: ./.github/actions/setup-node-bun - - - name: Set up Apple - uses: ./.github/actions/setup-apple - - - name: Set up Maestro - uses: ./.github/actions/setup-maestro - - name: Download iOS app artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: react-native-mobile-ios-app path: target/mobile-build/react-native/ios-transport - - name: Verify and extract iOS app artifact - id: ios_app_transport_verify - run: | - rm -rf target/mobile-build/react-native/ios - bash src/native/sdks/react-native/tools/ios-app-transport.sh verify-extract \ - --transport-dir target/mobile-build/react-native/ios-transport \ - --output-dir target/mobile-build/react-native/ios - - - name: List iOS app artifact - run: find target/mobile-build/react-native/ios -maxdepth 2 -print - - - name: Run iOS installed-app E2E - env: - CI_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - OLIPHAUNT_EXPO_IOS_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/ios - OLIPHAUNT_EXPO_IOS_CONFIGURATION: Release - OLIPHAUNT_EXPO_IOS_SDK: iphonesimulator - OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE: "0" - OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER: maestro - MAESTRO_DRIVER_STARTUP_TIMEOUT: "300000" - run: bash src/native/sdks/react-native/tools/mobile-e2e.sh ios - - - name: Upload iOS E2E reports - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + - name: Test installed app + uses: ./.github/actions/run-mobile-e2e with: - name: react-native-mobile-ios-e2e-reports - path: | - target/mobile/react-native/ios-e2e/reports - target/mobile/react-native/ios-e2e/logs - target/mobile/react-native/ios-e2e/*.log - if-no-files-found: ignore + platform: ios + source-sha: ${{ github.event.pull_request.head.sha || github.sha }} e2e: name: E2E if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.action != 'closed') }} needs: - affected + - wasix-release-regression - native-extension-lifecycle-aggregate - mobile-e2e-android - mobile-e2e-ios @@ -3418,6 +3391,14 @@ jobs: GATE_LABEL: final release execution qualification run: bun .github/scripts/check-ci-gate.mts selected + - name: Check WASIX release regression + id: wasix_regression_gate + env: + NEEDS_JSON: ${{ toJson(needs) }} + SELECTED_JOBS_JSON: ${{ needs.affected.outputs.wasix_release_regression_required == 'true' && '["wasix-release-regression"]' || '[]' }} + GATE_LABEL: WASIX release regression + run: bun .github/scripts/check-ci-gate.mts selected + required: name: Required if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.action != 'closed') }} @@ -3500,6 +3481,13 @@ jobs: name: wasix-release-regression-evidence path: target/qualification/wasix-release-regression-evidence + - name: Download required same-run native evidence + if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'native-extension-lifecycle') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: native-extension-lifecycle-evidence + path: target/qualification/native-extension-lifecycle-evidence + - name: Write exact-SHA qualification record id: qualification_record env: @@ -3508,6 +3496,7 @@ jobs: CI_PLAN_PATH: target/qualification/affected-plan/ci-plan.json WASIX_RELEASE_REGRESSION_REQUIRED: ${{ needs.affected.outputs.wasix_release_regression_required }} WASIX_EVIDENCE_ROOT: target/qualification/wasix-release-regression-evidence + NATIVE_EVIDENCE_ROOT: target/qualification/native-extension-lifecycle-evidence PRODUCER_RECEIPTS_JSON: ${{ format('[{0}]', needs.js-sdk-package.outputs.producer_receipt || '') }} run: bash .github/scripts/release-candidate.sh write diff --git a/.github/workflows/extension-artifacts-native.yml b/.github/workflows/extension-artifacts-native.yml index ba17956e0..7e908dd67 100644 --- a/.github/workflows/extension-artifacts-native.yml +++ b/.github/workflows/extension-artifacts-native.yml @@ -2,6 +2,14 @@ name: extension-artifacts-native on: workflow_call: inputs: + cache-save-if: + description: Resolved cache-save policy from the calling CI workflow. + required: true + type: boolean + phase: + description: all, android-static, or android-package + default: all + type: string matrix: required: true type: string @@ -14,7 +22,6 @@ env: OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ inputs.job-targets }} CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - HEAVY_CACHE_SAVE_IF: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} defaults: run: shell: bash @@ -49,7 +56,7 @@ jobs: uses: ./.github/actions/setup-apple - name: Set up Android - if: ${{ startsWith(matrix.target, 'android-') }} + if: ${{ startsWith(matrix.target, 'android-') && inputs.phase != 'android-package' }} uses: ./.github/actions/setup-android with: gradle-cache: "false" @@ -63,46 +70,76 @@ jobs: run: bash tools/dev/bun.sh test ./tools/packaging/windows-vc-runtime-closure.test.mts - name: Set up Rust + if: ${{ inputs.phase != 'android-package' }} uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} - name: Prepare native compiler cache path - if: ${{ matrix.target == 'ios-xcframework' }} + if: ${{ inputs.phase != 'android-package' && (matrix.target == 'ios-xcframework' || startsWith(matrix.target, 'android-')) }} run: mkdir -p "$CCACHE_DIR" - name: Restore native compiler cache - id: restore_ios_extension_ccache - if: ${{ matrix.target == 'ios-xcframework' }} + id: restore_extension_ccache + if: ${{ inputs.phase != 'android-package' && (matrix.target == 'ios-xcframework' || startsWith(matrix.target, 'android-')) }} uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ env.CCACHE_DIR }} - key: liboliphaunt-native-extension-ccache-v2-${{ matrix.target }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-apple/**', '.github/scripts/setup-native-build-tools.sh', 'src/third-party/postgres/**', 'src/third-party/icu/**', 'src/third-party/openssl/**', 'src/native/runtime/sources/**', 'src/extensions/contracts/**', 'src/extensions/catalog/extensions.source.json', 'src/extensions/catalog/native-components.toml', 'src/extensions/external/postgis/tools/preprocess-sql.mts', 'src/extensions/contrib/postgres18.toml', 'src/extensions/external/*/source.toml', 'src/extensions/external/*/recipe.toml', 'src/extensions/external/*/dependencies/**/source.toml', 'src/extensions/external/*/dependencies/**/recipe.toml', 'src/extensions/external/*/patches/**', 'src/extensions/external/*/dependencies/**/patches/**', 'src/extensions/generated/extensions.catalog.json', 'src/extensions/generated/contrib-build.tsv', 'src/extensions/generated/pgxs-build.tsv', 'src/extensions/generated/mobile/static-extensions.tsv', 'src/extensions/generated/mobile/static-registry.json', 'src/native/runtime/bin/build-*.sh', '!src/native/runtime/bin/*.test.sh', 'src/extensions/artifacts/native/tools/build-windows-extensions.sh', 'src/extensions/artifacts/native/tools/windows-extension-sources.mts', 'src/extensions/external/postgis/tools/windows/**', 'src/native/runtime/bin/build-output.bash', 'src/native/runtime/bin/common.sh', 'src/native/runtime/bin/fetch-pinned-git-checkout.sh', 'src/third-party/icu/tools/build.sh', 'src/native/runtime/bin/mobile-*.sh', 'src/native/runtime/bin/postgis-dependency-cache.sh', 'src/native/runtime/bin/postgres-backend-objects.mk', 'src/native/postgres-tools/crates/tools/Cargo.toml', 'src/native/postgres-tools/crates/tools/build.rs', 'src/native/postgres-tools/crates/tools/src/**', 'src/native/runtime/include/**', 'src/native/runtime/patches/**', 'src/extensions/contrib/portable-uuid/**', 'src/native/runtime/postgres18/**', 'src/native/runtime/src/**') }} + key: liboliphaunt-native-extension-ccache-v2-${{ matrix.target }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-android/**', 'tools/dev/android-sdk.toml', '.github/actions/setup-apple/**', '.github/scripts/setup-native-build-tools.sh', 'src/third-party/postgres/**', 'src/third-party/icu/**', 'src/third-party/openssl/**', 'src/native/runtime/sources/**', 'src/extensions/contracts/**', 'src/extensions/catalog/extensions.source.json', 'src/extensions/catalog/native-components.toml', 'src/extensions/external/postgis/tools/preprocess-sql.mts', 'src/extensions/contrib/postgres18.toml', 'src/extensions/external/*/source.toml', 'src/extensions/external/*/recipe.toml', 'src/extensions/external/*/dependencies/**/source.toml', 'src/extensions/external/*/dependencies/**/recipe.toml', 'src/extensions/external/*/patches/**', 'src/extensions/external/*/dependencies/**/patches/**', 'src/extensions/generated/extensions.catalog.json', 'src/extensions/generated/contrib-build.tsv', 'src/extensions/generated/pgxs-build.tsv', 'src/extensions/generated/mobile/static-extensions.tsv', 'src/extensions/generated/mobile/static-registry.json', 'src/native/runtime/bin/build-*.sh', '!src/native/runtime/bin/*.test.sh', 'src/extensions/artifacts/native/tools/build-windows-extensions.sh', 'src/extensions/artifacts/native/tools/windows-extension-sources.mts', 'src/extensions/external/postgis/tools/windows/**', 'src/native/runtime/bin/build-output.bash', 'src/native/runtime/bin/common.sh', 'src/native/runtime/bin/fetch-pinned-git-checkout.sh', 'src/third-party/icu/tools/build.sh', 'src/native/runtime/bin/mobile-*.sh', 'src/native/runtime/bin/postgis-dependency-cache.sh', 'src/native/runtime/bin/postgres-backend-objects.mk', 'src/native/postgres-tools/crates/tools/Cargo.toml', 'src/native/postgres-tools/crates/tools/build.rs', 'src/native/postgres-tools/crates/tools/src/**', 'src/native/runtime/include/**', 'src/native/runtime/patches/**', 'src/extensions/contrib/portable-uuid/**', 'src/native/runtime/postgres18/**', 'src/native/runtime/src/**') }} restore-keys: | liboliphaunt-native-extension-ccache-v2-${{ matrix.target }}-${{ runner.os }}-${{ runner.arch }}- - name: Configure native compiler cache + if: ${{ inputs.phase != 'android-package' }} run: .github/scripts/setup-native-build-tools.sh + - name: Download Android static archives + if: ${{ inputs.phase == 'android-package' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: extension-static-${{ matrix.target }} + path: target/extension-inputs/static + + - name: Download Linux extension support + if: ${{ inputs.phase == 'android-package' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: extension-linux-support + path: target/extension-inputs/host + + - name: Restore extension package inputs + if: ${{ inputs.phase == 'android-package' }} + run: | + tar -xzf target/extension-inputs/static/static.tar.gz + tar -xzf target/extension-inputs/host/support.tar.gz + - name: Build native exact-extension artifacts timeout-minutes: 120 env: + OLIPHAUNT_EXTENSION_PHASE: ${{ inputs.phase }} OLIPHAUNT_EXTENSION_PRODUCTS: ${{ matrix.extensions_csv }} OLIPHAUNT_EXTENSION_TARGET: ${{ matrix.target }} OLIPHAUNT_BISON: /opt/homebrew/opt/bison/bin/bison - run: .github/scripts/run-planned-moon-job.sh extension-artifacts-native + run: | + if [[ "$OLIPHAUNT_EXTENSION_PHASE" == android-static ]]; then + OLIPHAUNT_RELEASE_FETCH_ASSETS=1 bash src/extensions/artifacts/native/tools/package-release-assets.sh + else + .github/scripts/run-planned-moon-job.sh extension-artifacts-native + fi - name: Validate produced iOS extension carriers id: validate_ios_extension_carriers if: ${{ matrix.target == 'ios-xcframework' }} run: bun src/native/runtime/tools/validate-ios-carrier-zips.mts --root target/extensions/native/release-assets/ios-xcframework - - name: Save bounded iOS exact-extension compiler cache - id: save_ios_extension_ccache - if: ${{ matrix.target == 'ios-xcframework' && env.HEAVY_CACHE_SAVE_IF == 'true' && steps.restore_ios_extension_ccache.outputs.cache-hit != 'true' }} + - name: Save native extension compiler cache + id: save_extension_ccache + if: ${{ (matrix.target == 'ios-xcframework' || startsWith(matrix.target, 'android-')) && inputs.phase != 'android-package' && inputs.cache-save-if && steps.restore_extension_ccache.outputs.cache-hit != 'true' }} continue-on-error: true uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ env.CCACHE_DIR }} - key: liboliphaunt-native-extension-ccache-v2-${{ matrix.target }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-apple/**', '.github/scripts/setup-native-build-tools.sh', 'src/third-party/postgres/**', 'src/third-party/icu/**', 'src/third-party/openssl/**', 'src/native/runtime/sources/**', 'src/extensions/contracts/**', 'src/extensions/catalog/extensions.source.json', 'src/extensions/catalog/native-components.toml', 'src/extensions/external/postgis/tools/preprocess-sql.mts', 'src/extensions/contrib/postgres18.toml', 'src/extensions/external/*/source.toml', 'src/extensions/external/*/recipe.toml', 'src/extensions/external/*/dependencies/**/source.toml', 'src/extensions/external/*/dependencies/**/recipe.toml', 'src/extensions/external/*/patches/**', 'src/extensions/external/*/dependencies/**/patches/**', 'src/extensions/generated/extensions.catalog.json', 'src/extensions/generated/contrib-build.tsv', 'src/extensions/generated/pgxs-build.tsv', 'src/extensions/generated/mobile/static-extensions.tsv', 'src/extensions/generated/mobile/static-registry.json', 'src/native/runtime/bin/build-*.sh', '!src/native/runtime/bin/*.test.sh', 'src/extensions/artifacts/native/tools/build-windows-extensions.sh', 'src/extensions/artifacts/native/tools/windows-extension-sources.mts', 'src/extensions/external/postgis/tools/windows/**', 'src/native/runtime/bin/build-output.bash', 'src/native/runtime/bin/common.sh', 'src/native/runtime/bin/fetch-pinned-git-checkout.sh', 'src/third-party/icu/tools/build.sh', 'src/native/runtime/bin/mobile-*.sh', 'src/native/runtime/bin/postgis-dependency-cache.sh', 'src/native/runtime/bin/postgres-backend-objects.mk', 'src/native/postgres-tools/crates/tools/Cargo.toml', 'src/native/postgres-tools/crates/tools/build.rs', 'src/native/postgres-tools/crates/tools/src/**', 'src/native/runtime/include/**', 'src/native/runtime/patches/**', 'src/extensions/contrib/portable-uuid/**', 'src/native/runtime/postgres18/**', 'src/native/runtime/src/**') }} + key: ${{ steps.restore_extension_ccache.outputs.cache-primary-key }} - name: Show native compiler cache stats if: ${{ always() && runner.os != 'Windows' }} @@ -113,7 +150,40 @@ jobs: echo "ccache was not installed before the build stopped" fi + - name: Pack Android static archives + if: ${{ inputs.phase == 'android-static' }} + env: + EXTENSION_TARGET: ${{ matrix.target }} + run: | + mkdir -p target/extension-inputs + tar -czf target/extension-inputs/static.tar.gz target/liboliphaunt-mobile-extension-release/"$EXTENSION_TARGET"/android-*/out + + - name: Upload Android static archives + if: ${{ inputs.phase == 'android-static' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: extension-static-${{ matrix.target }} + path: target/extension-inputs/static.tar.gz + retention-days: 30 + if-no-files-found: error + + - name: Pack Linux extension support + if: ${{ matrix.target == 'linux-x64-gnu' }} + run: | + mkdir -p target/extension-inputs + tar -czf target/extension-inputs/support.tar.gz target/liboliphaunt-pg18-linux-x64-gnu-extension-release/install + + - name: Upload Linux extension support + if: ${{ matrix.target == 'linux-x64-gnu' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: extension-linux-support + path: target/extension-inputs/support.tar.gz + retention-days: 30 + if-no-files-found: error + - name: Upload native exact-extension artifacts + if: ${{ inputs.phase != 'android-static' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: liboliphaunt-native-extension-artifacts-${{ matrix.target }} diff --git a/.github/workflows/liboliphaunt-native-desktop.yml b/.github/workflows/liboliphaunt-native-desktop.yml index 598118fe4..a84c5ddd1 100644 --- a/.github/workflows/liboliphaunt-native-desktop.yml +++ b/.github/workflows/liboliphaunt-native-desktop.yml @@ -2,6 +2,10 @@ name: liboliphaunt-native-desktop on: workflow_call: inputs: + cache-save-if: + description: Resolved cache-save policy from the calling CI workflow. + required: true + type: boolean matrix: required: true type: string @@ -14,7 +18,6 @@ env: OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ inputs.job-targets }} CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - HEAVY_CACHE_SAVE_IF: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} defaults: run: shell: bash @@ -45,6 +48,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} - name: Set up MSVC if: ${{ runner.os == 'Windows' }} diff --git a/.github/workflows/mobile-e2e.yml b/.github/workflows/mobile-e2e.yml index 341c1ab32..ad5530a98 100644 --- a/.github/workflows/mobile-e2e.yml +++ b/.github/workflows/mobile-e2e.yml @@ -88,22 +88,8 @@ jobs: ref: ${{ needs.resolve.outputs.sha }} persist-credentials: false - - name: Set up Node and Bun - id: setup_android_e2e_node - uses: ./.github/actions/setup-node-bun - - - name: Reclaim Android emulator disk - id: reclaim_android_emulator_disk - run: bash .github/scripts/reclaim-android-mobile-build-disk.sh - - - name: Set up Android - id: setup_android_e2e - uses: ./.github/actions/setup-android - with: - gradle-cache: "false" - - - name: Set up Maestro - uses: ./.github/actions/setup-maestro + - name: Set up Bun + uses: ./.github/actions/setup-bun - name: Download Android app artifact id: download_android_app @@ -122,41 +108,11 @@ jobs: --artifact react-native-mobile-android-app-android-x86_64 find target/mobile-build/react-native/android -maxdepth 2 -type f -print - - name: Provision runner KVM access - run: | - test -c /dev/kvm - if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then - sudo chmod a+rw /dev/kvm - fi - - - name: Start Android emulator - id: start_android_emulator - env: - OLIPHAUNT_ANDROID_EMULATOR_API: "35" - OLIPHAUNT_ANDROID_EMULATOR_DISK_HEADROOM_MB: "2048" - OLIPHAUNT_ANDROID_EMULATOR_PARTITION_SIZE_MB: "6144" - run: tools/ci/start-android-emulator-ci.sh - - - name: Run Android installed-app E2E - id: android_app_e2e - env: - CI_HEAD_SHA: ${{ needs.resolve.outputs.sha }} - OLIPHAUNT_EXPO_ANDROID_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/android - OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE: release - OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE: "0" - OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER: maestro - run: bash src/native/sdks/react-native/tools/mobile-e2e.sh android - - - name: Upload Android E2E reports - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + - name: Test installed app + uses: ./.github/actions/run-mobile-e2e with: - name: react-native-mobile-android-e2e-reports - path: | - target/mobile/react-native/android-e2e/reports - target/mobile/react-native/android-e2e/logs - ${{ runner.temp }}/oliphaunt-android-emulator.log - if-no-files-found: ignore + platform: android + source-sha: ${{ needs.resolve.outputs.sha }} ios: name: ios-installed-app @@ -173,14 +129,8 @@ jobs: ref: ${{ needs.resolve.outputs.sha }} persist-credentials: false - - name: Set up Node and Bun - uses: ./.github/actions/setup-node-bun - - - name: Set up Apple - uses: ./.github/actions/setup-apple - - - name: Set up Maestro - uses: ./.github/actions/setup-maestro + - name: Set up Bun + uses: ./.github/actions/setup-bun - name: Download iOS app artifact id: download_ios_app @@ -198,37 +148,11 @@ jobs: --job Builds \ --artifact react-native-mobile-ios-app - - name: Verify and extract iOS app artifact - id: verify_ios_app_transport - run: | - rm -rf target/mobile-build/react-native/ios - bash src/native/sdks/react-native/tools/ios-app-transport.sh verify-extract \ - --transport-dir target/mobile-build/react-native/ios-transport \ - --output-dir target/mobile-build/react-native/ios - find target/mobile-build/react-native/ios -maxdepth 2 -print - - - name: Run iOS installed-app E2E - id: ios_app_e2e - env: - CI_HEAD_SHA: ${{ needs.resolve.outputs.sha }} - OLIPHAUNT_EXPO_IOS_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/ios - OLIPHAUNT_EXPO_IOS_CONFIGURATION: Release - OLIPHAUNT_EXPO_IOS_SDK: iphonesimulator - OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE: "0" - OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER: maestro - MAESTRO_DRIVER_STARTUP_TIMEOUT: "300000" - run: bash src/native/sdks/react-native/tools/mobile-e2e.sh ios - - - name: Upload iOS E2E reports - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + - name: Test installed app + uses: ./.github/actions/run-mobile-e2e with: - name: react-native-mobile-ios-e2e-reports - path: | - target/mobile/react-native/ios-e2e/reports - target/mobile/react-native/ios-e2e/logs - target/mobile/react-native/ios-e2e/*.log - if-no-files-found: ignore + platform: ios + source-sha: ${{ needs.resolve.outputs.sha }} required: name: E2E diff --git a/.github/workflows/mobile-extension-packages.yml b/.github/workflows/mobile-extension-packages.yml index c814f74df..849260043 100644 --- a/.github/workflows/mobile-extension-packages.yml +++ b/.github/workflows/mobile-extension-packages.yml @@ -2,6 +2,10 @@ name: Mobile extension packages on: workflow_call: inputs: + cache-save-if: + description: Resolved cache-save policy from the calling CI workflow. + required: true + type: boolean family: type: string required: true @@ -23,7 +27,7 @@ defaults: shell: bash jobs: package: - name: Builds / Mobile Extension Packages + name: Packages / Mobile Extensions runs-on: ubuntu-24.04 timeout-minutes: 30 steps: @@ -39,6 +43,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} - name: Download native exact-extension artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 94d58620e..86b38f32e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -204,6 +204,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} REQUIRES_WASIX_EVIDENCE: ${{ steps.release_plan.outputs.requires_wasix_release_regression_evidence }} + REQUIRES_NATIVE_EVIDENCE: ${{ steps.release_plan.outputs.requires_native_extension_lifecycle_evidence }} PRODUCTS_JSON: ${{ steps.release_plan.outputs.products_json }} run: | qualification_args=( @@ -219,6 +220,9 @@ jobs: --artifact artifact-build-plan --artifact oliphaunt-release-candidate ) + if [[ "$REQUIRES_NATIVE_EVIDENCE" == true ]]; then + qualification_args+=(--artifact native-extension-lifecycle-evidence) + fi if [[ "$REQUIRES_WASIX_EVIDENCE" == true ]]; then qualification_args+=(--artifact wasix-release-regression-evidence) fi @@ -290,6 +294,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} REQUIRES_WASIX_EVIDENCE: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).requires_wasix_release_regression_evidence }} + REQUIRES_NATIVE_EVIDENCE: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).requires_native_extension_lifecycle_evidence }} PRODUCTS_JSON: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).products_json }} run: | qualification_args=( @@ -305,6 +310,9 @@ jobs: --artifact artifact-build-plan --artifact oliphaunt-release-candidate ) + if [[ "$REQUIRES_NATIVE_EVIDENCE" == true ]]; then + qualification_args+=(--artifact native-extension-lifecycle-evidence) + fi if [[ "$REQUIRES_WASIX_EVIDENCE" == true ]]; then qualification_args+=(--artifact wasix-release-regression-evidence) fi @@ -354,6 +362,20 @@ jobs: --run-id "$CI_RUN_ID" \ --job "E2E / WASIX Release Regression" \ --artifact wasix-release-regression-evidence + - name: Download required exact-SHA native evidence + if: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).has_release_changes == 'true' && inputs.approval_run_id == '' && fromJSON(needs.plan-candidate.outputs.release_plan).requires_native_extension_lifecycle_evidence == 'true' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + CI_RUN_ID: ${{ steps.ci_qualification.outputs.run_id }} + run: | + bash .github/scripts/download-build-artifacts.sh \ + CI \ + "$RELEASE_HEAD_SHA" \ + target/release-candidate/native-evidence \ + --run-id "$CI_RUN_ID" \ + --job "E2E / Native Extension Lifecycle Evidence" \ + --artifact native-extension-lifecycle-evidence - name: Verify exact-SHA qualification record id: verify_qualification if: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).has_release_changes == 'true' && inputs.approval_run_id == '' }} @@ -361,12 +383,15 @@ jobs: PRODUCTS_JSON: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).products_json }} CI_RUN_ID: ${{ steps.ci_qualification.outputs.run_id }} WASIX_EVIDENCE_REQUIRED: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).requires_wasix_release_regression_evidence }} + NATIVE_EVIDENCE_REQUIRED: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).requires_native_extension_lifecycle_evidence }} run: | bash .github/scripts/release-candidate.sh verify \ target/release-candidate/oliphaunt-release-candidate.json \ --plan target/release-candidate/affected-plan/ci-plan.json \ --qualification-mode release \ --products-json "$PRODUCTS_JSON" \ + --native-evidence-required "$NATIVE_EVIDENCE_REQUIRED" \ + --native-evidence-root target/release-candidate/native-evidence \ --wasix-evidence-required "$WASIX_EVIDENCE_REQUIRED" \ --wasix-evidence-root target/release-candidate/wasix-evidence - name: Require the explicitly selected prepared candidate diff --git a/.github/workflows/wasix-host.yml b/.github/workflows/wasix-host.yml new file mode 100644 index 000000000..05fe30b76 --- /dev/null +++ b/.github/workflows/wasix-host.yml @@ -0,0 +1,183 @@ +name: WASIX host +on: + workflow_call: + inputs: + matrix: + required: true + type: string + job-targets: + required: true + type: string + napi-targets: + required: true + type: string + llvm-version: + required: true + type: string + cache-save-if: + required: true + type: boolean +permissions: + contents: read +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + WASMER_LLVM_VERSION: ${{ inputs.llvm-version }} + OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ inputs.job-targets }} +defaults: + run: + shell: bash +jobs: + build: + name: WASIX host (${{ matrix.target_id }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: ${{ fromJson(inputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Moon + uses: ./.github/actions/setup-moon + + - name: Set up Rust + uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} + + - name: Set up MSVC + if: ${{ runner.os == 'Windows' }} + uses: ./.github/actions/setup-msvc + + - name: Download portable WASIX build outputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: liboliphaunt-wasix-runtime-portable + path: . + + - name: Download portable WASIX tools compiler outputs + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:build-aot') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: postgres-tools-wasix-compiler-output + path: target/postgres-tools/wasix/assets + + - name: Download portable WASIX extension compiler outputs + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'extension-artifacts-wasix:build-aot') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: extensions-wasix-compiler-output + path: target/extensions/wasix/assets + + - name: Install Wasmer LLVM 22.1 for AOT generation + uses: ./.github/actions/setup-wasmer-llvm + with: + cache-save-if: ${{ inputs.cache-save-if }} + url: ${{ matrix.llvm_url }} + sha256: ${{ matrix.llvm_sha256 }} + bytes: ${{ matrix.llvm_bytes }} + version: ${{ env.WASMER_LLVM_VERSION }} + + - name: Build, validate, and smoke target AOT artifacts + env: + AOT_TARGET: ${{ matrix.target }} + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-portable", "postgres-tools-wasix:compiler-output", "extension-artifacts-wasix:compiler-output"]' + run: .github/scripts/run-planned-moon-job.sh liboliphaunt-wasix-aot + + - name: Upload target PostgreSQL WASIX tools + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:package-aot') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: postgres-tools-wasix-release-assets-${{ matrix.target_id }} + path: target/postgres-tools/wasix/release-assets/*-aot-${{ matrix.target_id }}.tar.gz + if-no-files-found: error + + - name: Stage target AOT artifact envelope + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'liboliphaunt-wasix:runtime-aot') }} + env: + AOT_TARGET: ${{ matrix.target }} + run: | + target="${AOT_TARGET:?AOT_TARGET is required}" + source="target/oliphaunt-wasix/aot/$target" + if [ ! -d "$source" ]; then + echo "missing AOT output directory: $source" >&2 + exit 1 + fi + upload="target/oliphaunt-wasix/aot-upload" + rm -rf "$upload" + mkdir -p "$upload/files" + cp -R "$source/." "$upload/files/" + printf '%s\n' "$target" >"$upload/target-triple.txt" + + - name: Upload target artifacts + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'liboliphaunt-wasix:runtime-aot') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: liboliphaunt-wasix-runtime-aot-${{ matrix.target_id }} + path: | + target/oliphaunt-wasix/aot-upload/** + if-no-files-found: error + + - name: Upload target WASIX tools compiler outputs + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:build-aot') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: postgres-tools-wasix-aot-${{ matrix.target_id }} + path: target/postgres-tools/wasix/aot + if-no-files-found: error + + - name: Upload target extension AOT artifacts + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'extension-artifacts-wasix:build-aot') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: liboliphaunt-wasix-extension-aot-${{ matrix.target_id }} + path: target/extensions/wasix/aot-artifacts + if-no-files-found: error + + - name: Download same-run WASIX exact-extension outputs + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + pattern: liboliphaunt-wasix-extension-artifacts-* + path: target/extensions/wasix/release-assets + merge-multiple: true + + - name: Set up macOS smoke timeout + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) && runner.os == 'macOS' }} + shell: bash + run: brew list coreutils >/dev/null 2>&1 || HOMEBREW_NO_AUTO_UPDATE=1 brew install coreutils + + - name: Build WASIX Node-API release assets + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) }} + shell: bash + env: + OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD: "1" + OLIPHAUNT_ICU_DATA_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/wasix-build/work/icu-wasix/share/icu + OLIPHAUNT_WASM_GENERATED_AOT_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/aot + OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT: ${{ github.workspace }}/target/extension-artifacts + OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/assets + OLIPHAUNT_WASIX_NAPI_ARTIFACT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["extension-artifacts-wasix:build-target", "extension-artifacts-wasix:build-aot", "liboliphaunt-wasix:runtime-aot"]' + run: .github/scripts/run-planned-moon-job.sh wasix-napi + + - name: Upload WASIX Node-API release assets + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: oliphaunt-wasix-napi-release-assets-${{ matrix.target_id }} + path: target/oliphaunt-wasix-napi/release-assets + if-no-files-found: error + + - name: Upload WASIX Node-API optional npm package + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: oliphaunt-wasix-napi-npm-package-${{ matrix.target_id }} + path: target/oliphaunt-wasix-napi/npm-packages/*.tgz + if-no-files-found: error diff --git a/src/benchmarks/perf/moon.yml b/src/benchmarks/perf/moon.yml index f491892d3..58823c1f9 100644 --- a/src/benchmarks/perf/moon.yml +++ b/src/benchmarks/perf/moon.yml @@ -118,6 +118,7 @@ tasks: bun src/wasix/sdks/ts/tools/stage-host.mts bash src/wasix/sdks/ts/tools/integration/smoke-browser.sh --benchmark deps: + - "oliphaunt-query-ts:build" - "perf-tools:wasix-plan" - "liboliphaunt-wasix:runtime-portable" - "database-resources:build-wasix-standard" diff --git a/src/benchmarks/wasix/README.md b/src/benchmarks/wasix/README.md index bf2b43d41..75ca62313 100644 --- a/src/benchmarks/wasix/README.md +++ b/src/benchmarks/wasix/README.md @@ -16,7 +16,7 @@ and `sdk-worker` ownership values, so Promise shape is not mistaken for main-thread safety. ```sh -bun run --cwd src/wasix/sdks/ts package:build +moon run oliphaunt-wasix-ts:package bash src/wasix/sdks/ts/tools/integration/smoke-browser.sh --benchmark # or: moon run perf-tools:wasix-browser-measure ``` @@ -43,7 +43,7 @@ the built SDK tree, every harness source, and the installed PGlite closure. For a harness smoke check without a full sample set, run: ```sh -bun run --cwd src/wasix/sdks/ts package:build +moon run oliphaunt-wasix-ts:package bash src/wasix/sdks/ts/tools/integration/smoke-browser.sh --benchmark --quick ``` diff --git a/src/docs/internal/CI_RELEASE_PROCESS_AUDIT_2026-09-29.md b/src/docs/internal/CI_RELEASE_PROCESS_AUDIT_2026-09-29.md new file mode 100644 index 000000000..2f65d9a33 --- /dev/null +++ b/src/docs/internal/CI_RELEASE_PROCESS_AUDIT_2026-09-29.md @@ -0,0 +1,1153 @@ +# CI inventory and correction backlog — 2026-09-29 + +Audit baseline: `origin/main` at `32ce7b29510b74333e799601b69a71fd28122e80` +(#225). Evidence, inventories and timings below describe that baseline. The +implementation status is recorded first. Checked items mean implemented and +locally checked; they do not assert hosted qualification. Local timing comparisons +are identified separately from hosted end-to-end savings. + +## First correction batch — local implementation + +Branch: `f0rr0/fix-ci-cache-reuse`. This batch changes cache/setup plumbing; +product selection, compiler flags, runtime tests and release admission remain +unchanged. Hosted warm-run timings are still required before claiming savings. + +| Finding | Implemented | Remaining verification/work | +| --- | --- | --- | +| CI-06 | Postmaster maps its four Cargo output directories from checked-in workspaces, without including their parent | Measure hosted restore/save and warm compilation. Use the executor's patched dependency context for runtime metadata; preserve upstream dependency-only cache policy | +| CI-07 | Android ABIs now use the existing target-scoped, bounded native extension ccache restore/save path | Measure warm hit rate and produced-artifact equivalence; no desktop cache expansion | +| CI-08 | One caller-resolved save policy reaches Rust, LLVM, Gradle and all four reusable producer/package workflows | Verify manual opt-in/opt-out on GitHub; source/action checks pass locally | +| CI-13 | Both normal and replay iOS installed-app jobs disable PostgreSQL build-tool installation | Android provisioning separation is implemented in the second batch; simulator/Xcode verification remains enabled | +| CI-16 | Deleted the unused summary action | Mobile execution-definition consolidation is implemented in the second batch | +| CI-19 | Normal WASIX and Postmaster use one cached builder setup with the existing recipe label, context and cache scope, before Postmaster compilation | Measure hosted Buildx reuse; shared acquisition deadlines now bound source/bootstrap retries | + +Local validation includes the workflow/security/planner gate, real pinned +Postmaster source preparation and locked Cargo metadata for its cache owners, +metadata collection before generated dependencies exist, shared builder-label +reuse, and the pinned Rust-cache cleanup on fixture output using the configured +paths. The affected artifact-packaging, native-extension packaging and WASIX TS +unit tasks, plus CI-tool formatting/lint, passed locally. The failed Wasmer-root +metadata probe identified why cache metadata must use the executor's patched +dependency configuration; that probe does not indicate a runtime-build regression. + +## Second correction batch — local implementation + +The changes retain product coverage and release admission. Existing Moon tasks, +source fetchers, artifact validators and platform tools own the work; there is no +new CI framework. Same-run artifact transfers preserve their existing validation. + +| Findings | Implemented | Remaining proof or limitation | +| --- | --- | --- | +| CI-02 | Timeout fixtures separate preparation from the deliberate child timeout and cover delayed startup plus an expired deadline | Full release-tool suite passes; no production retry or timeout relaxation | +| CI-03/12/13 | Rust and Deno defaults come from their manifests; Android uses its SDK manifest; Expo's installed React Native version catalog must match the explicitly provisioned Expo NDK | Native NDK remains unchanged. Clean hosted Gradle build must confirm no implicit download. Replay installs no native compiler/CMake; source checks request only their capabilities | +| CI-04/05 | Evidence generation uses all five required modes; the existing uncached WASIX regression task owns its producer dependencies and fresh observations | Truncated materialization evidence is rejected. Planner retains the existing requirement that portable WASIX production receives lifecycle qualification, without repeating its four producer dependencies | +| CI-09/23 | Per-host WASIX AOT and Node-API work share a reusable workflow; Linux consumers wait only for Linux; browser/TS packaging starts after source gates | Final all-host aggregates remain. manylinux keeps its container boundary; no incompatible Cargo cache reuse is claimed | +| CI-11 | WASIX lifecycle is grouped under E2E, alongside native lifecycle | Candidate-bound native release evidence is completed in the third batch | +| CI-14 | Matrix labels use short capability names; full selected tasks appear in the job summary | Product/aggregate display names are completed in the fifth batch; protocol IDs stay stable | +| CI-16/26 | CI and replay use one installed-app action. Maestro and its installer/flow are removed; continuous current-launch logs feed the existing structured receipt validator | Failure, crash, capture death, app death and stale PASS rejection are tested. Actual iOS simulator and Android emulator runs remain required | +| CI-17 | Browser-host Git and crate sources use the shared bounded, exact-pin fetcher and safe archive extraction | Live exact sources, transport fault tests and the full browser/TS package build pass; acquisition now precedes runtime-dependent consumers | +| CI-20 | Producer input groups exclude unrelated Markdown and unit fixtures; SDK README packaging does not select runtime compilation | Actual Moon affected-selection regressions pass. Postmaster's shipped README gets a cheap producer in the third batch | +| CI-21 | WASIX compilation cache has a compatible fallback. Reuse requires the current Moon input hash and verified compiler-output checksums; source preparation, staging and profile validation still run | Cold/warm, corrupted output and changed-input fault tests pass. Full hosted compiler output and warm timing still need validation | +| CI-22 | Android static compilation overlaps one Linux support producer. Both ABI packagers consume that same-run support output instead of compiling Linux again | SQL-only package fixture passes and missing inputs fail closed. Actual Android native artifacts still need hosted qualification | +| CI-24 | The existing transfer runner submits dependency-ready batches to Moon, allowing independent tasks to overlap | Real pinned Moon fixture proves sibling overlap, join ordering and no transferred-producer replay; existing task-failure checks pass | +| CI-25 | A frozen, validated iOS carrier can replace only the unchanged React Native package metadata dependency in ordinary affected runs | Changed native inputs, missing/corrupt cache and explicit product/full qualification retain producers. Same-SHA empty diffs are handled without invoking Moon on empty stdin | + +New intermediate handoffs retain 30 days so failed jobs can be rerun throughout +GitHub's [supported rerun window](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/re-run-workflows-and-jobs). +Release-input/proof retention is unchanged. Browser-host Cargo state has its own +source/toolchain-keyed cache, and its built host files travel with SDK consumer +inputs so consumers do not silently compile the browser host again. + +Local verification covers the exact pinned workflow/security gate, actual Moon +planning and transfer execution, Android installer fault cases, native extension +packaging, WASIX build orchestration, React Native and WASIX TypeScript owner +checks, extension metadata/evidence checks, source-fetch fault cases, and the full +release-tool suite. The browser host compiled and the TypeScript npm package +passed its package validator (`oliphaunt-wasix-ts:package`, 7m55s locally). The +mobile receipt tests and actual Moon transfer fixture also pass under a locally +built GNU Bash 3.2.57; this checks shell compatibility, not macOS platform APIs. +Test logs are in `/tmp/oliphaunt-ci-cache-fixes` for this local +session. A complete Linux workflow run cannot establish Apple/Windows/device +behavior: those platform runs and cache hit/timing measurements remain outstanding. +No new GitHub run or exact-SHA `Qualified` result is claimed for this working tree. + +| Verification command | Local result | +| --- | --- | +| `bash tools/ci/check-workflows.sh` with pinned Moon 2.5.4 | Passed actionlint/security checks, 66 planner tests, six artifact-transfer tests, cache/setup tests and real Moon scheduling fixtures | +| `bash tools/release/release-check.sh` | Passed metadata and release implementation checks, including delayed-start timeout fixtures | +| `moon run oliphaunt-react-native:test` and relevant format/lint/typecheck tasks | Passed; actual Apple transport checks remain macOS-only | +| `moon run oliphaunt-wasix-ts:test oliphaunt-wasix-ts:typecheck oliphaunt-wasix-ts:format-check` | Passed, including 352 SDK tests | +| `moon run oliphaunt-wasix-ts:package` | Compiled the pinned browser host, built the SDK and validated the npm archive | +| `bash src/third-party/tools/source-fetch-core.test.sh` and `bash src/third-party/tools/fetch-sources.sh production-all --validate-only` | Passed exact-pin, retry/failover, archive validation and checkout-preservation checks | + +## Third correction batch — rebuilds, release proof and coverage + +| Findings | Implemented | Verification and limit | +| --- | --- | --- | +| CI-01 | All five previously unwired SDK runtime tasks now have hosted execution owners: four in native consumers and WASIX resources in the WASIX regression job. They consume existing same-run artifacts, require positive test execution, and remain uncached | Rust SQL/runtime, mobile broker, Swift, Kotlin and both WASIX seed resource tests pass locally against real payloads. These Linux host tests do not replace device/platform qualification | +| CI-10 | Coalesced four identical Cargo feature/target configurations and removed a duplicate executor library compilation already covered by the complete product executor suite | Invocation/selection comparison preserves all previous selections while reducing 25 Cargo invocations to 20. Distinct compiler and compiler-free feature profiles remain separate; hosted compile savings are unmeasured | +| CI-11 | Native aggregate/shard proof is bound into the release candidate and revalidated before native extension-carrier publication, including runtime-owned contrib. Aggregate proof has the same 90-day retention as WASIX, includes all shard receipts, and supports reruns | Candidate write/verify and tamper fixtures pass, including wrong source/run, missing shards, incomplete published extension coverage and modified evidence. SDK-only releases retain their existing consumer qualification | +| CI-20 | Postmaster's shipped README has a cheap declared producer. Release assembly consumes its output, while prose-only affected CI avoids the runtime build and regression closure | Actual Moon affected/full-release planning tests pass. Runtime/compiler source changes still select their original producers | + +The native suites exposed an unused `pg_config` preflight requirement that did +not match shipped runtime archives; the shared preflight now requires only the +actual runtime inputs. A runnable staging fixture checks missing inputs, +inherited-path isolation, optional tools/broker, cleanup and exit propagation. +Swift preparation no longer regenerates bindings already owned by its Moon +dependency; the standalone Swift wrapper retains explicit generation. Kotlin's +runtime task similarly reuses the declared binding producer. + +Local checks include the full release-tool suite, candidate/receipt tests, +workflow/security and actual Moon planner/transfer checks, and the five real +SDK runtime tasks. The Postmaster source/fault suite and native evidence owner +tests also pass. The complete patched Wasmer compiler/runtime suite still +requires hosted qualification; the Cargo selection comparison proves retained +selections, not their runtime outcomes. Native payloads were rebuilt and packaged from this working +tree. WASIX resource tests used the unchanged runtime/resources downloaded from +successful baseline CI run `36567197390` at `32ce7b2`; that local exercise is not +exact-SHA qualification of these changes. Logs are under +`/tmp/oliphaunt-ci-remaining`. Full hosted platform runs and cold/warm timing +remain required before claiming a measured CI speedup. + +## Fourth correction batch — acquisition deadlines + +Source fetching and repository-owned bootstrap downloads now use one shared +shell deadline. Retries, mirrors, lock waits and dependent requests spend the +same budget. APT update/install defaults to 15 minutes, complete source scopes +to 30 minutes, and individual source pins to 15 minutes. The +[maintainer pattern and budget table](../maintainers/testing.md#acquisition-deadlines) +define the scope, override, cleanup and package-manager boundaries. + +Verification passed: `source-inputs:test`, `dev-tools:test`, +`dev-tools:test-mobile-setup`, `ci-tools:test`, `ci-workflows:llvm-install-unit`, +WASIX orchestration/installer tests, Postmaster builder identity tests and the +complete pinned workflow gate. Deadline fixtures cover final-attempt expiry, +shared retry/mirror budgets, lock ownership, child termination, cancellation, +APT update-to-install admission and preservation of valid Android packages. +Helper and PostgreSQL transport tests also pass under GNU Bash 3.2.57 on Linux. +Live WASIX sources fetched and verified exact pins; the final Docker recipe +completed APT in 56.2 seconds, installed the pinned compiler assets and passed +compiler smoke/version checks. Its warm build reused every layer, and the +recipe label passed Postmaster validation. Actual Moon affected queries select +all helper consumers. Logs: `/tmp/oliphaunt-acquisition-*`. Hosted platform and +wall-clock qualification remain outstanding; no release qualification is claimed. + +## Fifth correction batch — platform review and remaining efficiency + +The acquisition review corrected GNU timer discovery when Windows System32's +`timeout.exe` shadows Git Bash's Coreutils executable. It prefers `gtimeout`, +then a verified GNU `timeout`, then `/usr/bin/timeout`. Android setup checks this +prerequisite before inspecting or repairing SDK caches, and its macOS action +installs Coreutils when needed. Missing timer support must not masquerade as +corrupt command-line tools. The browser-host patch loop now works on Bash 3.2 +and propagates failure to read its patch list. + +| Findings | Implemented | Verification and limit | +| --- | --- | --- | +| CI-14 | Visible jobs distinguish Builds, Packages, Tests and E2E; mobile ABI labels identify seed production, mixed JavaScript/ICU work is explicit, and runtime labels use WASIX | Stable job IDs, aggregate gates and release/replay job-name references are preserved. The public `wasm_target` dispatch input remains compatible | +| CI-15 | Eleven planning, proof-aggregate and artifact-consuming finalizer jobs skip Moon task-output restore/save | Real graph checks prove no normally cached local task subtree remains in these jobs. The transfer adapter runs artifact-dependent work with `MOON_CACHE=off`. Verified tool archives still use their existing cache | +| CI-18 | Independent planner observations run through four native `xargs` workers, each with its own output file | All 93 parsed JSON outputs matched the serial run. Local elapsed time fell from 92.60 to 59.32 seconds (36%); this is the observation phase, not total CI wall time. A fault check rejects failed workers and covers paths containing spaces | + +The pinned workflow gate passed, including 66 planner tests, seven artifact +transfer/policy tests, scheduling fixtures and workflow/security checks. +Timer discovery, retry/deadline and process-cleanup fixtures pass on Bash 3.2.57 +on Linux. The Android installer and bootstrap installer fixtures also pass +with child shells using Bash 3.2. These are shell compatibility checks, not +Windows, macOS, iOS simulator or Android emulator qualification. + +The source-acquisition owner task passed again against the reviewed helper +(3m19s). A real Docker build completed its pinned APT transaction in 61.4 seconds +and compiler acquisition in 15.4 seconds, then verified wasixcc 0.4.3, Clang +21.1.2 and Binaryen 130. These runs followed a local machine restart that +interrupted the earlier build attempts; interrupted attempts are not counted +as passing checks. The full browser-host build also passed with Bash 3.2.57 +selected for the script and child shells, including real source acquisition, +patch application, Rust compilation, WASM optimization and Rollup (7m40s). + +A fresh read-only cache inventory found 284 Moon entries totaling 291.6 MB, +including 128 under 1 KB. Rust caches totaled 8,197.2 MB and LLVM 1,065.1 MB. +No remote caches were deleted. Retention/pruning of those larger caches still +needs warm restore/save and hit-rate measurements. Baseline successful source +jobs took roughly 23–201 seconds, with setup often dominant; warm timings alone +do not establish safe cold-run deadlines. Uncached release checks still consume +Git/release history. An isolated offline Cargo probe confirmed that packaging a +dependency first does not let a later standalone package resolve its unpublished +version; selecting both crates in the same invocation succeeds. Retain that +multi-crate staging, rather than deleting the repeated dependency selections. + +### Still open + +- CI-10: measure remaining distinct Postmaster feature/profile costs before + changing their guarantees or overlapping portable and host compilation. +- CI-15: budget/prune the large compiler caches using actual warm restore/save + size and hit rates; jobs without reusable Moon outputs now skip that cache. +- CI-18: measure cold source-group setup costs before changing group deadlines; + prove release-check inputs before caching them. Cargo's unpublished dependency + staging remains necessary; consolidate package ownership only if measured cost + justifies changing that graph. +- CI-19: measure hosted cache reuse and acquisition tails against the implemented + budgets; source/bootstrap transactions now share deadlines across retries and mirrors. +- Hosted verification: compare equivalent cold/warm runs and unchanged artifact + contracts, including both Android ABIs, iOS simulator and all WASIX hosts. + +## Assessment + +The second pass found larger structural waste than the first: duplicate +toolchain and host-runtime builds, affected selection that promotes documentation +or unit-test changes into full compiler pipelines, and whole-platform barriers +between otherwise independent producers and consumers. The backlog now contains +**26 findings**, including eight new structural findings, CI-19 through CI-26. + +For wall time, work on both long chains: Postmaster and WASIX runtime/AOT/SDK. +Fixing only the longest job moves the bottleneck rather than removing it. +For ordinary PR feedback, narrow compilation inputs before tuning small tests. +For runner cost, remove duplicate host builds and ineffective caches. These +three objectives overlap, but their savings must not be added together. + +The highest-return order is: + +1. Share the already-cached WASIX Docker builder with Postmaster; fix its Rust + cache mapping and the WASIX compilation-cache restore policy. +2. Stop documentation and isolated unit-test changes from scheduling unrelated + compilation; separate package changes from compiler input changes. +3. Remove cross-host WASIX barriers and start independent browser-host work + early; overlap Postmaster host compilation only after measuring cache fixes. +4. Reuse Android's duplicate Linux support build and avoid serializing it with + independent Android compilation; persist the useful extension compiler cache. +5. Remove Maestro's status-label-only role from SDK smoke, preserving exact-launch + receipts and failure detection. Address Android-to-iOS package coupling. + +At the audit baseline, correctness fixes CI-01 through CI-05 were also needed. +CI-01 through CI-05 are now implemented; hosted platform qualification remains. +Several recent failures detected real product/release defects; indiscriminate +test deletion would lose useful proof without addressing the largest delays. + +There is no justification here for a new CI framework, another product +registry, a generalized evidence service, a new retry service, or wholesale +workflow generation. Moon, product manifests, the existing candidate record, +and GitHub Actions already provide the required pieces. + +## Scope and evidence + +Inspected all seven workflows, their local actions and execution adapters, +the expanded Moon project/task graph, product/SDK task definitions, release +admission and artifact transfers, and representative underlying scripts/tests. +The inventory contains **56 Moon projects, 373 tasks across 45 task-owning +projects, 27 release manifest entries, and 15 local composite actions**. +The seven workflow files define 76 job entries before matrix expansion and +reusable-workflow calls. These are different counts, not competing inventories. + +An exhaustive current-tree source matrix selects 122 check targets in 21 groups, +3 policy targets, and 54 test targets in 17 groups. Actual affected/product runs +select fewer. There are 33 `ci-` task mappings, including native lifecycle. + +Downloaded the latest 100 workflow records and inspected the failed job +inventories for every failed CI/Release run in the September 25–29 subset. +That subset contained 53 runs: CI had 14 successes, 8 failures, 14 cancellations, +12 skipped runs and one running run; Release had two successes and two failures. +**This is not a flake rate.** These were different commits and scopes; +cancellations include ordinary supersession/merge cancellation, and skipped +runs include the deliberate PR-close tombstones. + +Timing figures use GitHub job `started_at`/`completed_at`, summed without OS +billing multipliers. They are runner-minutes, not billed minutes or CPU time. +Latest-attempt job lists can mix retained successes with rerun jobs. The clean +successful first-attempt run below is the cost baseline; the rerun's elapsed +hours are not treated as continuous computation. + +| Run | Purpose / result | Executed jobs | Runner-minutes | Wall time | +| --- | --- | ---: | ---: | ---: | +| [36492755110](https://github.com/f0rr0/oliphaunt/actions/runs/36492755110) | Merged release candidate `7b192697`; success | 105 | 800.8 | 122.0 min | +| [36480008083](https://github.com/f0rr0/oliphaunt/actions/runs/36480008083) | Corresponding release PR; success | 104 | 764.7 | About 95 min | +| [36521743783](https://github.com/f0rr0/oliphaunt/actions/runs/36521743783) | Focused Apple release-workflow fix; success | 9 | 4.3 | About 4 min | +| [36523497369](https://github.com/f0rr0/oliphaunt/actions/runs/36523497369) | Full manual `34e49318`; Android recovered, qualification receipt failed | Mixed attempts | Not a comparable single-attempt sample | Excluded | +| [36567197390](https://github.com/f0rr0/oliphaunt/actions/runs/36567197390) | Current main `32ce7b29`, manual full qualification | Running during collection | Excluded from complete-run totals | Excluded | + +The baseline's work distribution was: + +| Responsibility | Jobs, including aggregates | Runner-minutes | +| --- | ---: | ---: | +| Planning | 2 | 1.0 | +| Checks | 15 | 23.3 | +| Tests | 14 | 27.3 | +| Builds, packaging and build-owned consumer checks | 64 | 713.2 | +| E2E | 8 | 35.7 | +| Required + Qualified | 2 | 0.3 | + +Native extension producers accounted for **225.8 minutes** across seven targets; +Postmaster portable/target/finalization jobs for **147.2 minutes**. Together +they account for roughly 47% of baseline runner time. Optimizing small JSON +receipt validators will not materially change that bill. + +## Second-pass wall-time analysis + +Times below are minutes since the start of successful run `36492755110`, not +durations that can be summed across overlapping jobs. + +| Chain / checkpoint | Started | Finished | What controls the next stage | +| --- | ---: | ---: | --- | +| Cheap checks/tests ready | — | 6.6 | Heavy producers can start | +| Postmaster portable + qualification | 6.6 | 78.7 | Every Postmaster host waits for the whole job | +| Postmaster macOS + qualification | 78.8 | 120.8 | Longest host; final aggregation follows | +| WASIX portable producer | 10.7 | 54.1 | Runtime, tools, extensions and seeds share one producer job | +| WASIX AOT hosts | 54.2 | 78.1 | Linux x64 finished at 67.5; consumers wait for Windows | +| WASIX Node-API hosts | 78.1 | 91.7 | Linux x64 finished at 84.6; TS waits for Windows | +| WASIX Linux regression | 78.7 | 96.2 | Linux-only proof starts after the AOT matrix barrier | +| WASIX TS package + consumers | 91.8 | 102.4 | Includes independent 6m34s browser-host compilation | +| iOS app / installed E2E | 49.1 / 62.9 | 62.7 / 73.9 | Extension carriers, app build, simulator/driver startup | +| Android app / installed E2E | 52.5 / 62.5 | 62.5 / 65.1 | Slowest Android extension producer then app build | +| Final qualification | — | 122.0 | All selected branches must succeed | + +**The ceiling matters:** even deleting Postmaster's entire chain would leave +the WASIX TS branch finishing at minute 102.4, plus final gates. That is only +about 19 minutes of possible total improvement in this run. Postmaster's +147.2 runner-minutes are not 147.2 minutes of wall-time savings. + +| Priority | Structural correction | Measured exposure | Scope of benefit / limitation | +| --- | --- | --- | --- | +| First | Postmaster builder reuse, CI-19 | APT layer 39m52s; normal WASIX cached image setup 24s | Removes a demonstrated long-tail setup risk. Other observed APT layers were only 59–64s; do not promise 40 minutes on every run | +| First | Select work by actual inputs, CI-20 | README edits select full Postmaster or WASIX pipelines | Avoids whole expensive branches on focused PRs; no reduction for a legitimate full-product change | +| First | Effective compiler caches, CI-06/07/08/21 | Warm downstream WASIX outputs still preceded by 12m23s core compilation | Shortens both long branches; validate cache compatibility and cold behavior | +| Next | Host-specific WASIX dependencies, CI-09/23 | 10.6-minute Linux AOT wait, then another 7.2-minute TS wait | Reduces the second-longest chain; waits overlap other work and are not directly additive | +| Next | Android host support reuse/overlap, CI-22 | Linux support rebuilds took 24m49s and 20m06s across the two ABI jobs | Large compute saving; waiting for Linux before cross-compiling would preserve most wall time | +| Next | Remove status-label UI driver, CI-26 | iOS app passed at 23:38:12; Maestro finished at 23:42:50 | Roughly 4m38s avoidable tail in this installed-app sample; not the full-run critical path | +| After those | Transfer adapter scheduling, CI-24 | Windows AOT's serial step took 20m38s | Some independent tasks could overlap; CPU and compiler locks limit gains | +| Focused Android work | Decouple unchanged iOS carrier metadata, CI-25 | Real Android Kotlin edit selects iOS runtime production | Avoids unrelated Apple work; was not Android's limiting input in the full baseline | + +The portable producer itself holds core output for roughly another 26 minutes +while extension/tool work finishes. That is a scheduling observation, not a +26-minute savings estimate: full consumers need those extensions, and splitting +the producer carelessly duplicates its Docker/compiler workspace. Fix input +selection, caches and host barriers before adding more portable build jobs. + +Postmaster is a published GitHub-assets product: +[v0.1.0](https://github.com/f0rr0/oliphaunt/releases/tag/liboliphaunt-wasix-postmaster-v0.1.0) +contains Linux x64/arm64 and macOS arm64 carriers. It is not an SDK-registry +package, but that does not make it unreleased. Removing its required coverage +would be a separate product-support decision, not a CI optimization assumed by +this audit. + +## Workflow and ownership inventory + +| Workflow | Trigger / role | Owner and important boundary | +| --- | --- | --- | +| [ci.yml](../../../.github/workflows/ci.yml) — 3,522 lines, 60 job entries | PR, merge group, main push, manual qualification | Moon chooses task scope; Actions provisions hosts and transfers artifacts; `Required` aggregates selected work; `Qualified` records eligible exact-main proof | +| [release.yml](../../../.github/workflows/release.yml) — 1,855 lines, 8 jobs | Manual prepare or publish | Release tools select products, request missing qualification, freeze one candidate, publish it, verify public delivery, refresh docs | +| [extension-artifacts-native.yml](../../../.github/workflows/extension-artifacts-native.yml) | Reusable target producer | `extension-artifacts-native:build-target`; four caller partitions, seven full-matrix targets | +| [liboliphaunt-native-desktop.yml](../../../.github/workflows/liboliphaunt-native-desktop.yml) | Reusable desktop producer | Native runtime/tools/resources tasks; Linux and other-host partitions | +| [broker-runtime.yml](../../../.github/workflows/broker-runtime.yml) | Reusable broker producer | `oliphaunt-broker:build-release-assets`; Linux and other-host partitions | +| [mobile-extension-packages.yml](../../../.github/workflows/mobile-extension-packages.yml) | Reusable packaging | `extension-packages:package-mobile`; separate Android/iOS callers | +| [mobile-e2e.yml](../../../.github/workflows/mobile-e2e.yml) | Manual/reusable diagnostic replay | Resolves existing exact-SHA app artifacts, then installs/tests them; not an automatic second CI run | + +The local action layer owns tool installation: Moon, Node, Bun, Deno, Rust and +Rust tools, Swift, Apple, Android, MSVC, Wasmer LLVM, Maestro and npm publishing. +`setup-node-bun` composes existing installers. `collect-ci-summary` has no callers. + +### Product/lane inventory + +The following accounts for every current CI task-to-job mapping. Stable IDs +are shown because human display names are not consistently descriptive. + +| Job ID or related IDs | Product task owners / actual work | Target or consumer boundary | +| --- | --- | --- | +| `liboliphaunt-native-desktop` | `liboliphaunt-native` build/package/artifact tests; `postgres-tools-native` package/tests; native standard/ICU seeds | Linux x64/arm64, macOS arm64, Windows x64 | +| `liboliphaunt-native-android` | Native runtime build/package | Android arm64-v8a and x86_64 | +| `liboliphaunt-native-ios` | Native runtime XCFramework build/package | iOS device/simulator slices | +| `liboliphaunt-native-android-abi`, `liboliphaunt-native-ios-abi` | ABI finalization **and database-resource seed production** | Mobile compatibility domains; iOS seed work can require macOS | +| `liboliphaunt-native-release-assets` | Native asset aggregation | All selected native targets | +| `extension-artifacts-native` | Exact extension compilation/package | Four desktop targets, two Android ABIs, iOS XCFramework | +| `extension-artifacts-wasix` | Exact portable extension archive packaging | Portable WASIX | +| `extension-packages` | Public Cargo/npm/Maven/Apple extension carriers | Selected extensions and target families | +| `mobile-extension-packages` | Mobile extension carriers | Android/iOS partitioned | +| `broker-runtime`, `broker-release-assets` | Broker binaries and aggregate assets | Four desktop targets | +| `node-direct`, `node-direct-release-assets` | Native Node addon, built-artifact qualification, aggregate assets | Four desktop targets | +| `liboliphaunt-wasix-runtime` | Core portable runtime, extension/tool compiler outputs, portable tools, WASIX standard/ICU seeds | Linux-hosted portable production; multiple product owners share the job | +| `liboliphaunt-wasix-aot` | Core/tool/extension AOT; Rust SDK and pgwire host execution | Four native AOT hosts | +| `liboliphaunt-wasix-release-assets` | WASIX asset aggregation | Portable plus selected AOT hosts | +| `wasix-napi`, `wasix-napi-release-assets` | WASIX Node-API addon and aggregation | Four desktop targets | +| `wasix-postmaster` | Portable inputs, host carriers and release finalization | Separate portable job; Linux x64/arm64 and macOS arm64 host jobs; runtime-patch/regression/recovery tasks also explicitly invoked in YAML | +| `rust-sdk-package` | Native SDK, build helper, native bindings, broker crate and query crate; packed consumer compilation | Linux; package closure before execution | +| `wasix-rust-package` | WASIX Rust SDK and pgwire packages/consumer compilation | Linux; runtime tests are elsewhere | +| `js-sdk-package` | Native TS, query TS, WASIX tools TS **and ICU data** packages | Linux; heterogeneous products under a JS SDK label | +| `wasix-ts-sdk-package` | Browser-host build, TS package, browser/native consumers; PostgreSQL tools browser/native consumers | Linux; artifact-dependent tests and independent packaging combined | +| `swift-bindings`, `swift-sdk-package` | XCFramework production on macOS; source/carrier assembly on Linux | Swift SDK | +| `kotlin-sdk-package`, `kotlin-maven-staging` | Maven SDK packaging, staging | Kotlin/JVM/Android SDK | +| `react-native-sdk-package` | npm package and clean package consumer | React Native SDK | +| `native-consumers` | Native TS Node/Bun/Deno, Rust installed SDK, broker consumer; release-only published-dependency TS variant | Canonical Linux x64 artifacts | +| `native-extension-lifecycle` | Native direct/broker/server, restart and backup/restore | Linux x64; three full-catalog shards plus aggregate receipt | +| `mobile-build-android`, `mobile-build-ios` | Expo installed-app production | Android x86_64 emulator app; iOS simulator app | + +Additional hosted execution: `wasix-release-regression`, native lifecycle +aggregation and Android/iOS installed-app E2E. WASIX regression is an explicit +workflow call to the evidence collector, not a `ci-`-mapped Moon root. + +Product selection is not a second directory-based planner: keep release product +identity in product manifests, execution/data dependencies in Moon, and runner +and transport topology in Actions. `tools/ci` and `.github/scripts` currently +split adapter ownership; neither should acquire independent SDK policy. + +### SDK proof comparison + +| SDK/surface | Source proof | Packed/installed proof | Missing or misleading hosted ownership | +| --- | --- | --- | --- | +| Native Rust | Rust unit/doc tests, formatting, Clippy | Packed crate compile plus real installed direct/broker consumer | `test-integration` not hosted; environment-dependent tests can return without execution in the ordinary test lane | +| WASIX Rust | Rust unit/doc/public API tests | Packed crate compile; AOT tests on four hosts; Linux exhaustive extension regression | `test-integration` resource tests not hosted; `test-regression` task bypassed by collector | +| Native TS | TypeScript/unit/format/lint | Packed SDK on Node/Bun/Deno, native direct/broker execution | Linux execution is intentional, not proof on every desktop host | +| WASIX TS | TypeScript/unit/format/lint | Packed browser and native consumers, tools consumers | Independent package build delayed behind runtime/addon matrices | +| Swift | Linux portable tests; Apple platform tests; iOS broker compile check | XCFramework/source package; React Native app exercises the native backend | Public Swift `NativeRuntimeTests` suite disabled without env; `test-native` not hosted | +| Kotlin | JVM and Android unit tests, plugin check, lint/format | Maven package; React Native app exercises the native backend | `NativeBindingsTest` assumes prepared PGDATA and skips otherwise; owner task not hosted | +| React Native | TS/unit, C++ tests, codegen | Packed consumer, actual Android/iOS app builds and installed E2E | Retain distinct device/platform checks; these caught actual product failures | +| Shared query / native bindings / mobile bindings | Shared protocol tests and Rust/TS source gates | Included in dependent SDK package closures | Mobile bindings' native broker roundtrip task not hosted | + +Five currently unreachable integration tasks are confirmed against the expanded +task graph and the actual check/test matrix writer, including explicit +Postmaster workflow roots: + +- `oliphaunt-rust:test-integration` +- `oliphaunt-swift:test-native` +- `oliphaunt-kotlin:test-native-bindings` +- `oliphaunt-mobile-bindings:test-native` +- `oliphaunt-wasix-rust:test-integration` + +This does **not** mean the runtimes or mobile broker are untested: installed +consumers, native extension lifecycle, AOT, and mobile app E2E already execute. +It means those particular SDK guarantees are not provided by their existing +dedicated tests. Compare coverage before wiring overlapping suites wholesale. +The broker's separate local `test-integration` is not another such gap: its +`postgres_client` test is exercised by the hosted `test-consumer` wrapper. + +Docs generation and public-site checks have a separate delivery path, including +the release docs-refresh job. `docs:test` and `docs:test-package` are not selected +by the quality-tag adapter. Vercel's live deployment configuration was not +audited; do not claim GitHub's `Required` proves that site's build or availability. + +## Correction inventory + +P1: correctness/reliability or a demonstrated large recurring cost. P2: measured +efficiency/maintenance improvement. P3: small cleanup. Owners below are code +ownership areas, not invented individual assignees. + +### CI-01 — P1 — Make runtime test ownership truthful + +- [x] Wire the unique guarantees from the five tasks above into their SDK's + artifact-consuming lane; remove or mark genuinely redundant local aliases. +- [x] Change selection tests to assert final executable roots/dependency closure, + not merely that a task occurs in the affected-task inventory. + +**Evidence:** `write-affected-moon-target-matrices.mts` selects quality/unit, +coverage and quality/static/format/smoke tags. The five tasks have neither +qualifying tags nor hosted `ci-` roots/dependents. The planner test at +`tools/ci/ci-plan-node-products.test.mts:448` asserts native Rust/Swift affected +selection, which passes without proving hosted execution. Swift's native suite +is env-disabled, Kotlin uses `assumeTrue`, and WASIX resource tests are ignored +unless explicitly requested. + +**Owner:** SDK Moon tasks and `tools/ci`. **Done when:** full qualification and +relevant SDK-only changes select actual execution; a missing runtime fails the +runtime lane; the selected test count is positive; existing same-run artifacts +are consumed without compiling a second runtime. Preserve cheap unit lanes. + +### CI-02 — P1 — Remove the deadline fixture's scheduling race + +- [x] Give fixture preparation a separate budget from the deliberate child + timeout; start/coordinate the timeout test after the child can actually run. + Keep a separate deterministic assertion for an already-expired deadline. + +**Evidence:** `tools/release/public-consumer-smoke.test.mts:103` grants the timeout +scenario 2 seconds including process startup/staging. The shell floors the +deadline to seconds and returns 1 if it expires before starting the child, +whereas the fixture expects 124 and a child PID. The full local release suite +has failed here repeatedly while isolated execution passed. This audit's +unchanged control passed; adding a two-second delay only before reading the +timeout scenario's context reproduced exit 1 and `shared public-consumer +deadline reached`. + +**Owner:** release tools. **Done when:** delayed startup cannot turn this into a +false failure, while hanging descendants are still terminated and genuine +consumer failure is never retried. Do not delete the timeout/credential tests or +solve this by retrying the entire release suite. + +### CI-03 — P1 — Provision the Android app's actual NDK before its build + +- [x] Resolve Expo/React Native's intended Gradle `ndkVersion` and provision + that exact version through the existing bounded SDK installer before Gradle. +- [x] Use one shared pin if platform compatibility allows it; otherwise make the + native-runtime and Expo-app versions two explicit, owned requirements. + +**Evidence:** [job 109274401218](https://github.com/f0rr0/oliphaunt/actions/runs/36523497369/job/109274401218) +successfully installed setup's `27.0.12077973`, then Gradle independently fetched +`27.1.12297006` and failed with `Archive is not a ZIP archive`. The same app +succeeded on rerun. `ANDROID_NDK_HOME` does not set the generated app's Gradle +version. The existing installer already has bounded retries; this second +download bypasses it. + +**Owner:** Android setup and Expo runner. **Done when:** a clean runner has every +declared NDK before compilation and Gradle does no surprise NDK installation. +Do not silently change the released native ABI/toolchain merely to match Expo. +Android documents explicit selection via +[`android.ndkVersion`](https://developer.android.com/studio/projects/install-ndk). + +### CI-04 — P1 — Use one WASIX lifecycle mode contract + +- [x] Make the evidence table and qualification validator consume the same + required mode definition, including materialization and physical backup/restore. + +**Evidence:** `src/extensions/tools/extension-evidence.mts` declares five modes, +but `evidenceMatrix()` enumerates four and omits materialization. A real report +with materialization removed passes the table's current-evidence check and +fails candidate validation. The collector normally records all five, and the +candidate validator catches the omission: this is validator drift, not proof +that an incomplete release was published. + +**Owner:** extension evidence contract, consumed by release qualification. +**Done when:** the same truncated report is rejected by both entry points; normal +reports pass; deliberate compatibility with frozen old report schemas remains +explicit. Reuse the existing contract module rather than add another registry. + +### CI-05 — P1 — Put WASIX qualification dependencies in the task graph + +- [x] Make the existing `oliphaunt-wasix-rust:test-regression` task the execution + owner for fresh lifecycle observations and let the workflow collect its receipt. +- [x] Remove the hand-maintained four-producer insertion in + `requiredTasksForAffected()` after graph-based selection replaces it. + +**Evidence:** the collector runs `runtime-smoke.sh regression` directly, bypassing +the existing Moon task. `tools/ci/ci_plan.mts:298` separately inserts runtime AOT, +extension portable/AOT, and tools AOT dependencies. #222 repaired an actual +missed-input/selection failure here. The next new input should not require +another synchronized YAML/planner/script list. + +**Owner:** WASIX Rust task + CI adapter. **Done when:** changing a required producer +changes the graph once, planner/transfer tests follow it, and qualification gets +fresh positive observations rather than a cached success without a receipt. + +### CI-06 — P1 — Stop discarding Postmaster's compiler cache + +- [x] Configure the existing Rust cache action for the actual Postmaster Cargo + workspaces/target directories; expose its existing mapping input through + `setup-rust` if needed. Include the browser-host custom Cargo directory in the + same audit. Keep saved compiler state bounded and keyed to toolchain/source + inputs; continue verifying produced artifacts. + +**Evidence:** setup currently supplies `. -> target`. Postmaster compiles under +`target/oliphaunt-wasix-postmaster/runtime/...`, but that intermediate directory +is not a Cargo target root. The pinned cache action interprets it as a profile, +keeps only `build`, `.fingerprint` and `deps`, and removes its `runtime` child. +A fixture executing the pinned cleanup code confirmed that the nested compiled +artifact is deleted. The portable job restored an approximately 89 MB cache, +then spent **62m49s** in `runtime-build`; the cache saved afterward was still +approximately 89 MB. macOS runtime compilation took **27m42s**. + +**Owner:** Rust setup and Postmaster/browser-host producers. **Done when:** a +second clean hosted run restores the intended compiler dependencies and shows +a material reduction in compilation time, with correct rebuilds after pin, +patch, compiler or target changes. Do not claim the whole 62 minutes is saved: +that task also performs other work. The native cache action already supports +[workspace/target mappings](https://github.com/Swatinem/rust-cache); +the inspected cleanup implementation is +[pinned here](https://github.com/Swatinem/rust-cache/blob/e18b497796c12c097a38f9edb9d0641fb99eee32/src/cleanup.ts). + +### CI-07 — P1 — Persist compiler state for expensive native extension lanes + +- [x] Extend the existing bounded extension ccache restore/save pattern first + to Android; measure other targets before expanding it. + +**Evidence:** the reusable extension workflow persists ccache only for +`ios-xcframework`, despite configuring it for other targets. The successful +Android arm64 extension job took **44.8 minutes**; ccache reported **73 hits / +6,168 cacheable calls (1.18%)**, 6,095 misses, and only about 0.1 GB used of its +2 GB limit. Android x86_64 took 37.7 minutes. This is a stronger optimization +candidate than deleting short regression tests. + +**Owner:** native extension producers. **Done when:** unchanged-source warm runs +reuse a bounded cache, publish identical valid outputs and record hit/miss +statistics; compiler/source changes remain safe. Budget storage alongside CI-15. +Do not transfer final product qualification from a previous source commit. + +### CI-08 — P2 — Honor manual cache-save policy in reusable workflows + +- [x] Pass the parent workflow's resolved cache-save choice to reusable + producers instead of recomputing it as push-only inside them. + +**Evidence:** CI exposes `save_heavy_caches` for manual main qualification and +computes `HEAVY_CACHE_SAVE_IF` accordingly. `extension-artifacts-native.yml` +redefines it as `event == push && ref == main`; the manual opt-in cannot save +its iOS extension cache. The `setup-rust` wrapper likewise does not expose every +lower-level cache option, so each caller's effective policy needs checking. + +**Owner:** CI reusable-workflow interfaces. **Done when:** main push, manual +opt-in, manual opt-out and PR cases have tested, consistent save behavior. + +### CI-09 — P2 — Start independent WASIX packaging earlier + +- [x] Run browser-host compilation and TS package production after source + checks, then feed the package to artifact-dependent consumers. +- [x] Narrow Linux-only consumers' dependencies to Linux producers where the + gain justifies the existing reusable-workflow partition pattern. + +**Evidence:** the baseline's Linux AOT completed at about minute 67.5, but WASIX +regression waited for Windows AOT until minute 78.1: roughly 10.6 minutes of +unnecessary waiting for that consumer. Linux Node-API finished at minute 84.6; +the TS job waited for Windows until minute 91.8, then spent **6m34s compiling +the browser host** before package/browser tests. The host build has no runtime +artifact dependency. Native Linux consumers also depend on desktop aggregates +despite existing Linux partitions. + +**Owner:** WASIX SDK and CI topology. **Done when:** the independent package +producer begins after cheap gates, each consumer waits only for bytes it uses, +and aggregate release gates still require every selected platform. These gains +improve feedback; they do not add directly to total wall-time savings while +Postmaster remains the critical path. Do not create a job per tiny task. + +### CI-10 — P2 — Reduce Postmaster compile variants before cutting behavior tests + +- [x] Consolidate equivalent Cargo feature/profile invocations in + `src/wasix/postmaster/wasmer/tests.sh`; keep genuinely different feature tests. +- [ ] Measure remaining distinct feature/profile compilation costs on cold and + warm hosted runs. +- [ ] After fixing cache layout, evaluate starting independent native host + compilation before portable guest qualification completes, only if the + remaining critical-path gain warrants the added artifact transfer. + +**Evidence:** portable `runtime-patch-tests` took **23m31s**, alongside +`runtime-build` at 62m49s. The script invokes many filtered Cargo runs with +several repeated feature sets. The critical path was approximately 6.6 minutes +to start Postmaster, 72.1 portable-job minutes, 42.0 macOS-target minutes, then +aggregation. macOS's actual initdb stress and backend-wave stress were only +1m20s and 50s; immediate recovery took 34s. Deleting those tests first targets +the wrong cost and loses reliability coverage. + +**Owner:** Postmaster. **Done when:** timings distinguish compilation from test +execution, equivalent invocations are grouped without losing named tests, and +warm/cold measurements justify any topology change. No promised percentage +saving until measured. + +Second-pass correction: the 62m49s `runtime-build` duration was **not mostly +Rust compilation**. Its Docker APT layer alone took 39m52s; see CI-19. Fixing +the Rust cache alone would not remove that delay. Portable tasks also overlap, +so their individual durations must not be summed into a job duration. + +### CI-11 — P2 — Make release evidence consistent without weakening admission + +- [x] Decide and document which product guarantees need a candidate-bound + receipt, then bind native lifecycle proof through the existing candidate + record if native extensions require the same release-level guarantee. +- [x] Put WASIX regression under the E2E aggregate, alongside native lifecycle, + while preserving the independent release-product requirement check. + +**Evidence:** native lifecycle already validates exact source/tree, selected +extension/shard coverage, artifact hashes, direct/broker/server and lifecycle +PASS records. The real report had 39 extensions, three shards and 46 consumed +artifact hashes. It is enforced by CI but is not bound/replayed by publication +like WASIX evidence. Native evidence retention is 30 days, versus 90 for WASIX +and the candidate. WASIX is displayed as E2E but actually required by `Builds`. + +The generic gate proves the jobs the plan selected; it does not independently +prove that the plan selected every product-required job. A synthetic incomplete +plan passed generic candidate coverage and was rejected by the separate WASIX +release requirement. Preserve that independent check. Runtime-specific receipt +contents can differ; shared source/run/attempt/digest binding should not. + +**Owner:** release qualification + extension owners. **Done when:** omission, +wrong source/run, future attempts and missing modes fail consistently, retained +successful jobs from earlier attempts still work, and required evidence lasts +for the supported approval window. JSON revalidation is cheap and is not a +second lifecycle execution. + +### CI-12 — P2 — Remove duplicated authoritative toolchain pins + +- [x] Read Rust's default from `rust-toolchain.toml` and Android defaults from + `tools/dev/android-sdk.toml` instead of repeating them in action inputs. +- [x] Review Node/Bun/Deno/npm/LLVM pin consumers for the same pattern; distinguish + intentional fixture values from authoritative live configuration. + +**Evidence:** Rust `1.93.1` appears in both the root toolchain and +`setup-rust-tools` default. Android's NDK/CMake/API defaults repeat the TOML. +`ci.yml` defines `NPM_VERSION` without a consumer in that workflow; release +publishing has its own live definition. Deno's caller and verified installer +also share a version contract. Manual parallel edits are avoidable. + +**Owner:** tool installers. **Done when:** bumping the owner pin updates the +effective local and hosted setup; fixture pins remain explicit test inputs; +verification still rejects wrong versions. Do not substitute mutable latest. + +### CI-13 — P2 — Install only the capabilities a lane consumes + +- [x] Separate Android compilation setup from Java/Gradle checks and emulator + execution using the existing setup action's narrow inputs/capabilities. +- [x] Disable Apple build-dependency installation in installed-app replay when + it only needs Xcode/simulator tools. + +**Evidence:** `setup-android` always provisions NDK and CMake. It is also used for +Kotlin formatting and installed-APK E2E, which do not compile native code. +Apple replay calls the general Apple action, whose build-dependency setup is +already optional. These are unnecessary download/failure opportunities even +when hosted images make the observed setup fast. + +**Owner:** platform setup actions and capability tags. **Done when:** formatting +and APK replay succeed without native compiler installation, compilation still +receives its exact toolchain, and no second general provisioning layer appears. + +### CI-14 — P2 — Make display names describe work and keep protocol IDs stable + +- [x] Use short capability/owner group labels with full Moon targets in the job + summary; do not concatenate every expanded project/task title. +- [x] Rename visible heterogeneous jobs and aggregates consistently: runtime + producers, packages, consumers, lifecycle, qualification. Clarify mobile ABI + jobs' seed-production responsibility and the mixed JS/ICU package job. +- [x] Use WASIX in human-facing runtime names; retain `.wasm` and upstream + WebAssembly target terminology where technically correct. Migrate the public + `wasm_target` dispatch input deliberately if renamed (retained for compatibility). + +**Evidence:** a successful check job name was **384 characters**. Names include +`Builds / broker-runtime`, human-readable names, reusable caller prefixes, and +E2E under Builds. `Checks / Policy` combines release metadata, workflow behavior +and broker license auditing. These make failures harder to locate, without +adding useful proof. + +**Owner:** CI adapter/display labels. **Done when:** names are short and stable, +the exact selected targets remain visible, and consumers of `Required`, +`Qualified`, `Builds` and other protocol identities continue to work. + +### CI-15 — P2 — Budget caches and artifact retention by purpose + +- [x] Skip Moon task-output cache transfers for jobs with no reusable local + tasks; retain verified tool archive caching and validate the task graph. +- [ ] Fix directory effectiveness first, then bound/retire low-value cache + entries using observed size and warm-run savings. +- [ ] Retain release inputs/proof for the supported window; give disposable + intermediate/debug artifacts an explicit shorter policy where replay permits. + +**Evidence:** the cache snapshot had **258 entries / 10.68 GB**: Rust caches +8.29 GB, Linux Wasmer LLVM 1.07 GB, 207 Moon entries only 0.23 GB. Numerous Moon +entries are nearly empty; the largest storage opportunity is not their count. +The baseline stored **114 artifacts / 2.07 GB**, including repeated envelopes +of native/iOS inputs and packaged outputs, many with default 90-day retention. +Some copies are necessary fan-out/release inputs; don't delete them by name. + +**Owner:** CI artifact/cache consumers. **Done when:** each retained artifact has +a consumer and retention reason; warm-run benefits exceed restore/save costs; +expired evidence fails explicitly. Actual eviction/billing limits were not +queried, so 10.68 GB alone does not establish paid usage or thrashing. GitHub +documents [cache limits, immutability and eviction](https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching). + +### CI-16 — P2 — Consolidate mobile execution definitions and drop dead setup + +- [x] Share the installed-app execution/setup definition between main CI and + manual mobile replay while retaining same-run versus explicit-run artifact + resolution at their respective boundaries. +- [x] Delete the unused `.github/actions/collect-ci-summary/action.yml` unless + a real caller is introduced as part of CI-14. + +**Evidence:** Android/iOS steps, environment values, emulator configuration, +transport handling and report upload are maintained both in `ci.yml` and +`mobile-e2e.yml`. The latter has no workflow caller today but remains a useful +manual diagnostic entry point. It is not currently an automatic duplicate run. +`collect-ci-summary` has no references and merely prints command suggestions. + +**Owner:** mobile CI adapter. **Done when:** normal CI and replay run the same +installed-app assertions, replay does not rebuild apps, and platform setup fixes +have one owner. Prefer one existing reusable workflow or a small composite +action; do not add both and a generator. + +### CI-17 — P2 — Move fragile source acquisition out of late consumer work + +- [x] Reuse the existing exact-pin source acquisition behavior for the WASIX + browser host's upstream Git/crate downloads, with bounded retries and complete + validation before package/consumer work. + +**Evidence:** `src/wasix/browser-host/build-sdk.sh` owns its own Git fetch and two +one-shot `curl` downloads; they execute after the long runtime/addon dependency +chain. Digests and commits are checked, which is good. This audit did not +observe a browser-host network failure; this is a concrete unprotected path, +not a claimed measured flake rate. + +**Owner:** browser-host producer, using `src/third-party` acquisition helpers. +**Done when:** transient download failures recover within a bound, invalid bytes +fail without replacing valid output, and exhausted failures occur in the +producer lane. No blanket test retries or unverifiable fallback source. + +### CI-18 — P3 — Right-size source groups and deadlines after the major fixes + +- [x] Run independent planner observations concurrently with a bounded native + worker pool; retain output equivalence and fail the gate on a worker error. +- [ ] Keep capability grouping, but measure setup versus execution before + changing group size; use bounded category-appropriate job deadlines. +- [ ] Review always-uncached deterministic release checks and duplicate package + prerequisite invocations only after their complete inputs are declared. + +**Evidence:** generic check/test jobs all allow 90 minutes although the baseline's +slowest source groups finished in roughly 6 minutes. Grouping is alphabetical, +up to eight static/four unit targets. `release-tools:test` and `metadata` are +uncached; most product unit checks are cached. Broker crate packaging invokes +Cargo packaging of query/bindings after declaring their package dependencies. +These are real cleanup candidates but much smaller than compiler costs. + +**Owner:** source-task owners and CI capability adapter. **Done when:** a stuck +short source check stops promptly, cold setup has enough budget, and cached +checks invalidate on every input they actually consume. Don't add a historical +duration scheduler, force serial formatting before every unit test, or remove +valid cold-package checks to save seconds. + +### CI-19 — P1 — Reuse the existing WASIX builder in Postmaster + +- [x] Use the existing Buildx/GHA cache setup for both WASIX consumers, preserving + Postmaster's recipe label and immutable recipe identity. +- [x] Prepare and validate the toolchain before expensive runtime compilation; + bound the overall acquisition transaction as well as individual retries. + `tools/dev/acquisition.sh` provides the common pattern and fault checks; + [acquisition deadlines](../maintainers/testing.md#acquisition-deadlines) + documents owner budgets and boundaries. A real cold Docker build completed + the pinned TLS-verified APT transaction in 65.6 seconds and verified all + compiler assets and their versions. + +**Evidence:** `fresh_ensure_docker_image` in +`src/wasix/postmaster/lib/common.sh:1694` invokes a separate raw Docker build of +the same `src/wasix/runtime/assets/build/docker` recipe. Postmaster's workflow +does not restore the `wasix-builder` cache already used by the normal WASIX +producer. The sysroot step invokes this after six serial Cargo binary builds +in `src/wasix/postmaster/wasmer/bin/build-runtime.sh`. + +In [the baseline portable job](https://github.com/f0rr0/oliphaunt/actions/runs/36492755110/job/109167195616), +the APT layer took **2,392.3 seconds**, including 133 MB downloaded in 23m16s. +The regular WASIX job's cached builder setup took **24 seconds**. The same +Postmaster APT layer took **64.1 seconds** on the release PR and **59.2 seconds** +on the current-main run. This is a demonstrated bad tail, not a typical +40-minute saving. The old install helper combined outer retries with APT retries +and per-request timeouts without a transaction-wide deadline. It now bounds +update, install and retry waits together to 15 minutes by default. + +**Owner:** shared WASIX build-tool setup + Postmaster. **Done when:** both jobs +restore a compatible builder, a recipe change invalidates it, and Postmaster's +recipe-label validation still rejects stale images. Merely retagging the current +normal image is insufficient: it lacks Postmaster's required label. Use the +existing setup/cache policy; no registry, published builder product or new +promotion service. Docker's [GHA cache backend](https://docs.docker.com/build/cache/backends/gha/) +already supports the scoped reuse needed here. + +### CI-20 — P1 — Stop turning documentation and unit-test edits into compiler work + +- [x] Narrow producer inputs and distinguish source compilation, package content, + unit fixtures and runtime qualification in the existing Moon task definitions. +- [x] Add small affected-selection regressions for these concrete boundaries. + +**Evidence:** the pinned Moon query plus the actual `jobs-for-affected` planner, +run with one changed file at a time, selected: + +| Changed existing file | Heavy jobs selected | +| --- | --- | +| `src/wasix/postmaster/README.md` | Full Postmaster portable and host pipeline | +| `src/wasix/postmaster/lib/durable-publication.test.mts` | Full Postmaster pipeline in addition to unit work | +| `src/wasix/sdks/ts/README.md` | WASIX portable, AOT, Node-API, extensions and TS consumers | +| `src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh` | WASIX and Postmaster, plus native extension producers/packages | + +Postmaster's carrier/portable/package inputs include broad project globs; the +runtime Docker input directory includes its tests. The planner then expands +required producer closures. Selecting the Postmaster job starts its explicitly +listed heavy workflow roots regardless of which narrow source edit caused it. + +**Owner:** product Moon inputs first, CI planner only where the job envelope +cannot represent the resulting scope. **Done when:** a README still repackages +any package containing it, and a unit-test edit still runs that test, without +invalidating unrelated compilers. Compiler recipes, patches, source pins, +catalogs and real runtime tests must retain their full dependencies. Do not +solve this with repository-wide docs ignores or by skipping package integrity. +Selection alone does not prove all tasks rebuilt; Postmaster's uncached heavy +tasks and CI-21 show why these selections are nevertheless expensive today. + +### CI-21 — P1 — Let WASIX reuse compatible compiler state across light commits + +- [x] Add a compatible restore fallback to the existing compilation cache, + scoped to all required compiler/profile inputs and trusted cache provenance. +- [x] Avoid running the uncached compiler prerequisite when a complete, + validated matching output closure can be restored. + +**Evidence:** `ci.yml`'s `Restore WASIX compilation cache` uses the head SHA as +its key and only the PR base / push-before / current SHA as its restore key. +There is no compatible prefix fallback. Manual dispatch searches the current +SHA twice. An intervening cheap workflow-only commit that saved no compilation +cache therefore breaks reuse of the previous heavy build. + +The baseline saved a roughly 296 MB compilation cache for `7b192697`; current +main's manual run searched only `32ce7b29`, never that prior compatible key. +This establishes a lookup defect, not proof that the older entry could never +have been evicted. In the current-main WASIX log, `compiler-output` still took +**12m23s**, although downstream portable runtime, seeds, tools and extension +outputs restored in milliseconds. Its task hash matched the baseline. +`src/wasix/runtime/moon.yml:129` deliberately disables compiler-output caching +because generated container Makefiles reference `/work`; dependencies still +execute before cached downstream tasks. + +**Owner:** WASIX compilation task/cache contract. **Done when:** an unchanged +compiler input after a docs/workflow commit or manual dispatch gets useful +reuse; changing sources/toolchain/flags cannot reuse incompatible output. +Keep exact-current-source qualification. Do not simply enable Moon caching on +the entire absolute-path compiler tree or accept old release receipts. + +### CI-22 — P1 — Remove duplicate Linux builds from Android extension production + +- [x] Reuse one matching Linux extension-support output across both Android + ABIs, preferably from the existing Linux extension producer. +- [x] Let independent Android compilation proceed before the support input is + needed for packaging; measure bounded overlap against runner CPU/memory. + +**Evidence:** `package_android_target` in +`src/extensions/artifacts/native/tools/package-release-assets.sh:712` first calls +`build_mobile_host_extension_runtime`, then `build_mobile_static_artifacts`. +Both Android jobs build a Linux x64 PostgreSQL runtime with selected extensions; +the separate Linux extension job builds the same class of support output too. + +| Baseline producer | Linux runtime phase | Android archive phase | +| --- | ---: | ---: | +| Android arm64 | 24m49s | 17m18s | +| Android x86_64 | 20m06s | 14m03s | +| Dedicated Linux x64 | 25m24s | — | + +The two Android host phases alone consume **44m55s**. Existing iOS packaging +already overlaps host/device/simulator lanes. Android's host dependency is +real today: the artifact packager copies SQL/control/data **and host dynamic +modules**, alongside the Android static archives. A static-archive-only +replacement would violate the existing package layout; whether every host +module is needed by downstream mobile consumers remains a separate question. + +**Owner:** native extension build/package boundary. **Done when:** both ABIs +consume matching catalog/version/feature support files, target archive and +platform checks pass, and measurements show reduced compute without a longer +Android critical path. Waiting for the whole Linux job before starting Android +compilation mostly trades duplicate compute for serial waiting. First use +compatible existing output/cache; do not add a generic build service or assume +the base runtime and extension-enabled runtime are interchangeable. + +### CI-23 — P1 — Organize WASIX host production around each consuming host + +- [x] Partition Linux consumer dependencies from other AOT/Node-API hosts using + the pattern already used for native producers. +- [x] Prefer keeping same-host AOT and Node-API production/qualification together + where that removes transfers and repeated setup without changing ABI needs. +- [x] Keep all-platform aggregation at the final release/qualification boundary. + +**Evidence:** every Node-API host starts after the whole AOT matrix. Linux-only +regression also waits for Windows. The TS consumer then waits for every Node-API +host. Baseline Linux AOT was ready at minute **67.5**, but the matrix finished +at **78.1**; Linux Node-API finished at **84.6**, but TS started at **91.8**. +These are workflow barriers, not requirements to test Linux bytes. + +**Owner:** WASIX Actions topology + existing task ownership. **Done when:** +Linux proof can finish while Windows/macOS builds continue, all selected hosts +remain mandatory at the final gate, and independent browser-host packaging +starts early (CI-09). Do not claim Rust build reuse across incompatible target +or glibc environments: Linux Node-API uses a compatibility-container boundary. +Avoid creating a separate job for every small task. This finding expands CI-09; +its savings must not be counted twice. + +### CI-24 — P2 — Preserve safe local parallelism after artifact transfer + +- [x] First reduce unnecessary transfer boundaries by colocating dependent + same-host work. Then measure remaining serial local tasks before changing the + existing execution adapter. +- [x] If material, execute only dependency-ready batches in the existing + resolver, keeping transferred producers skipped and their consumers uncached. + +**Evidence:** `.github/scripts/run-planned-moon-job.sh:55` loops through planned +targets with one `MOON_CACHE=off moon run --upstream none` invocation per target +when dependencies were transferred. This preserves ordering but serializes +independent local tasks. The Windows AOT step took **20m38s**, including +extension AOT 7m46s, core AOT 2m25s, pgwire AOT test 3m50s, integration 1m49s, +tools AOT 26s and Rust SDK AOT test 4m11s. + +**Validation rejects a tempting shortcut:** in an isolated pinned-Moon fixture, +`moon run --upstream none a b c` started `c` before its declared `a`/`b` +prerequisites finished. Simply batching every root is incorrect. Moon's +[execution plan](https://moonrepo.dev/docs/guides/exec-plan) also does not provide +an already-completed-producer import primitive; target exclusion is documented +as forthcoming. Do not base this fix on an assumed scheduler feature. + +**Owner:** existing artifact-transfer adapter. **Done when:** the small ordering +fixture passes, transferred producers never rerun, invalid transferred bytes +still fail, and measured wall time improves under actual CPU/Cargo-lock limits. +No new scheduler framework. The total serial duration is not the possible +saving; core saturation can make extra concurrency slower. + +### CI-25 — P2 — Avoid rebuilding iOS to package an unchanged carrier for Android + +- [x] Reuse verified frozen iOS carrier metadata when native iOS inputs and + version have not changed; keep its ownership with the native carrier output. + +**Evidence:** changing the real file +`src/native/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntBrokerService.kt` +selects both Android and iOS runtime/ABI jobs. The dependency path is Android +app → React Native npm package → `finalize-runtime-ios-abi`. +`src/native/sdks/react-native/moon.yml:114` and +`tools/stage-release-artifacts.mts` in that project require iOS archive bytes to +generate the package's carrier manifest. This is a genuine integrity dependency, +not merely a redundant `needs` entry. The shared +`src/native/sdks/swift/tools/ios-carrier-manifest.mts:1120` already supports a +`baseCarrierManifest` input for a frozen base carrier. + +**Owner:** native carrier metadata + React Native packaging. **Done when:** an +Android-only change uses the same complete canonical npm package contract +without recompiling unchanged Apple code, while a changed native carrier +regenerates and validates its manifest. Do not omit iOS files from a special +CI-only npm package or weaken archive hashes. No whole-baseline saving claimed: +Android extensions, not iOS runtime metadata, were that run's limiting input. + +### CI-26 — P2 — Remove UI-driver overhead from self-running SDK smoke + +- [x] Make the installed app's exact-launch structured result authoritative on + both platforms, without requiring Maestro to read example status labels. +- [x] Reuse the existing continuous log capture and report validation, with + failure precedence, process/capture failure detection and an overall deadline. + +**Evidence:** `src/examples/native/react-native-expo/maestro/installed-smoke.yaml` +only waits for the passed label and asserts result labels. It performs no SDK +interaction or lifecycle transition; the app runs those tests itself. In +[baseline iOS E2E](https://github.com/f0rr0/oliphaunt/actions/runs/36492755110/job/109183796475), +the app reported all **39 extensions passing at 23:38:12.942**, about eight +seconds after launch. Maestro finished at **23:42:50.918**, another **4m38s** +later. Simulator preparation and uninstall/install accounted for additional +minutes; they are not all removable test execution. + +The `run_maestro_installed_smoke` loop in +`src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh:51` has no +overall deadline while waiting for the driver; the flow timeout does not bound +driver startup. The existing non-Maestro branch is **not a safe drop-in**: it +polls the previous 30 seconds of logs, hardcodes the process name and checks +PASS before failure. Exact-launch continuous capture currently lives inside +the Maestro branch and should become the shared SDK-smoke mechanism. + +**Owner:** installed-app runner, shared with manual replay (CI-16). +**Done when:** real app install/launch, SQL/extensions, ICU/resource and receipt +validation remain; stale PASS, explicit FAIL, crash, dead capture and missing +receipt all fail within a bound. Keep separate lifecycle tests. Retain Maestro +only for a real UI interaction guarantee if one is required. A bare switch to +the old log mode or accepting UI success alone would weaken correctness. + +## Failure history: what to keep and what is already corrected + +| Evidence | Interpretation | Current disposition | +| --- | --- | --- | +| [36523497369 Android](https://github.com/f0rr0/oliphaunt/actions/runs/36523497369/job/109274401218): corrupt NDK archive; same-source rerun succeeded | Confirmed infrastructure/acquisition flake | CI-03 exact-package repair and shared acquisition deadlines implemented; hosted verification pending | +| Public-consumer full local suite failed; unchanged isolated control passed; delayed-start probe failed | Confirmed timing-sensitive test harness | CI-02 corrected; delayed-start and expired-deadline fixtures pass | +| [36523497369 Qualified](https://github.com/f0rr0/oliphaunt/actions/runs/36523497369/job/109397213590): attempt-1 producer receipt rejected on attempt 3 | Qualification bookkeeping defect, not failed SDK behavior | Fixed in #225; preserve same-run/source and reject future attempts | +| [36518236951](https://github.com/f0rr0/oliphaunt/actions/runs/36518236951/job/109245492836): Apple carrier selection lost `PRODUCTS_JSON` and query failure was hidden | Release assembly defect | Fixed in #224 with workflow-step behavioral regression and real artifact replay | +| [36451399513](https://github.com/f0rr0/oliphaunt/actions/runs/36451399513) and [36451108400](https://github.com/f0rr0/oliphaunt/actions/runs/36451108400): missing Moon projects / skipped required WASIX regression | Planner/ownership drift | Immediate defects fixed in #222; structural correction remains CI-05 | +| [36449348921](https://github.com/f0rr0/oliphaunt/actions/runs/36449348921): declared npm carrier version rewrite rejected in `bun.lock` | Release normalization contract omitted valid derived data | Fixed in #220; keep semantic non-version-change rejection | +| [36240045462](https://github.com/f0rr0/oliphaunt/actions/runs/36240045462/job/108405753480): browser `instant` panic during WASIX initdb | Useful real browser execution failure | Later complete candidate passed; do not remove browser execution as flaky | +| [36348549841](https://github.com/f0rr0/oliphaunt/actions/runs/36348549841): Windows `fsync` unresolved symbol; iOS missing `backupDataForJsi:completion:` selector | Real platform compilation defects | Later complete candidate passed; retain platform link/app builds | +| [36345320273](https://github.com/f0rr0/oliphaunt/actions/runs/36345320273): Windows import-library contract unit failures | Useful cheap contract gate | Caught before expensive producers; not evidence of a flaky test | +| [36310991740](https://github.com/f0rr0/oliphaunt/actions/runs/36310991740): WASIX `clang --version` SIGPIPE/source-fingerprint rejection; Android backend exited before ReadyForQuery | Build/runtime failures; log is not enough to classify these as harmless infrastructure | Preserve regression coverage; investigate the exact signature if it recurs; do not add automatic success retries | +| [36227119157](https://github.com/f0rr0/oliphaunt/actions/runs/36227119157/job/108374367397): iOS app preparation/build failed | Earlier app packaging failure | Later candidate passed; the terminal job summary alone does not prove a remaining defect | + +The retry semantics now used by #225 fit GitHub's model: a rerun retains the +original source SHA/ref and can rerun just failed jobs. See +[GitHub's rerun documentation](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/re-run-workflows-and-jobs). +Earlier successful job evidence must remain usable without accepting another +run, source commit, or future attempt. + +## Verification ROI and ordering decisions + +| Verification | Decision | Reason | +| --- | --- | --- | +| Workflow syntax/security and real planner behavior | Keep early | Cheap; actual planning failures justify it. Test final execution, not only intermediate lists | +| Source-only formatting/unit/lint before expensive producers | Keep | Already mostly correct; don't replace with speculative all-platform setup | +| Packed package reopening / clean consumer | Keep | Source tests cannot detect omitted files, bad exports or installed resource discovery | +| Native extension lifecycle plus WASIX lifecycle | Keep | Different runtimes and loading/materialization behavior; not interchangeable | +| Representative AOT execution on all supported hosts | Keep | Portable/Linux success does not prove host machine-code deserialization/execution | +| Exhaustive extension catalog on every AOT host | Do not add | Current Linux exhaustive + per-host representative split is right-sized | +| iOS and Android app builds/E2E | Keep | Platform constraints differ; real app builds caught defects that portable tests did not | +| Maestro reading self-running example status labels | Remove from SDK qualification after CI-26 | No unique SDK interaction; exact-launch app receipts already carry the result. Preserve authoritative failure handling | +| Rust consumer built with stubs, then real embedded carriers | Keep distinct guarantees | Compile/package closure versus resource-free installed execution; the second build was about 24 seconds in the baseline | +| Source tests replayed at publication | Do not reintroduce | Current frozen-candidate publication avoids the old blanket replay | +| Digest/identity/registry checks after artifact transfer or approval | Keep | Validate new bytes or mutable external state; not repeated source qualification | +| PR release qualification + exact merge qualification | Retain current guarantee | Combined sample cost was 1,565.5 runner-minutes, but different source identities are intentional. Reusing compiler outputs is the first optimization, not accepting a PR receipt as main | +| Main push plus manual full dispatch for the same SHA | Avoid operationally when an adequate run already exists | Concurrency serializes non-PR same-SHA runs; it does not deduplicate them. Release already has a missing-qualification resolver | +| Source spelling/prose assertions | No blanket deletion proposed | Reviewed matches were mostly parsed contracts, generated outputs, logs or behavioral results, not proof that arbitrary source strings should stay unchanged | +| Unused summary action | Delete | No caller or unique proof | + +The earlier September audit's blanket problems should not be carried forward +as if still present: main pushes now use affected scope, the focused workflow +PR above ran in about four minutes, release metadata and mutation tests have +separate owners, frozen publication avoids rebuilding the candidate, and the +manual mobile replay is not automatically fired after normal CI. + +## Recommended execution order + +1. **Remove long setup/recompilation tails:** CI-19, CI-06, CI-21 and CI-08. + These use existing builder/cache facilities and attack both long branches. +2. **Stop selecting work that did not change:** CI-20. Prove documentation, + unit-test, SDK-only and compiler-input boundaries with the actual planner. +3. **Shorten the second critical path:** CI-09/23 together. Measure again before + adding Postmaster overlap (CI-10) or changing the transfer adapter (CI-24). +4. **Shorten mobile feedback:** CI-07/22, then CI-26 and CI-25. Distinguish saved + runner time from end-to-end Android/iOS completion time. +5. **In parallel with those priorities, fix correctness:** CI-01 through CI-05 + and CI-11. Coverage and admission defects do not become optional because + they are not timing improvements. +6. **Then routine maintenance:** CI-12 through CI-18. Naming, small validators + and unused setup are worthwhile, but do not lead a wall-time initiative. + +Measure each structural change on a complete full run and a focused PR, with +cold versus compatible-warm caches identified. Reuse the existing job and phase +timestamps; no benchmark service is needed. Compare the final `Required`/ +`Qualified` finish, the new longest chain, setup/compiler/test time and total +runner-minutes. A successful isolated job speedup is insufficient if a newly +introduced prerequisite delays the consumer. The present sample cannot support +a defensible whole-pipeline percentage or p95 promise. + +Defer catalog-test sharding until it becomes a limiting stage. WASIX regression +spent 8m53s in the library's full extension tests and 2m17s in server extension +tests; useful behavior dominates those sections. The existing native shard +pattern is available if required later, but additional WASIX jobs would repeat +Rust setup/compilation and need complete aggregate evidence. Do not remove +extension materialization/restart/restore coverage or shard every small suite. + +Complexity cuts, ranked by recurring maintenance value: **shrink** the manual +WASIX dependency insertion into its existing Moon owner; **shrink** duplicate +evidence mode rules into the existing contract; **shrink** duplicated mobile +execution definitions; **native** use the existing Rust-cache directory mapping; +**delete** the unused summary action. No credible net line/dependency saving is +claimed before implementation; several valuable fixes are configuration or +coverage corrections rather than deletions. + +## Audit validation and limits + +Executed during this audit: + +- Real pinned Moon `query projects`, `query tasks` and `task-graph --json`, the + current CI config mapper, and the actual check/test matrix writer. +- Dependency reachability probe confirming the five SDK tasks are absent from + hosted roots, with explicit workflow-owned Postmaster tasks accounted for. +- Unchanged public-consumer fixture: pass; same fixture with delayed timeout + setup: expected reproduction of the current false failure. +- Pinned Rust-cache cleanup fixture: confirmed deletion of the custom nested + compiler artifact under the current root-workspace mapping. +- Read-only GitHub run/job/step/artifact/cache inspection; selected causal logs + and complete successful baseline timings. +- Second-pass phase attribution across Postmaster portable/host, normal WASIX, + Windows AOT/Node-API, both Android extension ABIs, Linux extension production + and iOS installed-app execution. Compared the slow Postmaster APT layer with + two faster runs instead of treating the slow sample as a fixed saving. +- One-file affected queries for both READMEs, the Postmaster unit test, the APT + installer test and an existing Android Kotlin implementation file; passed + directly affected tasks to the real planner with the expanded project/task + graph. No tracked files were modified to simulate these changes. +- Isolated pinned-Moon artifact-transfer fixture: a/b depend on an imported + producer; c depends on a/b. Confirmed `--upstream none a b c` skips dependency + ordering and fails c, ruling out the naive batch-all replacement. +- Prior same-tree evidence probes: native receipt verified; missing native PASS + markers rejected; incomplete WASIX materialization accepted by table but + rejected by candidate; incomplete product-required plan rejected at release. + +The temporary audit inputs and probes are under +`/tmp/oliphaunt-ci-audit-2026-09-29`; prior receipt probes are under +`/tmp/oliphaunt-evidence-audit`. They are local scratch evidence, not release +artifacts or a new maintained testing framework. GitHub links and exact source +locations above provide the durable provenance. + +The audit itself changed no product code, CI workflow, repository setting, cache +entry or release, and launched no expensive build matrix. The subsequent local +CI/setup correction batch is recorded at the top of this document. +Whole-system inventory does not imply exhaustive line-by-line review of every +test body. No statistical flake rate, runner cost in currency, live registry +publication success, or Vercel configuration correctness is inferred from this +sample. diff --git a/src/docs/maintainers/assets.md b/src/docs/maintainers/assets.md index 43990109c..e1f8470cf 100644 --- a/src/docs/maintainers/assets.md +++ b/src/docs/maintainers/assets.md @@ -179,7 +179,9 @@ containing only `noble`, `noble-updates`, and `noble-security` with the `main` and `universe` components. Every update and install explicitly binds that source, disabled source-parts discovery (`Dir::Etc::sourceparts=-`), a reset list directory, and the verified CA bundle. A transient failure retries the -complete update/install transaction with a fixed bound; it never falls back to +complete update/install transaction within one 15-minute default deadline, +including APT retries and outer retry waits (see +[acquisition deadlines](testing.md#acquisition-deadlines)); it never falls back to a live mirror or disables TLS verification. `ca-certificates` is installed in the same pinned transaction as the builder packages. diff --git a/src/docs/maintainers/release.md b/src/docs/maintainers/release.md index 1b7b399be..c5ce660f3 100644 --- a/src/docs/maintainers/release.md +++ b/src/docs/maintainers/release.md @@ -13,7 +13,7 @@ input, for example `["oliphaunt-js"]`. Leave all platform selectors at `all`. Moon selects those owners' tasks, downstream compatibility checks and required producer dependencies. An empty product array retains the exhaustive audit. Publication accepts this record only for covered products at the exact candidate -SHA; it still verifies the immutable artifacts and any required WASIX evidence. +SHA; it still verifies the immutable artifacts and required native/WASIX evidence. Generated release PRs and their main merge automatically select the products whose Release Please manifest versions advance. Exact main pushes and eligible main dispatches can produce publishable qualification; PR checks use the same @@ -24,6 +24,21 @@ dispatches are not automatically repeated. Cross-commit producer reuse remains an explicit acceptance item rather than permission to substitute arbitrary older artifacts. +`Qualified` binds every planned native extension lifecycle aggregate and WASIX +regression proof to the candidate's source SHA/tree and CI repository/run. A +successful earlier attempt of that same run can supply proof when only failed +jobs are rerun; a different run or newer attempt cannot. Both aggregate proof +artifacts retain 90 days. The native artifact includes its complete shard +receipts, so it does not depend on the shorter-lived diagnostic shard uploads. + +Publishing native extension carriers, including runtime-owned contrib, requires +the native proof. Release revalidates the aggregate and every shard, checks the +published SQL extension set, and compares the evidence digest with the candidate +record. SDK-only publications continue to qualify their consumers against their +declared payload versions. WASIX publications retain the existing product-derived +regression requirement. Missing, changed, wrong-run or incomplete required proof +blocks publication; an overall successful CI result does not replace it. + ## Model A product owns its SemVer, changelog, source identity, Release Please component, diff --git a/src/docs/maintainers/testing.md b/src/docs/maintainers/testing.md index 0002752e3..c313d8cce 100644 --- a/src/docs/maintainers/testing.md +++ b/src/docs/maintainers/testing.md @@ -27,11 +27,12 @@ validation. - PR: Moon-affected `check` and `test` tasks, release intent, and the selected package, artifact, and E2E jobs. Measured `coverage` is an explicit local/manual lane; it is not part of the `Required` PR gate. -- Main: affected checks, builds, runtime tests, and selected E2E. This does not - currently emit a `Qualified` release record. -- Full manual: the complete selected source/runtime/package/E2E graph. Only - an exhaustive dispatch with all target selectors produces the exact-SHA - `Qualified` release record. Coverage and benchmarks remain optional. +- Main: affected checks, builds, runtime tests, and selected E2E. A release + version change selects its product qualification closure and can emit an + exact-SHA `Qualified` record. +- Manual: the selected product or full source/runtime/package/E2E graph. All + platform selectors must remain `all` for publishable qualification on main. + Coverage and benchmarks remain optional. - Release: package-native dry-runs, artifact manifests, checksums, attestations, registry checks, exact-extension evidence, binary compatibility-floor inspection, and selected artifact behavior evidence. @@ -119,6 +120,83 @@ Reusable benchmark datasets, benchmark plans, and published reports belong in `src/benchmarks/`. Executable benchmark harnesses belong in `src/benchmarks/perf/` unless the harness is intentionally part of a product's public developer API. +## Acquisition deadlines + +Repository-owned downloads use `tools/dev/acquisition.sh`. Start one budget at +an acquisition's entry point, before lock waits or transport, and reuse it for +all retries, mirrors and dependent requests. Nested operations can shorten the +budget but cannot restart it. Compilation and test execution have their own +budgets; do not wrap an entire build in an acquisition timeout. + +```sh +. "$repo_root/tools/dev/acquisition.sh" +oliphaunt_acquisition_start 'example sources' 900 +# Per-endpoint cap, attempts, retry delay, curl command, existing secure arguments. +oliphaunt_acquisition_curl 300 3 5 curl --fail --location \ + --proto '=https' --proto-redir '=https' --tlsv1.2 --output "$partial" "$url" +# For Git, package managers and source validation processes: +oliphaunt_acquisition_run 300 git fetch --depth=1 "$url" "$commit" +``` + +The caller owns URL/pin validation, TLS policy, archive limits, checksums, +validation and atomic promotion. Downloads must target a staging file with +`--output`, never append retry responses to stdout. Pass curl's connection, +size and low-speed limits normally, but let the helper own `--retry` and +`--max-time`. Its single attempts each receive the remaining time. Curl's +[`--retry-max-time`](https://curl.se/docs/manpage.html#--retry-max-time) alone +allows its final attempt to run past the retry timer. + +| Acquisition | Default total | Maximum per endpoint/command | +| --- | --- | --- | +| Source scope / individual source pin | 30 / 15 min | Git fetch and archive endpoint: 5 min | +| PostgreSQL source archive, both origins | 3 min | 90 sec per origin | +| WASIX builder APT update + install + retries | 15 min | Update: 5 min; install: remaining budget | +| WASIX compiler asset set | 30 min | 15 min per asset | +| Android SDK setup / emulator packages (separate operations) | 30 min each | Command-line-tools origin: 4 min; SDK packages: remaining budget | +| Moon + Proto + plugins | 15 min | 5 min per asset or registry request | +| Bun/Deno; Node; npm publisher | 5 min each | Bun/Deno origin: 2 min; others: remaining budget | +| Wasmer LLVM | 30 min | Remaining budget | +| Maintainer binary; winflexbison | 3 min each | Remaining budget | +| Swift signing keys | 2 min | Remaining budget | + +`OLIPHAUNT_ACQUISITION_TIMEOUT_SECONDS` overrides an operation's total (integer +1–7200); it never raises an endpoint cap or replenishes an enclosing operation. +Defaults allow cold installation while bounding repeated failures. The APT +budget is deliberately above observed normal cold transactions (roughly a +minute), below the observed 40-minute failure tail. Tune with hosted timings, +not another retry layer. Separate scripts/actions have separate transactions; +this is not a workflow-wide download allowance. + +Shell and curl suffice for bootstrap downloads. Git, APT and SDK-manager +processes require GNU `timeout` (`coreutils`, `brew install coreutils` on macOS). +The shared helper verifies GNU identity, preferring `gtimeout` and falling back +to `/usr/bin/timeout` when Windows System32 shadows Git Bash's executable. +Android setup checks the timer before treating an SDK installation as invalid; +the macOS setup action provisions Coreutils when missing. +Commands receive TERM on expiry and KILL five seconds later if needed, including +children in their process group. Expiry reports the acquisition label and a +nonzero status (124 on expiry, or 137 after a forced kill; owners may add their +own failure status). +Interrupted curl attempts are not retried or mirrored. Existing traps clean staging and preserve prior +installations; short atomic promotion and rollback finish outside the timed +child. Source validation/extraction also consumes the source budget. Other +installers' local validation and promotion are not independently hard-timed. +The portable shared clock uses epoch seconds; per-process timers bound active +work. Runner clock corrections can shift subsequent remaining-time calculations. + +The helper's test owns clock arithmetic and process termination; source and +installer fault tests own preservation, retry/failover and validation. Moon +inputs and WASIX recipe identity include the helper. Build the WASIX Dockerfile +from the repository root; its adjacent `Dockerfile.dockerignore` limits context +to the recipe and helper. To build from another checkout, set `REPO_ROOT` and +`WASIX_TOOLCHAIN_ROOT` together so the recipe and COPY inputs refer to that tree. + +Upstream actions and general dependency resolution (Cargo, npm, Homebrew, +Chocolatey, pip and host build-tool setup) retain their existing job/setup +budgets. This helper does not replace package-manager retry or installation +semantics. A deadline stops a bad acquisition promptly; it does not turn a +failed transfer into successful qualification. + ## Moon Tasks Moon task names are intentionally narrow: @@ -148,6 +226,25 @@ runtime evidence. React Native installed-app smokes delegate runtime materialization to the Expo platform scripts and hard-fail there if native artifacts cannot be built or located. +The Linux native consumer job also owns these uncached SDK runtime suites: + +- `oliphaunt-rust:test-integration`: native smoke and SQL behavior. +- `oliphaunt-mobile-bindings:test-native`: shared broker streaming and lifecycle. +- `oliphaunt-swift:test-native`: Swift native runtime behavior. +- `oliphaunt-kotlin:test-native-bindings`: Kotlin native binding behavior. + +They consume the same-run packaged runtime, with tools and broker archives when +needed. The shared `src/native/sdks/tests/with-runtime.sh` stages each invocation +in an isolated temporary directory; Moon retains the producers for local runs +and the hosted transfer runner substitutes downloaded artifacts. Swift/Kotlin +share one generated binding dependency. These suites require executed tests and +valid runtime inputs; zero tests or unavailable artifacts cannot pass as proof. + +`oliphaunt-wasix-rust:test-integration` runs the standard and ICU seed resource +tests in the existing WASIX regression job, consuming its portable runtime/AOT +and same-run database resource artifacts. This host coverage supplements the +installed mobile apps and per-platform package checks. + React Native installed-app smoke is split by platform: ```sh @@ -159,37 +256,21 @@ PR jobs run RN static, unit, Codegen, JSI, config-plugin, and package checks. Affected PR, main, and explicit manual lanes run the installed Android/iOS app smokes selected by the CI plan. -Installed-app E2E runner choice is closed, not a recurring research task. -Decision (2026-06-08): Oliphaunt uses the pinned open-source Maestro CLI -through GitHub-hosted emulator/simulator jobs. This is not an open research loop. -Reopen that decision only when a written implementation proposal names an -installed-app E2E requirement that the pinned open-source Maestro CLI cannot -satisfy. Do not keep re-checking Maestro, Detox, Appium, EAS, Firebase Test -Lab, BrowserStack, Sauce, AWS Device Farm, or other hosted-device services while -implementing this plan. Routine maintenance verifies the pinned installer, flow -files, app artifacts, runner behavior, and CI logs for the selected Maestro -lanes; it does not revisit provider selection. - -`tools/dev/setup-maestro.sh` installs only the exact versioned release asset and -SHA-256 recorded in `tools/dev/maestro.toml`; that manifest is the -single release pin. It does not execute the vendor's network installer. Version -upgrades change the reviewed manifest metadata and must keep the staged -archive/layout/version and atomic-promotion regression tests green; incomplete -or inconsistent metadata fails before any download. +Installed-app SDK smoke runs the real app on a hosted emulator or simulator. +The app performs the SQL, extension, and resource assertions itself. CI and +manual replay share `.github/actions/run-mobile-e2e`; both require a validated +structured receipt from continuous logs captured for that launch. Explicit +failure, app death, dead capture, and a missing receipt fail within the smoke +budget. Lifecycle and crash-recovery drills retain their separate assertions. The native Node addon uses the Rust Node-API adapter. It no longer downloads Node C headers or a separate Windows import library through a custom fallback. -Prior provider research is historical context, not a standing checklist. Maestro -pin upgrades are dependency maintenance; they do not reopen the runner decision -unless they expose a concrete installed-app E2E requirement this path cannot -meet. - The default installed-app path must remain free and public-checkout reproducible. Paid hosted-device providers, SaaS-only runners, and required private runner infrastructure are not part of the default proof path. When mobile E2E breaks, inspect the selected implementation first: app artifact shape, -simulator/emulator setup, Maestro flow files, logs, and CI runner assumptions. +simulator/emulator setup, exact-launch logs, receipts, and CI runner assumptions. Debug the chosen implementation first. Do not restart provider research unless the failure proves a concrete requirement this model cannot satisfy. diff --git a/src/examples/moon.yml b/src/examples/moon.yml index 29cebf100..6ab8d17e0 100644 --- a/src/examples/moon.yml +++ b/src/examples/moon.yml @@ -99,12 +99,9 @@ tasks: - "integration-examples:react-native-android-build" inputs: - "native/react-native-expo/src/**/*" - - "native/react-native-expo/maestro/**/*" - "native/react-native-expo/package.json" - project: "oliphaunt-react-native" group: "code" - - "/tools/dev/maestro.toml" - - "/tools/dev/setup-maestro.sh" - "/target/mobile-build/react-native/android/**/*" options: cache: false @@ -163,12 +160,9 @@ tasks: - "integration-examples:react-native-ios-build" inputs: - "native/react-native-expo/src/**/*" - - "native/react-native-expo/maestro/**/*" - "native/react-native-expo/package.json" - project: "oliphaunt-react-native" group: "code" - - "/tools/dev/maestro.toml" - - "/tools/dev/setup-maestro.sh" - "/target/mobile-build/react-native/ios/**/*" options: cache: false diff --git a/src/examples/native/react-native-expo/maestro/installed-smoke.yaml b/src/examples/native/react-native-expo/maestro/installed-smoke.yaml deleted file mode 100644 index 771a20c6c..000000000 --- a/src/examples/native/react-native-expo/maestro/installed-smoke.yaml +++ /dev/null @@ -1,15 +0,0 @@ -appId: ${APP_ID} -name: Oliphaunt installed app smoke -tags: - - oliphaunt - - smoke - - installed-app ---- -- extendedWaitUntil: - visible: - id: liboliphaunt-smoke-status-passed - timeout: ${SMOKE_TIMEOUT_MS} -- assertVisible: - id: liboliphaunt-smoke-status-passed -- assertVisible: - id: liboliphaunt-smoke-result diff --git a/src/extensions/artifacts/native/moon.yml b/src/extensions/artifacts/native/moon.yml index c5e725ff0..1d73bebbf 100644 --- a/src/extensions/artifacts/native/moon.yml +++ b/src/extensions/artifacts/native/moon.yml @@ -87,20 +87,18 @@ tasks: group: sources - /src/native/runtime/sources/*.toml - /src/extensions/artifacts/native/tools/**/* + - "!/src/extensions/artifacts/native/tools/**/*.test.*" - /tools/packaging/check-linux-consumer-baseline.sh - /src/third-party/tools/source-fetch-core.mts - /src/extensions/tools/extension-upstream-licenses.mts - - /tools/packaging/linux-abi-baseline.test.sh - "/tools/packaging/*.{mjs,mts}" - /src/extensions/tools/extension-artifact-archive-policy.mts - /src/extensions/artifacts/native/tools/native-extension-asset-index-contract.mts - /src/native/runtime/tools/native-runtime-payload-policy.json - /tools/packaging/windows-vc-runtime-policy.json - /tools/release/platform-compatibility-policy.mts - - /tools/release/platform-compatibility-policy.test.mts - /tools/packaging/platform-binary-contract.mts - /tools/packaging/strip-native-binaries.sh - - /tools/packaging/platform-binary-contract.test.mts - /tools/packaging/release-notices.mts - /tools/packaging/release-directory-safety.mts - /tools/packaging/windows-vc-runtime-closure.mts diff --git a/src/extensions/artifacts/native/tools/package-release-assets.sh b/src/extensions/artifacts/native/tools/package-release-assets.sh index 9ef9cf2bf..4727a318b 100755 --- a/src/extensions/artifacts/native/tools/package-release-assets.sh +++ b/src/extensions/artifacts/native/tools/package-release-assets.sh @@ -702,9 +702,11 @@ package_ios_target() { package_android_target() { local source_runtime runtime mobile_extensions android_root android_static_target - build_mobile_host_extension_runtime mobile_extensions="$(mobile_module_extensions_csv)" - build_mobile_static_artifacts "$mobile_extensions" + if [ "${OLIPHAUNT_EXTENSION_PHASE:-all}" != android-package ]; then + build_mobile_host_extension_runtime + build_mobile_static_artifacts "$mobile_extensions" + fi source_runtime="$(host_extension_runtime_root)" require_dir "$source_runtime" "mobile host extension runtime" runtime="$(prepare_extension_release_runtime "$source_runtime")" @@ -720,7 +722,9 @@ package_android_target() { ;; *) fail "Android target packager called for $target_id" ;; esac - tools/dev/bun.sh tools/packaging/platform-binary-contract.mts --target "$target_id" --root "$android_root/out" + if [ -n "$mobile_extensions" ]; then + tools/dev/bun.sh tools/packaging/platform-binary-contract.mts --target "$target_id" --root "$android_root/out" + fi local sql_name pg_major creates_extension stem dependencies shared_preload desktop_prebuilt mobile_prebuilt mobile_static_required mobile_static_targets data_files artifact_policy runtime_artifact android_archive static_prefix while IFS=$'\t' read -r sql_name pg_major creates_extension stem dependencies shared_preload desktop_prebuilt mobile_prebuilt mobile_static_required mobile_static_targets data_files artifact_policy; do [ -n "$sql_name" ] || continue @@ -753,10 +757,22 @@ package_android_target() { done < <(catalog_rows) } -fetch_extension_source_assets +phase="${OLIPHAUNT_EXTENSION_PHASE:-all}" +case "$phase:$target_id" in + all:*) fetch_extension_source_assets ;; + android-static:android-*) fetch_extension_source_assets ;; + android-package:android-*) ;; + *) fail "invalid extension phase $phase for $target_id" ;; +esac echo "==> Reading exact extension catalog" bun "$packager" list-catalog >"$catalog_file" write_indexes +if [ "$phase" = android-static ]; then + # SQL-only selections still transfer an empty, valid native output directory. + mkdir -p "$mobile_extension_work_root/$target_id/${target_id%-v8a}/out" + build_mobile_static_artifacts "$(mobile_module_extensions_csv)" + exit 0 +fi case "$target_id" in macos-arm64|linux-x64-gnu|linux-arm64-gnu|windows-x64-msvc) diff --git a/src/extensions/artifacts/native/tools/package-release-assets.test.sh b/src/extensions/artifacts/native/tools/package-release-assets.test.sh new file mode 100644 index 000000000..b68aa39df --- /dev/null +++ b/src/extensions/artifacts/native/tools/package-release-assets.test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +export OLIPHAUNT_EXTENSION_TARGET=android-arm64-v8a +export OLIPHAUNT_EXTENSION_PRODUCTS=oliphaunt-extension-pgtap +export OLIPHAUNT_EXTENSION_HOST_RUNTIME_ROOT="$scratch/host" +export OLIPHAUNT_MOBILE_EXTENSION_WORK_ROOT="$scratch/mobile" +export OLIPHAUNT_EXTENSION_RELEASE_ASSET_DIR="$scratch/assets" +export OLIPHAUNT_EXTENSION_RELEASE_STAGE_ROOT="$scratch/stage" +producer=src/extensions/artifacts/native/tools/package-release-assets.sh +sql="$scratch/host/share/postgresql/extension" +mkdir -p "$sql" +bun - "$scratch/host/postgres" <<'JS' +import {writeFileSync} from 'node:fs'; +import {elfFixture} from './tools/packaging/testdata/release-fixture-utils.mts'; +writeFileSync(process.argv[2], elfFixture({machine: 62, requiredVersions: ['GLIBC_2.17']})); +JS +printf "default_version = '1.3.5'\n" > "$sql/pgtap.control" +printf 'select 1;\n' > "$sql/pgtap--1.3.5.sql" +# SQL-only selection has a valid empty transfer and needs neither compiler. +OLIPHAUNT_EXTENSION_PHASE=android-static bash "$producer" +[ -d "$scratch/mobile/android-arm64-v8a/android-arm64/out" ] +OLIPHAUNT_EXTENSION_PHASE=android-package bash "$producer" +[ "$(find "$scratch/assets" -name '*-pgtap-*-runtime.tar.gz' | wc -l)" = 1 ] +# A package-only invocation must fail on missing inputs, never rebuild them. +if OLIPHAUNT_EXTENSION_PHASE=android-package OLIPHAUNT_EXTENSION_PRODUCTS=oliphaunt-extension-vector \ + bash "$producer" > "$scratch/missing-static.log" 2>&1; then + echo 'accepted missing static archive' >&2; exit 1 +fi +grep -Eq 'no ELF binaries|missing Android static archive for vector' "$scratch/missing-static.log" +rm -rf "$scratch/host" +if OLIPHAUNT_EXTENSION_PHASE=android-package bash "$producer" > "$scratch/missing-host.log" 2>&1; then + echo 'accepted missing Linux support' >&2; exit 1 +fi +grep -q 'missing mobile host extension runtime' "$scratch/missing-host.log" +if OLIPHAUNT_EXTENSION_PHASE=android-package OLIPHAUNT_EXTENSION_TARGET=ios-xcframework \ + bash "$producer" > "$scratch/wrong-phase.log" 2>&1; then + echo 'accepted Android phase for iOS' >&2; exit 1 +fi +grep -q 'invalid extension phase' "$scratch/wrong-phase.log" +echo 'Android split extension producer checks passed' diff --git a/src/extensions/artifacts/wasix/moon.yml b/src/extensions/artifacts/wasix/moon.yml index c328f25c5..8b2f966bb 100644 --- a/src/extensions/artifacts/wasix/moon.yml +++ b/src/extensions/artifacts/wasix/moon.yml @@ -25,7 +25,7 @@ tasks: tags: ["artifact", "ci-liboliphaunt-wasix-runtime"] command: "bash src/extensions/artifacts/wasix/tools/build-portable.sh" deps: ["liboliphaunt-wasix:runtime-portable"] - inputs: ["tools/build-portable.sh", "/src/wasix/runtime/assets/build/**/*", "/src/wasix/runtime/tools/extension-build-scripts.mts", "/src/wasix/runtime/tools/xtask/**/*", {project: "extensions", group: "build"}] + inputs: ["tools/build-portable.sh", "/src/wasix/runtime/assets/build/**/*", "!/src/wasix/runtime/assets/build/**/*.test.*", "/src/wasix/runtime/tools/extension-build-scripts.mts", "/src/wasix/runtime/tools/xtask/**/*", {project: "extensions", group: "build"}] outputs: ["/target/extensions/wasix/assets/**/*"] options: {runFromWorkspaceRoot: true, cache: local} build-aot: diff --git a/src/extensions/evidence/matrix.toml b/src/extensions/evidence/matrix.toml index 62f5d9802..3705288bc 100644 --- a/src/extensions/evidence/matrix.toml +++ b/src/extensions/evidence/matrix.toml @@ -100,7 +100,7 @@ extension = "amcheck" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -108,7 +108,7 @@ extension = "auto_explain" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -116,7 +116,7 @@ extension = "bloom" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -124,7 +124,7 @@ extension = "btree_gin" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -132,7 +132,7 @@ extension = "btree_gist" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -140,7 +140,7 @@ extension = "citext" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -148,7 +148,7 @@ extension = "cube" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -156,7 +156,7 @@ extension = "dict_int" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -164,7 +164,7 @@ extension = "dict_xsyn" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -172,7 +172,7 @@ extension = "earthdistance" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -180,7 +180,7 @@ extension = "file_fdw" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -188,7 +188,7 @@ extension = "fuzzystrmatch" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -196,7 +196,7 @@ extension = "hstore" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -204,7 +204,7 @@ extension = "intarray" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -212,7 +212,7 @@ extension = "isn" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -220,7 +220,7 @@ extension = "lo" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -228,7 +228,7 @@ extension = "ltree" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -236,7 +236,7 @@ extension = "pageinspect" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -244,7 +244,7 @@ extension = "pg_buffercache" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -252,7 +252,7 @@ extension = "pg_freespacemap" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -260,7 +260,7 @@ extension = "pg_hashids" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -268,7 +268,7 @@ extension = "pg_ivm" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -276,7 +276,7 @@ extension = "pg_surgery" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -284,7 +284,7 @@ extension = "pg_textsearch" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -292,7 +292,7 @@ extension = "pg_trgm" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -300,7 +300,7 @@ extension = "pg_uuidv7" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -308,7 +308,7 @@ extension = "pg_visibility" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -316,7 +316,7 @@ extension = "pg_walinspect" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -324,7 +324,7 @@ extension = "pgcrypto" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -332,7 +332,7 @@ extension = "pgtap" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -340,7 +340,7 @@ extension = "postgis" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -348,7 +348,7 @@ extension = "seg" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -356,7 +356,7 @@ extension = "tablefunc" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -364,7 +364,7 @@ extension = "tcn" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -372,7 +372,7 @@ extension = "tsm_system_rows" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -380,7 +380,7 @@ extension = "tsm_system_time" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -388,7 +388,7 @@ extension = "unaccent" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -396,7 +396,7 @@ extension = "uuid_ossp" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -404,5 +404,5 @@ extension = "vector" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] diff --git a/src/extensions/generated/docs/extension-evidence.json b/src/extensions/generated/docs/extension-evidence.json index 8720c014f..00b689fe6 100644 --- a/src/extensions/generated/docs/extension-evidence.json +++ b/src/extensions/generated/docs/extension-evidence.json @@ -16,7 +16,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -28,7 +29,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "amcheck" }, @@ -48,7 +50,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -60,7 +63,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "auto_explain" }, @@ -80,7 +84,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -92,7 +97,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "bloom" }, @@ -112,7 +118,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -124,7 +131,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "btree_gin" }, @@ -144,7 +152,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -156,7 +165,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "btree_gist" }, @@ -176,7 +186,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -188,7 +199,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "citext" }, @@ -208,7 +220,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -220,7 +233,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "cube" }, @@ -240,7 +254,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -252,7 +267,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "dict_int" }, @@ -272,7 +288,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -284,7 +301,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "dict_xsyn" }, @@ -304,7 +322,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -316,7 +335,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "earthdistance" }, @@ -336,7 +356,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -348,7 +369,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "file_fdw" }, @@ -368,7 +390,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -380,7 +403,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "fuzzystrmatch" }, @@ -400,7 +424,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -412,7 +437,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "hstore" }, @@ -432,7 +458,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -444,7 +471,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "intarray" }, @@ -464,7 +492,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -476,7 +505,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "isn" }, @@ -496,7 +526,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -508,7 +539,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "lo" }, @@ -528,7 +560,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -540,7 +573,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "ltree" }, @@ -560,7 +594,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -572,7 +607,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pageinspect" }, @@ -592,7 +628,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -604,7 +641,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_buffercache" }, @@ -624,7 +662,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -636,7 +675,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_freespacemap" }, @@ -656,7 +696,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -668,7 +709,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_hashids" }, @@ -688,7 +730,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -700,7 +743,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_ivm" }, @@ -720,7 +764,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -732,7 +777,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_surgery" }, @@ -752,7 +798,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -764,7 +811,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_textsearch" }, @@ -784,7 +832,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -796,7 +845,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_trgm" }, @@ -816,7 +866,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -828,7 +879,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_uuidv7" }, @@ -848,7 +900,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -860,7 +913,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_visibility" }, @@ -880,7 +934,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -892,7 +947,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_walinspect" }, @@ -912,7 +968,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -924,7 +981,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pgcrypto" }, @@ -944,7 +1002,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -956,7 +1015,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pgtap" }, @@ -976,7 +1036,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -988,7 +1049,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "postgis" }, @@ -1008,7 +1070,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1020,7 +1083,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "seg" }, @@ -1040,7 +1104,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1052,7 +1117,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "tablefunc" }, @@ -1072,7 +1138,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1084,7 +1151,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "tcn" }, @@ -1104,7 +1172,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1116,7 +1185,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "tsm_system_rows" }, @@ -1136,7 +1206,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1148,7 +1219,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "tsm_system_time" }, @@ -1168,7 +1240,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1180,7 +1253,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "unaccent" }, @@ -1200,7 +1274,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1212,7 +1287,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "uuid-ossp" }, @@ -1232,7 +1308,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1244,7 +1321,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "vector" } diff --git a/src/extensions/tests/native/moon.yml b/src/extensions/tests/native/moon.yml index 2fc7cd332..108e9534c 100644 --- a/src/extensions/tests/native/moon.yml +++ b/src/extensions/tests/native/moon.yml @@ -26,6 +26,7 @@ tasks: - project: "shared-test-fixtures" group: "fixtures" - "tools/**/*.mts" + - "/.github/scripts/{release-candidate-lib,write-release-candidate,verify-release-candidate}.mts" - "/tools/release/**/*.mts" - "/src/extensions/artifacts/native/tools/**/*.mts" - "/src/extensions/artifacts/packages/tools/**/*.mts" diff --git a/src/extensions/tests/native/tools/native-extension-lifecycle-receipts.test.mts b/src/extensions/tests/native/tools/native-extension-lifecycle-receipts.test.mts index 9a6bdb596..6949b999a 100644 --- a/src/extensions/tests/native/tools/native-extension-lifecycle-receipts.test.mts +++ b/src/extensions/tests/native/tools/native-extension-lifecycle-receipts.test.mts @@ -2,11 +2,25 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; - +import { + affectedPlanBinding, + assertBindingMatches, + assertCandidateBindingShape, + nativeEvidenceBinding, +} from '../../../../../.github/scripts/release-candidate-lib.mts'; import { compareText, exactExtensionProducts, @@ -368,3 +382,138 @@ test('aggregate verification rejects candidate, shard, and PASS-record drift eve } } }); + +test('release qualification binds and revalidates same-run native lifecycle shards', () => { + const value = fixture(['cube', 'earthdistance']); + try { + const root = path.join(value.root, 'evidence'); + mkdirSync(path.join(root, 'input'), { recursive: true }); + mkdirSync(path.join(root, 'output')); + const receipt = writeShard(value, 0, { shardCount: 1 }); + const transferred = path.join(root, 'input', path.basename(receipt)); + copyFileSync(receipt, transferred); + const output = path.join(root, 'output/aggregate-receipt.json'); + verifyReceipts({ + receipts: path.join(root, 'input'), + 'candidate-sha': CANDIDATE_SHA, + 'candidate-tree': CANDIDATE_TREE, + 'expected-extensions-csv': 'cube,earthdistance', + 'expected-shard-count': '1', + repository: 'f0rr0/oliphaunt', + 'run-id': '123', + 'run-attempt': '1', + output, + }); + const planFile = path.join(value.root, 'plan.json'); + writeFileSync( + planFile, + JSON.stringify({ + jobs: ['affected', 'native-extension-lifecycle'], + projects: [], + native_extension_lifecycle_sql_names: ['cube', 'earthdistance'], + native_extension_lifecycle_shard_count: 1, + }), + ); + const affectedPlan = affectedPlanBinding(planFile, false); + const provenance = { + repository: 'f0rr0/oliphaunt', + runId: '123', + runAttempt: 2, + sha: CANDIDATE_SHA, + tree: CANDIDATE_TREE, + selection: affectedPlan.nativeExtensionLifecycle, + }; + const candidateFile = path.join(value.root, 'candidate.json'); + const env = { + ...process.env, + CI_HEAD_SHA: CANDIDATE_SHA, + CI_CHECKED_OUT_SHA: CANDIDATE_SHA, + CI_SOURCE_TREE: CANDIDATE_TREE, + CI_PLAN_PATH: planFile, + CI_QUALIFICATION_MODE: 'full-payload', + WASIX_RELEASE_REGRESSION_REQUIRED: 'false', + NATIVE_EVIDENCE_ROOT: root, + GITHUB_REPOSITORY: provenance.repository, + GITHUB_WORKFLOW: 'CI', + GITHUB_RUN_ID: '123', + GITHUB_RUN_ATTEMPT: '2', + GITHUB_EVENT_NAME: 'push', + GITHUB_REF: 'refs/heads/main', + GITHUB_WORKFLOW_REF: 'f0rr0/oliphaunt/.github/workflows/ci.yml@refs/heads/main', + CI_RUN_ID: '123', + RELEASE_HEAD_SHA: CANDIDATE_SHA, + PRODUCER_RECEIPTS_JSON: '[]', + }; + const run = (command, ...args) => { + const result = Bun.spawnSync( + ['bun', `.github/scripts/${command}-release-candidate.mts`, candidateFile, ...args], + { env }, + ); + return { code: result.exitCode, output: result.stdout.toString() + result.stderr.toString() }; + }; + const written = run('write'); + assert.equal(written.code, 0, written.output); + const verifyArgs = [ + '--plan', + planFile, + '--wasix-evidence-required', + 'false', + '--native-evidence-required', + 'true', + '--native-evidence-root', + root, + ]; + const verified = run('verify', ...verifyArgs); + assert.equal(verified.code, 0, verified.output); + const missingProduct = run( + 'verify', + ...verifyArgs, + '--products-json', + '["oliphaunt-extension-vector"]', + ); + assert.notEqual(missingProduct.code, 0); + assert.match(missingProduct.output, /missing published extension vector/); + const binding = nativeEvidenceBinding(root, provenance); + const candidate = { + schemaVersion: 2, + ...provenance, + affectedPlan, + evidenceRequirements: { + wasixReleaseRegression: false, + nativeExtensionLifecycle: true, + artifacts: ['native-extension-lifecycle-evidence'], + }, + evidence: { wasixReleaseRegression: null, nativeExtensionLifecycle: binding }, + }; + assert.doesNotThrow(() => assertCandidateBindingShape(candidate)); + assert.throws( + () => + assertCandidateBindingShape({ ...candidate, evidence: { wasixReleaseRegression: null } }), + /native evidence digest/, + ); + for (const patch of [ + { runId: '124' }, + { repository: 'other/repo' }, + { runAttempt: 0 }, + { sha: 'c'.repeat(40) }, + { tree: 'c'.repeat(40) }, + { selection: { extensions: ['cube'], shardCount: 1 } }, + { selection: { extensions: ['cube', 'earthdistance'], shardCount: 2 } }, + ]) + assert.throws(() => nativeEvidenceBinding(root, { ...provenance, ...patch })); + writeFileSync(output, `${readFileSync(output, 'utf8')}\n`); + assert.throws( + () => + assertBindingMatches(binding, nativeEvidenceBinding(root, provenance), 'native evidence'), + /does not match/, + ); + const shard = JSON.parse(readFileSync(transferred, 'utf8')); + shard.passRecords.pop(); + writeFileSync(transferred, JSON.stringify(shard)); + assert.throws(() => nativeEvidenceBinding(root, provenance), /digest mismatch/); + rmSync(transferred); + assert.throws(() => nativeEvidenceBinding(root, provenance), /exactly 1 shard/); + } finally { + rmSync(value.root, { force: true, recursive: true }); + } +}); diff --git a/src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts b/src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts index 0724efe44..7800e7766 100644 --- a/src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts +++ b/src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts @@ -211,11 +211,36 @@ export function verifyReceipts(options) { `aggregate extension coverage drift: expected=${expected.join(',')}; actual=${actual.join(',')}`, ); } + let github; + if ( + options.repository !== undefined || + options['run-id'] !== undefined || + options['run-attempt'] !== undefined + ) { + const runId = Number(options['run-id']); + const runAttempt = Number(options['run-attempt']); + if ( + !/^[^/]+\/[^/]+$/.test(options.repository ?? '') || + !Number.isSafeInteger(runId) || + runId < 1 || + !Number.isSafeInteger(runAttempt) || + runAttempt < 1 + ) + fail('GitHub provenance requires repository, positive run ID and attempt'); + github = { + repository: options.repository, + workflow: 'CI', + runId, + runAttempt, + job: 'native-extension-lifecycle-aggregate', + }; + } const aggregateCore = { schema: 'oliphaunt-native-extension-lifecycle-aggregate-v1', candidateSha: options['candidate-sha'], candidateTree: options['candidate-tree'], target: 'linux-x64-gnu', + ...(github ? { github } : {}), shardCount: expectedShardCount, extensionCount: expected.length, extensions: expected, @@ -231,8 +256,11 @@ export function verifyReceipts(options) { .sort((left, right) => left.shardIndex - right.shardIndex), }; const aggregate = { ...aggregateCore, aggregateSha256: sha256(JSON.stringify(aggregateCore)) }; - writeFileSync(options.output, `${JSON.stringify(aggregate, null, 2)}\n`); - console.log(`native extension lifecycle aggregate verified: ${options.output}`); + if (options.output) { + writeFileSync(options.output, `${JSON.stringify(aggregate, null, 2)}\n`); + console.log(`native extension lifecycle aggregate verified: ${options.output}`); + } + return aggregate; } if (import.meta.main) { diff --git a/src/extensions/tools/collect-wasix-evidence.sh b/src/extensions/tools/collect-wasix-evidence.sh index 93dab05a1..a24e3e002 100755 --- a/src/extensions/tools/collect-wasix-evidence.sh +++ b/src/extensions/tools/collect-wasix-evidence.sh @@ -33,7 +33,7 @@ trap 'rm -rf "$OLIPHAUNT_EXTENSION_EVIDENCE_DIR"' EXIT # This command exercises every catalogued extension in direct, server, restart, # materialization, and physical backup/restore modes. The record command is deliberately # after it so a failing or interrupted run cannot produce passed evidence. -bash src/wasix/runtime/tools/runtime-smoke.sh regression +.github/scripts/run-planned-moon-job.sh wasix-release-regression bash src/extensions/tools/check-extension-model.sh \ --record-wasix-evidence-run "$run_id" \ --observed-at "$observed_at" diff --git a/src/extensions/tools/extension-evidence.mts b/src/extensions/tools/extension-evidence.mts index 53fd2cfda..1cbbed4b4 100644 --- a/src/extensions/tools/extension-evidence.mts +++ b/src/extensions/tools/extension-evidence.mts @@ -83,7 +83,7 @@ export function evidenceMatrix(catalog: Row): string { 'postgres-major = 18', 'artifact-family = "wasix-runtime"', 'platform-targets = ["portable"]', - 'runtime-modes = ["direct", "server", "restart", "backup-restore"]', + `runtime-modes = ${JSON.stringify(modes)}`, `evidence-required = ["${tier}"]`, '', ]; diff --git a/src/extensions/tools/extension-model.test.mts b/src/extensions/tools/extension-model.test.mts index f9946df5a..64b618a52 100644 --- a/src/extensions/tools/extension-model.test.mts +++ b/src/extensions/tools/extension-model.test.mts @@ -47,6 +47,7 @@ function run() { direct: 'passed', server: 'passed', restart: 'passed', + materialization: 'passed', 'backup-restore': 'passed', }, }, @@ -78,6 +79,9 @@ test('only observed passing results for the current source commit qualify', () = (run: any) => { run.results[0].runtimeModeStatuses['backup-restore'] = 'failed'; }, + (run: any) => { + delete run.results[0].runtimeModeStatuses.materialization; + }, (run: any) => { run.status = 'failed'; }, diff --git a/src/native/mobile-bindings/moon.yml b/src/native/mobile-bindings/moon.yml index 63dc4a07a..f150a763c 100644 --- a/src/native/mobile-bindings/moon.yml +++ b/src/native/mobile-bindings/moon.yml @@ -22,14 +22,12 @@ tasks: options: runFromWorkspaceRoot: true test-native: - tags: ["integration", "runtime", "requires-rust"] + tags: ["ci-native-consumers", "platform-linux-x64-gnu", "integration", "runtime", "requires-rust"] script: | set -e - . src/native/runtime/tools/runtime-preflight.sh - oliphaunt_runtime_native_host_require basic - cargo test -p oliphaunt --no-default-features --features mobile-bindings --test mobile_broker --locked -- --nocapture - deps: ["liboliphaunt-native:build-runtime-desktop-target", "~:cargo-sources"] - inputs: ["@group(cargo-workspace)", "src/**/*", "/src/native/sdks/rust/src/**/*", "/src/native/sdks/rust/tests/mobile_broker.rs", "/src/native/broker/src/**/*"] + bash src/native/sdks/tests/with-runtime.sh cargo nextest run -p oliphaunt --no-default-features --features mobile-bindings --test mobile_broker --locked --no-tests=fail --test-threads=1 + deps: ["liboliphaunt-native:package-runtime-desktop-target", "~:cargo-sources"] + inputs: ["/src/native/sdks/tests/with-runtime.sh", "/src/native/runtime/tools/runtime-preflight.sh", "@group(cargo-workspace)", "src/**/*", "/src/native/sdks/rust/src/**/*", "/src/native/sdks/rust/tests/mobile_broker.rs", "/src/native/broker/src/**/*"] options: runFromWorkspaceRoot: true cache: false diff --git a/src/native/runtime/bin/build-postgres18-android-arm64.sh b/src/native/runtime/bin/build-postgres18-android-arm64.sh index ba1a7ca4c..24ecc73e5 100755 --- a/src/native/runtime/bin/build-postgres18-android-arm64.sh +++ b/src/native/runtime/bin/build-postgres18-android-arm64.sh @@ -8,6 +8,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/mobile-postgis-extensions.sh" script_path="$script_dir/$(basename "$0")" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" pg_version="18.4" pg_sha256="81a81ec695fb0c7901407defaa1d2f7973617154cf27ba74e3a7ab8e64436094" diff --git a/src/native/runtime/bin/build-postgres18-ios-device.sh b/src/native/runtime/bin/build-postgres18-ios-device.sh index 071f3056e..b8b276c5c 100755 --- a/src/native/runtime/bin/build-postgres18-ios-device.sh +++ b/src/native/runtime/bin/build-postgres18-ios-device.sh @@ -8,6 +8,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/mobile-postgis-extensions.sh" script_path="$script_dir/$(basename "$0")" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" oliphaunt_mobile_target="ios-device" pg_version="18.4" diff --git a/src/native/runtime/bin/build-postgres18-ios-simulator.sh b/src/native/runtime/bin/build-postgres18-ios-simulator.sh index 77f89a788..d844ff891 100755 --- a/src/native/runtime/bin/build-postgres18-ios-simulator.sh +++ b/src/native/runtime/bin/build-postgres18-ios-simulator.sh @@ -8,6 +8,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/mobile-postgis-extensions.sh" script_path="$script_dir/$(basename "$0")" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" oliphaunt_mobile_target="ios-simulator" pg_version="18.4" diff --git a/src/native/runtime/bin/build-postgres18-linux.sh b/src/native/runtime/bin/build-postgres18-linux.sh index 02f6f053d..c25380229 100755 --- a/src/native/runtime/bin/build-postgres18-linux.sh +++ b/src/native/runtime/bin/build-postgres18-linux.sh @@ -7,6 +7,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/postgis-dependency-cache.sh" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" pg_version="18.4" pg_sha256="81a81ec695fb0c7901407defaa1d2f7973617154cf27ba74e3a7ab8e64436094" diff --git a/src/native/runtime/bin/build-postgres18-macos.sh b/src/native/runtime/bin/build-postgres18-macos.sh index f6e255ed9..c8183d6bd 100755 --- a/src/native/runtime/bin/build-postgres18-macos.sh +++ b/src/native/runtime/bin/build-postgres18-macos.sh @@ -7,6 +7,7 @@ script_path="$script_dir/$(basename "${BASH_SOURCE[0]}")" . "$script_dir/../../../third-party/icu/tools/build.sh" . "$script_dir/postgis-dependency-cache.sh" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" macos_deployment_target="${MACOSX_DEPLOYMENT_TARGET:-11.0}" case "$macos_deployment_target" in diff --git a/src/native/runtime/bin/build-postgres18-windows.sh b/src/native/runtime/bin/build-postgres18-windows.sh index 8e619ccdf..3399fe7ef 100755 --- a/src/native/runtime/bin/build-postgres18-windows.sh +++ b/src/native/runtime/bin/build-postgres18-windows.sh @@ -5,6 +5,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$script_dir/common.sh" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" cd "$repo_root" +source tools/dev/acquisition.sh source src/third-party/postgres/fetch-source.sh source src/native/runtime/tools/liboliphaunt-extension-guard.sh diff --git a/src/native/runtime/bin/check-postgres18-ios-simulator.sh b/src/native/runtime/bin/check-postgres18-ios-simulator.sh index 78beae366..a882d7e1d 100755 --- a/src/native/runtime/bin/check-postgres18-ios-simulator.sh +++ b/src/native/runtime/bin/check-postgres18-ios-simulator.sh @@ -5,6 +5,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/common.sh" . "$script_dir/../../../third-party/icu/tools/build.sh" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" pg_version="18.4" pg_sha256="81a81ec695fb0c7901407defaa1d2f7973617154cf27ba74e3a7ab8e64436094" diff --git a/src/native/runtime/moon.yml b/src/native/runtime/moon.yml index 383f604a4..44fc8b40d 100644 --- a/src/native/runtime/moon.yml +++ b/src/native/runtime/moon.yml @@ -54,6 +54,7 @@ fileGroups: - src/**/* - patches/**/* - postgres/**/* + - /tools/dev/acquisition.sh - /src/third-party/postgres/**/* - "!/src/third-party/postgres/**/*.test.*" - "!/src/third-party/postgres/testdata/**/*" diff --git a/src/native/runtime/tools/runtime-preflight.sh b/src/native/runtime/tools/runtime-preflight.sh index 866e50392..e5ec8a5a3 100644 --- a/src/native/runtime/tools/runtime-preflight.sh +++ b/src/native/runtime/tools/runtime-preflight.sh @@ -67,23 +67,14 @@ oliphaunt_runtime_native_host_postgres() { printf '%s\n' "${OLIPHAUNT_POSTGRES:-$(oliphaunt_runtime_native_host_install_dir)/bin/postgres$suffix}" } -oliphaunt_runtime_native_host_pg_config() { - case "$(uname -s)" in - MINGW* | MSYS* | CYGWIN*) suffix=.exe ;; - *) suffix= ;; - esac - printf '%s\n' "${OLIPHAUNT_PG_CONFIG:-$(oliphaunt_runtime_native_host_install_dir)/bin/pg_config$suffix}" -} - oliphaunt_runtime_native_host_export_defaults() { LIBOLIPHAUNT_PATH="$(oliphaunt_runtime_native_host_lib)" OLIPHAUNT_INSTALL_DIR="$(oliphaunt_runtime_native_host_install_dir)" OLIPHAUNT_INITDB="$(oliphaunt_runtime_native_host_initdb)" OLIPHAUNT_POSTGRES="$(oliphaunt_runtime_native_host_postgres)" - OLIPHAUNT_PG_CONFIG="$(oliphaunt_runtime_native_host_pg_config)" OLIPHAUNT_POSTGRES_TOOL_DIR="${OLIPHAUNT_POSTGRES_TOOL_DIR:-$OLIPHAUNT_INSTALL_DIR/bin}" export LIBOLIPHAUNT_PATH OLIPHAUNT_INSTALL_DIR OLIPHAUNT_INITDB - export OLIPHAUNT_POSTGRES OLIPHAUNT_PG_CONFIG OLIPHAUNT_POSTGRES_TOOL_DIR + export OLIPHAUNT_POSTGRES OLIPHAUNT_POSTGRES_TOOL_DIR } oliphaunt_runtime_native_host_require() { @@ -95,8 +86,7 @@ oliphaunt_runtime_native_host_require() { [ -f "$LIBOLIPHAUNT_PATH" ] && [ -d "$OLIPHAUNT_INSTALL_DIR" ] && [ -x "$OLIPHAUNT_INITDB" ] && - [ -x "$OLIPHAUNT_POSTGRES" ] && - [ -x "$OLIPHAUNT_PG_CONFIG" ] && return 0 + [ -x "$OLIPHAUNT_POSTGRES" ] && return 0 cat >&2 < 0 and all(int(suite.attrib.get(key, 0)) == 0 for key in ("skipped", "failures", "errors")), "native binding tests must execute and pass" +CHECK diff --git a/src/native/sdks/react-native/tools/check-package.test.mts b/src/native/sdks/react-native/tools/check-package.test.mts index 1127eb836..aba8e84c8 100644 --- a/src/native/sdks/react-native/tools/check-package.test.mts +++ b/src/native/sdks/react-native/tools/check-package.test.mts @@ -1,4 +1,8 @@ import test from 'node:test'; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { stageArtifacts } from './stage-release-artifacts.mts'; import { iosBaseLegalMetadata } from '../../swift/tools/ios-carrier-manifest.mts'; import assert from 'node:assert/strict'; import { validateReactNativePackagedCarrier } from './check-package.mts'; @@ -87,3 +91,49 @@ test('binds the React Native npm carrier bytes to selection-neutral staged evide /must match liboliphaunt-native 1\.2\.3/u, ); }); + +test('package staging accepts frozen current iOS metadata and rejects stale or corrupt metadata', () => { + const root = mkdtempSync(path.join(tmpdir(), 'rn-frozen-carrier-')); + const previous = process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER; + try { + const version = readFileSync( + new URL('../../../runtime/VERSION', import.meta.url), + 'utf8', + ).trim(); + const carrier = selectionNeutralCarrier(version); + const frozen = path.join(root, 'frozen.json'); + const work = path.join(root, 'work'); + const artifacts = path.join(root, 'artifacts'); + mkdirSync(path.join(work, 'package'), { recursive: true }); + writeFileSync(path.join(work, 'package/package.json'), '{}'); + writeFileSync(frozen, JSON.stringify(carrier)); + process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER = frozen; + stageArtifacts(artifacts, work); + const evidence = readFileSync( + path.join(artifacts, 'ios-carriers/oliphaunt-react-native-ios-carriers.json'), + 'utf8', + ); + assert.deepEqual(JSON.parse(evidence), carrier); + assert.equal( + readFileSync(path.join(work, 'package/oliphaunt-react-native-ios-carriers.json'), 'utf8'), + evidence, + ); + for (const mutate of [ + (value) => { + value.base.version = '999.0.0'; + }, + (value) => { + value.base.assets[0].sha256 = 'invalid'; + }, + ]) { + const invalid = structuredClone(carrier); + mutate(invalid); + writeFileSync(frozen, JSON.stringify(invalid)); + assert.throws(() => stageArtifacts(artifacts, work)); + } + } finally { + if (previous === undefined) delete process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER; + else process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER = previous; + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/src/native/sdks/react-native/tools/expo-android-runner.sh b/src/native/sdks/react-native/tools/expo-android-runner.sh index ff47165e7..c3b04a9a2 100755 --- a/src/native/sdks/react-native/tools/expo-android-runner.sh +++ b/src/native/sdks/react-native/tools/expo-android-runner.sh @@ -58,19 +58,10 @@ case "$build_type" in esac build_only="${OLIPHAUNT_EXPO_ANDROID_BUILD_ONLY:-0}" e2e_only="${OLIPHAUNT_EXPO_ANDROID_E2E_ONLY:-0}" -e2e_assertion_runner="${OLIPHAUNT_EXPO_ANDROID_E2E_ASSERTION_RUNNER:-${OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER:-log}}" -case "$e2e_assertion_runner" in - auto|log|maestro) - ;; - *) - echo "error: OLIPHAUNT_EXPO_ANDROID_E2E_ASSERTION_RUNNER must be auto, log, or maestro, got $e2e_assertion_runner" >&2 - exit 1 - ;; -esac + build_type_capitalized="$(printf '%s' "$build_type" | awk '{ print toupper(substr($0, 1, 1)) substr($0, 2) }')" apk="${OLIPHAUNT_EXPO_ANDROID_APK:-$example_dir/android/app/build/outputs/apk/$build_type/app-$build_type.apk}" build_artifact_dir="${OLIPHAUNT_EXPO_ANDROID_BUILD_ARTIFACT_DIR:-$root/target/mobile-build/react-native/android}" -maestro_flow="${OLIPHAUNT_EXPO_ANDROID_MAESTRO_FLOW:-$source_example_dir/maestro/installed-smoke.yaml}" app_id="${OLIPHAUNT_EXPO_ANDROID_APP_ID:-dev.oliphaunt.reactnative.example}" scheme="${OLIPHAUNT_EXPO_ANDROID_SCHEME:-reactnativeoliphauntexpo}" dev_client_scheme="${OLIPHAUNT_EXPO_ANDROID_DEV_CLIENT_SCHEME:-exp+react-native-oliphaunt-expo}" @@ -700,7 +691,12 @@ start_metro_if_needed() { fail "Expo Metro did not start on port $metro_port" } -trap cleanup EXIT +android_log_pid="" +cleanup_android_runner() { + stop_mobile_log_capture "$android_log_pid" + cleanup +} +trap cleanup_android_runner EXIT write_android_package_metrics() { local apk_bytes="$1" @@ -742,7 +738,7 @@ write_android_build_artifact_report() { } main() { - if ! { is_truthy "$e2e_only" && [ "$build_type" = "release" ] && [ "$e2e_assertion_runner" = "maestro" ]; }; then + if ! is_truthy "$e2e_only"; then need_cmd rg fi if [ "$build_type" = "debug" ]; then @@ -771,6 +767,17 @@ main() { prepare_expo_example_workspace pack_react_native_sdk_if_needed ensure_android_project + # Expo's useExpoVersionCatalog reads the installed React Native version catalog. + local expo_ndk + expo_ndk="$(bun - "$example_dir" <<'JS' +import {readFileSync} from 'node:fs'; +import {dirname, join} from 'node:path'; +const rn = require.resolve('react-native/package.json', {paths:[process.argv[2]]}); +console.log(Bun.TOML.parse(readFileSync(join(dirname(rn), 'gradle/libs.versions.toml'), 'utf8')).versions.ndkVersion ?? ''); +JS +)" + [ -n "$expo_ndk" ] || fail "React Native's version catalog must declare ndkVersion" + bash "$root/tools/dev/setup-android-sdk.sh" --sdk-root "$ANDROID_HOME" --expo-ndk-version "$expo_ndk" local runtime_resources jni_libs source_so static_registry_source source_so="$(find_android_liboliphaunt_so)" static_registry_source="$(mobile_static_registry_source_for_library "$source_so")" diff --git a/src/native/sdks/react-native/tools/expo-ios-runner.sh b/src/native/sdks/react-native/tools/expo-ios-runner.sh index 245e56123..3e381e12b 100755 --- a/src/native/sdks/react-native/tools/expo-ios-runner.sh +++ b/src/native/sdks/react-native/tools/expo-ios-runner.sh @@ -68,14 +68,7 @@ background_seconds="${OLIPHAUNT_EXPO_IOS_BACKGROUND_SECONDS:-3}" reuse_installed_app="${OLIPHAUNT_EXPO_IOS_REUSE_INSTALLED_APP:-0}" clean_simulator_install="${OLIPHAUNT_EXPO_IOS_CLEAN_INSTALL:-1}" e2e_only="${OLIPHAUNT_EXPO_IOS_E2E_ONLY:-0}" -e2e_assertion_runner="${OLIPHAUNT_EXPO_IOS_E2E_ASSERTION_RUNNER:-${OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER:-log}}" -case "$e2e_assertion_runner" in - auto | log | maestro) ;; - *) - echo "error: OLIPHAUNT_EXPO_IOS_E2E_ASSERTION_RUNNER must be auto, log, or maestro, got $e2e_assertion_runner" >&2 - exit 1 - ;; -esac + configuration="${OLIPHAUNT_EXPO_IOS_CONFIGURATION:-Debug}" sdk="${OLIPHAUNT_EXPO_IOS_SDK:-iphonesimulator}" destination="${OLIPHAUNT_EXPO_IOS_DESTINATION:-}" @@ -86,7 +79,6 @@ derived_data="$scratch_root/DerivedData" workspace="$example_dir/ios/reactnativeoliphauntexpo.xcworkspace" xcode_scheme="reactnativeoliphauntexpo" build_artifact_dir="${OLIPHAUNT_EXPO_IOS_BUILD_ARTIFACT_DIR:-$root/target/mobile-build/react-native/ios}" -maestro_flow="${OLIPHAUNT_EXPO_IOS_MAESTRO_FLOW:-$source_example_dir/maestro/installed-smoke.yaml}" expo_use_precompiled_modules="${OLIPHAUNT_EXPO_IOS_USE_PRECOMPILED_MODULES:-true}" use_ccache="${OLIPHAUNT_EXPO_IOS_USE_CCACHE:-1}" liboliphaunt_pod_mode="vendored-framework" diff --git a/src/native/sdks/react-native/tools/expo-runner-android-device.sh b/src/native/sdks/react-native/tools/expo-runner-android-device.sh index d9496df95..5edeebf23 100644 --- a/src/native/sdks/react-native/tools/expo-runner-android-device.sh +++ b/src/native/sdks/react-native/tools/expo-runner-android-device.sh @@ -2,90 +2,6 @@ # Shared Android device, logcat, lifecycle, and installed-app helpers for the # Expo Android runner. This file is sourced by expo-android-runner.sh. -should_use_maestro_e2e() { - [ "$runner" = "smoke" ] || return 1 - case "$e2e_assertion_runner" in - maestro) - maestro_binary >/dev/null || fail "missing required command: maestro; run tools/dev/setup-maestro.sh" - return 0 - ;; - auto) - maestro_binary >/dev/null - return - ;; - *) - return 1 - ;; - esac -} - -run_maestro_installed_smoke() { - local device_id="$1" - local adb="${2:-$ANDROID_HOME/platform-tools/adb}" - local reports_dir="$scratch_root/reports" - [ -f "$maestro_flow" ] || fail "missing Maestro installed-app smoke flow: $maestro_flow" - local maestro - maestro="$(maestro_binary)" || fail "missing required command: maestro; run tools/dev/setup-maestro.sh" - mkdir -p "$reports_dir" - echo "==> $maestro --device $device_id test $maestro_flow" - MAESTRO_CLI_NO_ANALYTICS=true \ - MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED=true \ - "$maestro" --device "$device_id" test \ - -e APP_ID="$app_id" \ - -e SMOKE_TIMEOUT_MS="$((timeout_seconds * 1000))" \ - "$maestro_flow" \ - >"$reports_dir/maestro.log" 2>&1 & - local maestro_pid=$! - local failure_receipt="" - while kill -0 "$maestro_pid" 2>/dev/null; do - failure_receipt="$(latest_android_failure_receipt "$adb")" - [ -z "$failure_receipt" ] || break - sleep 1 - done - if [ -z "$failure_receipt" ]; then - # Close the race where the app emits its authoritative failure receipt as - # Maestro exits after observing the failed UI state. - failure_receipt="$(latest_android_failure_receipt "$adb")" - fi - if [ -n "$failure_receipt" ]; then - { - printf '%s\n' "$failure_receipt" - printf 'maestroPid=%s\n' "$maestro_pid" - } >"$reports_dir/maestro-authoritative-failure.txt" - terminate_maestro_process "$maestro_pid" - wait "$maestro_pid" 2>/dev/null || true - printf '%s\n' "$failure_receipt" >&2 - tail -160 "$reports_dir/maestro.log" >&2 || true - return 2 - fi - local maestro_status=0 - wait "$maestro_pid" || maestro_status=$? - if [ "$maestro_status" -ne 0 ]; then - tail -160 "$reports_dir/maestro.log" >&2 || true - return 1 - fi - tail -80 "$reports_dir/maestro.log" >&2 || true -} - -latest_android_failure_receipt() { - local adb="$1" - "$adb" logcat -d -v raw ReactNativeJS:I '*:S' 2>/dev/null | - grep -F "$failure_tag" | - tail -1 || true -} - -terminate_maestro_process() { - local maestro_pid="$1" - kill -0 "$maestro_pid" 2>/dev/null || return 0 - # The checksum-pinned Maestro launcher ends with `exec "$JAVACMD" "$@"`, - # so this PID is the JVM rather than a shell wrapper that could orphan it. - kill -TERM "$maestro_pid" 2>/dev/null || true - # Maestro normally exits immediately on TERM. The KILL fallback prevents a - # wedged launcher from defeating the authoritative app-side fail-fast path. - sleep 0.2 - kill -KILL "$maestro_pid" 2>/dev/null || true -} - latest_metro_tag() { local offset="$1" local tag="$2" @@ -329,37 +245,35 @@ install_and_launch() { local url url="$(android_runner_url "$runner")" local shell_url="'$url'" - run "$adb" shell am start -a android.intent.action.VIEW -d "$shell_url" "$app_id" + local android_log_file="$scratch_root/logs/$runner-logcat.log" + local app_uid + app_uid="$("$adb" shell pm list packages -U --user current "$app_id" | + awk -v package="package:$app_id" '$1 == package {sub(/^uid:/, "", $2); sub(/\r$/, "", $2); print $2}')" + [[ "$app_uid" =~ ^[0-9]+$ ]] || fail "failed to resolve Android app UID for $app_id" + mkdir -p "$scratch_root/logs" + # The installed UID scopes every app process before launch, including early crashes. + "$adb" logcat -v raw --uid="$app_uid" ReactNativeJS:I AndroidRuntime:E '*:S' >"$android_log_file" 2>&1 & + android_log_pid=$! + run "$adb" shell am start -W -a android.intent.action.VIEW -d "$shell_url" "$app_id" if [ "$build_type" = "debug" ]; then dismiss_expo_dev_menu_onboarding "$adb" fi local logs pass - if should_use_maestro_e2e; then - [ "$lifecycle_smoke" != "1" ] || - fail "Maestro mobile E2E does not drive lifecycle transitions; use mobile-drill or set OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE=0" - local maestro_status=0 - run_maestro_installed_smoke "$device_id" "$adb" || maestro_status=$? - if [ "$maestro_status" -ne 0 ]; then - if [ "$maestro_status" -eq 2 ]; then - write_android_e2e_diagnostics "$adb" "authoritative-smoke-failure" - fail "Expo Android installed app emitted $failure_tag while Maestro was running" - fi - write_android_e2e_diagnostics "$adb" "maestro-failure" - fail "Expo Android installed-app Maestro smoke failed" - fi - logs="$("$adb" logcat -d)" - pass="$(latest_metro_tag "$metro_offset" "$success_tag")" - if [ -z "$pass" ]; then - pass="$(printf '%s\n' "$logs" | grep -F "$success_tag" | tail -1 || true)" - fi - if [ -n "$pass" ]; then - printf '\n%s\n' "$pass" - write_runner_report "$pass" - else - write_android_e2e_diagnostics "$adb" "missing-authoritative-pass-receipt" - fail "Expo Android installed-app smoke UI passed without an authoritative OLIPHAUNT_EXPO_SMOKE_PASS receipt" + if [ "$runner" = smoke ] && [ "$lifecycle_smoke" != 1 ]; then + local receipt_status=0 + mobile_app_is_alive() { "$adb" shell pidof "$app_id" >/dev/null 2>&1; } + pass="$(wait_for_mobile_receipt "$android_log_file" "$android_log_pid")" || receipt_status=$? + kill -0 "$android_log_pid" 2>/dev/null || receipt_status=3 + stop_mobile_log_capture "$android_log_pid" + android_log_pid="" + mobile_log_has_failure "$android_log_file" && receipt_status=2 + mobile_app_is_alive || receipt_status=4 + if [ "$receipt_status" != 0 ]; then + write_android_e2e_diagnostics "$adb" receipt-failure + fail "Android smoke receipt failed (status $receipt_status: timeout=1, failure=2, capture=3, app=4)" fi + write_runner_report "$pass" || fail "invalid Android smoke receipt" write_android_process_metrics "$adb" return fi diff --git a/src/native/sdks/react-native/tools/expo-runner-android-device.test.sh b/src/native/sdks/react-native/tools/expo-runner-android-device.test.sh index 45ae2dd3f..22fa35959 100644 --- a/src/native/sdks/react-native/tools/expo-runner-android-device.test.sh +++ b/src/native/sdks/react-native/tools/expo-runner-android-device.test.sh @@ -1,141 +1,123 @@ #!/usr/bin/env bash set -euo pipefail +root="$(git rev-parse --show-toplevel)" +source "$root/src/native/sdks/react-native/tools/expo-runner-common.sh" +scratch="$(mktemp -d)" +capture_pid="" +trap 'stop_mobile_log_capture "$capture_pid"; rm -rf "$scratch"' EXIT +success_tag=OLIPHAUNT_EXPO_SMOKE_PASS +failure_tag=OLIPHAUNT_EXPO_SMOKE_FAIL +timeout_seconds=0 +mobile_app_is_alive() { [ "$app_alive" = true ]; } +sleep 60 & +capture_pid=$! +app_alive=true +for scenario in pass fail crash missing dead-capture dead-app; do + printf '%s valid-receipt\n' "$success_tag" >"$scratch/log" + expected=0 + pid="$capture_pid" + app_alive=true + case "$scenario" in + fail) printf '%s explicit-failure\n' "$failure_tag" >>"$scratch/log"; expected=2 ;; + crash) printf 'FATAL EXCEPTION\n' >>"$scratch/log"; expected=2 ;; + missing) : >"$scratch/log"; expected=1 ;; + dead-capture) pid=99999999; expected=3 ;; + dead-app) app_alive=false; expected=4 ;; + esac + status=0 + wait_for_mobile_receipt "$scratch/log" "$pid" >"$scratch/out" 2>"$scratch/err" || status=$? + [ "$status" = "$expected" ] || { echo "$scenario: expected $expected, got $status" >&2; exit 1; } +done +# A fast terminal receipt remains readable while the current capture is alive. +timeout_seconds=5 +app_alive=true +: >"$scratch/log" +(sleep 0.1; printf '%s valid-receipt\n' "$success_tag" >>"$scratch/log") & +writer=$! +wait_for_mobile_receipt "$scratch/log" "$capture_pid" >"$scratch/out" +wait "$writer" +grep -Fq "$success_tag" "$scratch/out" +stop_mobile_log_capture "$capture_pid" +if kill -0 "$capture_pid" 2>/dev/null; then exit 1; fi +capture_pid="" +echo 'Mobile receipts: failure precedence, crash, missing receipt, capture/app death and cleanup passed' -root="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "must run inside the Oliphaunt git checkout" >&2 - exit 1 -} -. "$root/src/native/sdks/react-native/tools/expo-runner-android-device.sh" -test_root="$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-android-maestro-test.XXXXXX")" -trap 'rm -rf "$test_root"' EXIT - -scratch_root="$test_root/scratch" -maestro_flow="$test_root/installed-smoke.yaml" -app_id="dev.oliphaunt.test" -timeout_seconds=600 -failure_tag="OLIPHAUNT_EXPO_SMOKE_FAIL" -fake_maestro="$test_root/maestro" -fake_adb="$test_root/adb" -export FAKE_MAESTRO_STARTED="$test_root/maestro-started" -export FAKE_MAESTRO_TERMINATED="$test_root/maestro-terminated" -export FAKE_MAESTRO_PID="$test_root/maestro-pid" -export FAKE_MAESTRO_MODE=slow -export FAKE_ADB_FAILURE_FILE="$test_root/adb-failure" - -mkdir -p "$scratch_root" -printf 'appId: dev.oliphaunt.test\n---\n- assertVisible: smoke\n' >"$maestro_flow" - -cat >"$fake_maestro" <<'SH' -#!/usr/bin/env sh -printf '%s\n' "$$" >"$FAKE_MAESTRO_PID" -printf 'started\n' >"$FAKE_MAESTRO_STARTED" -case "$FAKE_MAESTRO_MODE" in - success) - printf 'simulated Maestro success\n' - exit 0 - ;; - error) - printf 'simulated Maestro failure\n' >&2 - exit 7 +# Exercise the installed-app entry point: logcat clearing must discard an old +# PASS, and only this app's receipt emitted after this launch can succeed. +source "$root/src/native/sdks/react-native/tools/expo-runner-android-device.sh" +mkdir -p "$scratch/sdk/platform-tools" "$scratch/app" +cat > "$scratch/sdk/platform-tools/adb" <<'ADB' +#!/usr/bin/env bash +set -eu +case "$*" in + devices) printf 'List of devices attached\nfixture\tdevice\n' ;; + 'install -r '*|'shell am force-stop '*|'shell pm clear '*|'shell pidof '*) ;; + 'shell pm list packages -U --user current dev.oliphaunt.fixture') + printf 'package:dev.oliphaunt.fixture.other uid:10099\r\n' + case "$ADB_SCENARIO" in + missing-uid) ;; + invalid-uid) printf 'package:dev.oliphaunt.fixture uid:invalid\r\n' ;; + *) printf 'package:dev.oliphaunt.fixture uid:10042\r\n' ;; + esac ;; - slow) - trap 'printf "terminated\n" >"$FAKE_MAESTRO_TERMINATED"; exit 143' TERM INT - count=0 - while [ "$count" -lt 50 ]; do - sleep 0.1 - count=$((count + 1)) + 'logcat -c') : > "$ADB_RECEIPT" ;; + 'logcat -v '*) + trap 'exit 0' TERM + uid_filter="" + for arg in "$@"; do + case "$arg" in --uid=*) uid_filter="${arg#--uid=}" ;; esac done - exit 0 + while [ ! -s "$ADB_RECEIPT" ]; do sleep 0.05; done + awk -v uid="$uid_filter" 'uid == "" || $1 == uid {sub(/^[0-9]+ /, ""); print}' "$ADB_RECEIPT" + while :; do sleep 0.05; done ;; - *) - exit 64 + 'shell am start -W '*) + case "$ADB_SCENARIO" in + stale) ;; + unrelated-pass) printf '10099 %s\n' "$ADB_CURRENT_RECEIPT" > "$ADB_RECEIPT" ;; + *) + { + case "$ADB_SCENARIO" in + unrelated-crash) printf '10099 FATAL EXCEPTION: main\n' ;; + app-crash) printf '10042 FATAL EXCEPTION: main\n' ;; + esac + printf '10042 %s\n' "$ADB_CURRENT_RECEIPT" + } > "$ADB_RECEIPT" + ;; + esac ;; + *) echo "unexpected adb invocation: $*" >&2; exit 9 ;; esac -SH -chmod +x "$fake_maestro" - -cat >"$fake_adb" <<'SH' -#!/usr/bin/env sh -if [ "$*" = "logcat -d -v raw ReactNativeJS:I *:S" ] && - [ -f "$FAKE_MAESTRO_STARTED" ] && [ -s "$FAKE_ADB_FAILURE_FILE" ]; then - cat "$FAKE_ADB_FAILURE_FILE" -fi -SH -chmod +x "$fake_adb" - -maestro_binary() { - printf '%s\n' "$fake_maestro" -} - -fail_test() { - echo "expo-runner-android-device.test.sh: $*" >&2 - exit 1 -} - -reset_fixture() { - rm -rf "$scratch_root/reports" - rm -f \ - "$FAKE_MAESTRO_STARTED" \ - "$FAKE_MAESTRO_TERMINATED" \ - "$FAKE_MAESTRO_PID" \ - "$FAKE_ADB_FAILURE_FILE" -} - -reset_fixture -printf '%s\n' "07-18 12:00:03.244 I ReactNativeJS: $failure_tag {\"error\":\"fixture\"}" \ - >"$FAKE_ADB_FAILURE_FILE" -start_seconds=$SECONDS -set +e -run_maestro_installed_smoke emulator-5554 "$fake_adb" \ - >"$test_root/fail-fast.stdout" 2>"$test_root/fail-fast.stderr" -status=$? -set -e -[ "$status" -eq 2 ] || fail_test "authoritative failure returned $status instead of 2" -# Loaded CI hosts can delay the one-second receipt poll and shell process -# reaping even after TERM is delivered. The assertions below independently -# prove that Maestro received TERM and is no longer running. -[ $((SECONDS - start_seconds)) -lt 10 ] || fail_test "authoritative failure did not terminate Maestro promptly" -[ -f "$FAKE_MAESTRO_TERMINATED" ] || fail_test "authoritative failure did not terminate Maestro" -maestro_pid="$(cat "$FAKE_MAESTRO_PID")" -if kill -0 "$maestro_pid" 2>/dev/null; then - fail_test "terminated Maestro process $maestro_pid is still running" -fi -grep -Fq "$failure_tag" "$scratch_root/reports/maestro-authoritative-failure.txt" || - fail_test "authoritative failure report omitted the app receipt" -grep -Fq "$failure_tag" "$test_root/fail-fast.stderr" || - fail_test "authoritative failure was not printed to stderr" - -reset_fixture -export FAKE_MAESTRO_MODE=success -run_maestro_installed_smoke emulator-5554 "$fake_adb" \ - >"$test_root/success.stdout" 2>"$test_root/success.stderr" || - fail_test "successful Maestro run was rejected" -grep -Fq "simulated Maestro success" "$scratch_root/reports/maestro.log" || - fail_test "successful Maestro output was not preserved" - -reset_fixture -export FAKE_MAESTRO_MODE=error -set +e -run_maestro_installed_smoke emulator-5554 "$fake_adb" \ - >"$test_root/error.stdout" 2>"$test_root/error.stderr" -status=$? -set -e -[ "$status" -eq 1 ] || fail_test "failed Maestro run returned $status instead of 1" -grep -Fq "simulated Maestro failure" "$scratch_root/reports/maestro.log" || - fail_test "failed Maestro output was not preserved" - -# Exercise the final logcat read after an immediate Maestro exit. This closes -# the race between the UI assertion ending and the authoritative app receipt. -reset_fixture -export FAKE_MAESTRO_MODE=success -printf '%s\n' "07-18 12:00:03.244 I ReactNativeJS: $failure_tag {\"error\":\"race\"}" \ - >"$FAKE_ADB_FAILURE_FILE" -set +e -run_maestro_installed_smoke emulator-5554 "$fake_adb" \ - >"$test_root/race.stdout" 2>"$test_root/race.stderr" -status=$? -set -e -[ "$status" -eq 2 ] || fail_test "final authoritative failure read returned $status instead of 2" - -echo "Android Maestro fail-fast tests passed" +ADB +chmod +x "$scratch/sdk/platform-tools/adb" +export ADB_RECEIPT="$scratch/adb-receipt" +export ADB_CURRENT_RECEIPT="$success_tag current-launch" +ANDROID_HOME="$scratch/sdk" +app_id=dev.oliphaunt.fixture +apk="$scratch/app.apk" +build_type=release +runner=smoke +lifecycle_smoke=0 +scratch_root="$scratch/app" +timeout_seconds=3 +wake_android_device() { :; } +android_runner_url() { printf 'fixture://smoke'; } +write_android_process_metrics() { :; } +write_android_e2e_diagnostics() { :; } +write_runner_report() { [ "$1" = "$ADB_CURRENT_RECEIPT" ]; } +for scenario in pass unrelated-crash app-crash stale unrelated-pass missing-uid invalid-uid; do + printf '10042 %s stale-launch\n' "$success_tag" > "$ADB_RECEIPT" + export ADB_SCENARIO="$scenario" + status=0 + ( + android_log_pid="" + trap 'stop_mobile_log_capture "$android_log_pid"' EXIT + install_and_launch + ) > "$scratch/$scenario-launch.log" 2>&1 || status=$? + case "$scenario" in + pass|unrelated-crash) [ "$status" = 0 ] ;; + *) [ "$status" != 0 ] ;; + esac || { cat "$scratch/$scenario-launch.log" >&2; echo "unexpected $scenario status: $status" >&2; exit 1; } +done +echo 'Android installed-app capture scopes receipts and crashes to the current app and launch' diff --git a/src/native/sdks/react-native/tools/expo-runner-common.sh b/src/native/sdks/react-native/tools/expo-runner-common.sh index cdec93561..a92faadf5 100644 --- a/src/native/sdks/react-native/tools/expo-runner-common.sh +++ b/src/native/sdks/react-native/tools/expo-runner-common.sh @@ -71,16 +71,38 @@ need_cmd() { command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" } -maestro_binary() { - if command -v maestro >/dev/null 2>&1; then - command -v maestro - return - fi - if [ -x "$HOME/.maestro/bin/maestro" ]; then - printf '%s\n' "$HOME/.maestro/bin/maestro" - return - fi - return 1 +# The app owns SDK assertions. Read only the continuous capture started for this +# launch, and reject failures/dead processes before accepting its receipt. +mobile_log_has_failure() { + grep -Eq "$failure_tag|Fatal error|FATAL EXCEPTION|terminating with uncaught exception" "$1" +} + +wait_for_mobile_receipt() { + local log_file="$1" capture_pid="$2" + local deadline=$((SECONDS + timeout_seconds)) logs pass + while :; do + logs="$(cat "$log_file")" || return 3 + if mobile_log_has_failure "$log_file"; then tail -20 "$log_file" >&2; return 2; fi + kill -0 "$capture_pid" 2>/dev/null || return 3 + mobile_app_is_alive || return 4 + pass="$(printf '%s\n' "$logs" | grep -F "$success_tag" | tail -1 || true)" + if [ -n "$pass" ]; then printf '%s\n' "$pass"; return 0; fi + [ "$SECONDS" -lt "$deadline" ] || return 1 + sleep 1 + done +} + +stop_mobile_log_capture() { + local pid="${1:-}" + [ -n "$pid" ] || return 0 + kill -TERM "$pid" 2>/dev/null || true + local attempts=20 + while kill -0 "$pid" 2>/dev/null && [ "$attempts" -gt 0 ]; do + sleep 0.1 + attempts=$((attempts - 1)) + done + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true } stat_mtime() { diff --git a/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh b/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh index 75a31cee2..9b46344e5 100644 --- a/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh +++ b/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh @@ -30,99 +30,6 @@ latest_metro_runner_failure() { } | grep -E "$failure_tag|metro:bundling:failed|Unable to resolve" | tail -20 || true } -should_use_maestro_e2e() { - [ "$runner" = "smoke" ] || return 1 - [ "$sdk" = "iphonesimulator" ] || return 1 - case "$e2e_assertion_runner" in - maestro) - maestro_binary >/dev/null || fail "missing required command: maestro; run tools/dev/setup-maestro.sh" - return 0 - ;; - auto) - maestro_binary >/dev/null - return - ;; - *) - return 1 - ;; - esac -} - -run_maestro_installed_smoke() { - local device_udid="$1" - local reports_dir="$scratch_root/reports" - [ -f "$maestro_flow" ] || fail "missing Maestro installed-app smoke flow: $maestro_flow" - local maestro - maestro="$(maestro_binary)" || fail "missing required command: maestro; run tools/dev/setup-maestro.sh" - mkdir -p "$reports_dir" - echo "==> $maestro --device $device_udid test $maestro_flow" - MAESTRO_CLI_NO_ANALYTICS=true \ - MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED=true \ - "$maestro" --device "$device_udid" test \ - -e APP_ID="$app_id" \ - -e SMOKE_TIMEOUT_MS="$((timeout_seconds * 1000))" \ - "$maestro_flow" \ - >"$reports_dir/maestro.log" 2>&1 & - local maestro_pid=$! - local failure_receipt="" - local capture_failed=0 - while kill -0 "$maestro_pid" 2>/dev/null; do - failure_receipt="$(latest_ios_simulator_capture_tag "$failure_tag")" - [ -z "$failure_receipt" ] || break - if ! ios_simulator_log_capture_is_alive; then - capture_failed=1 - break - fi - sleep 1 - done - if [ -z "$failure_receipt" ]; then - # Close the race where the app emits its terminal receipt as Maestro exits - # after observing the corresponding UI state. - failure_receipt="$(latest_ios_simulator_capture_tag "$failure_tag")" - fi - if [ -n "$failure_receipt" ]; then - { - printf '%s\n' "$failure_receipt" - printf 'maestroPid=%s\n' "$maestro_pid" - printf 'simulatorLog=%s\n' "${ios_simulator_log_file:-}" - } >"$reports_dir/maestro-authoritative-failure.txt" - terminate_ios_maestro_process "$maestro_pid" - wait "$maestro_pid" 2>/dev/null || true - printf '%s\n' "$failure_receipt" >&2 - tail -160 "$reports_dir/maestro.log" >&2 || true - return 2 - fi - if [ "$capture_failed" = "1" ]; then - { - printf 'maestroPid=%s\n' "$maestro_pid" - printf 'simulatorLog=%s\n' "${ios_simulator_log_file:-}" - printf 'reason=unified-log-capture-ended-before-maestro\n' - } >"$reports_dir/maestro-log-capture-failure.txt" - terminate_ios_maestro_process "$maestro_pid" - wait "$maestro_pid" 2>/dev/null || true - tail -160 "$reports_dir/maestro.log" >&2 || true - [ -z "${ios_simulator_log_file:-}" ] || tail -160 "$ios_simulator_log_file" >&2 || true - return 3 - fi - local maestro_status=0 - wait "$maestro_pid" || maestro_status=$? - if [ "$maestro_status" -ne 0 ]; then - tail -160 "$reports_dir/maestro.log" >&2 || true - return 1 - fi - tail -80 "$reports_dir/maestro.log" >&2 || true -} - -terminate_ios_maestro_process() { - local maestro_pid="$1" - kill -0 "$maestro_pid" 2>/dev/null || return 0 - # The checksum-pinned Maestro launcher ends with `exec "$JAVACMD" "$@"`, - # so this PID is the JVM rather than a shell wrapper that could orphan it. - kill -TERM "$maestro_pid" 2>/dev/null || true - sleep 0.2 - kill -KILL "$maestro_pid" 2>/dev/null || true -} - resolve_ios_app_process_name() { local app="$1" local plist="$app/Info.plist" @@ -178,7 +85,7 @@ start_ios_simulator_log_capture() { ios_simulator_log_pid=$! # Start the stream before launching the app so a fast Release smoke cannot - # age out while Maestro starts its driver. The scoped file is also durable + # age out before the runner observes it. The scoped file is also durable # evidence and cannot contain a receipt from an already-terminated launch. sleep "${OLIPHAUNT_EXPO_IOS_LOG_CAPTURE_STARTUP_SECONDS:-1}" if ! ios_simulator_log_capture_is_alive; then @@ -216,62 +123,6 @@ stop_ios_simulator_log_capture() { return 0 } -latest_ios_simulator_capture_tag() { - local tag="$1" - [ -n "${ios_simulator_log_file:-}" ] && [ -f "$ios_simulator_log_file" ] || return 0 - grep -F "$tag" "$ios_simulator_log_file" | tail -1 || true -} - -wait_for_ios_simulator_maestro_receipt() { - local grace_seconds="${OLIPHAUNT_EXPO_IOS_RECEIPT_GRACE_SECONDS:-15}" - case "$grace_seconds" in - '' | *[!0-9]*) - echo "OLIPHAUNT_EXPO_IOS_RECEIPT_GRACE_SECONDS must be a nonnegative integer, got $grace_seconds" >&2 - return 4 - ;; - esac - local deadline=$((SECONDS + grace_seconds)) - local pass failure_receipt - while :; do - failure_receipt="$(latest_ios_simulator_capture_tag "$failure_tag")" - if [ -n "$failure_receipt" ]; then - printf '%s\n' "$failure_receipt" >&2 - return 2 - fi - pass="$(latest_ios_simulator_capture_tag "$success_tag")" - if [ -n "$pass" ]; then - printf '%s\n' "$pass" - return 0 - fi - ios_simulator_log_capture_is_alive || return 3 - [ "$SECONDS" -lt "$deadline" ] || return 1 - sleep 1 - done -} - -write_ios_maestro_diagnostics() { - local device_udid="$1" - local process_name="$2" - local reason="$3" - local reports_dir="$scratch_root/reports" - mkdir -p "$reports_dir" - { - printf 'reason=%s\n' "$reason" - printf 'deviceUdid=%s\n' "$device_udid" - printf 'processName=%s\n' "$process_name" - printf 'simulatorLog=%s\n' "${ios_simulator_log_file:-}" - } >"$reports_dir/maestro-$reason.txt" - xcrun simctl spawn "$device_udid" log show \ - --style compact \ - --last 15m \ - --predicate "process == '$process_name'" \ - >"$reports_dir/maestro-unified-log-$reason.txt" 2>&1 || true - xcrun simctl io "$device_udid" screenshot \ - "$reports_dir/maestro-screen-$reason.png" >/dev/null 2>&1 || true - [ -s "$reports_dir/maestro-screen-$reason.png" ] || rm -f "$reports_dir/maestro-screen-$reason.png" - tail -200 "$reports_dir/maestro-unified-log-$reason.txt" >&2 || true -} - write_ios_process_metrics() { local launch_pid="$1" [ -n "$launch_pid" ] || return 0 @@ -707,16 +558,10 @@ install_and_launch() { local scratch_metro_offset dev_metro_offset scratch_metro_offset="$(file_bytes "$scratch_root/metro.log")" dev_metro_offset="$(file_bytes "$metro_dev_log")" - local use_maestro_e2e=0 app_process_name="" - if should_use_maestro_e2e; then - [ "$lifecycle_smoke" != "1" ] || - fail "Maestro mobile E2E does not drive lifecycle transitions; use mobile-drill or set OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE=0" - app_process_name="$(resolve_ios_app_process_name "$app")" || - fail "failed to resolve the installed iOS app executable for unified-log capture" - start_ios_simulator_log_capture "$device_udid" "$app_process_name" || - fail "failed to start exact-launch iOS unified-log capture" - use_maestro_e2e=1 - fi + local app_process_name + app_process_name="$(resolve_ios_app_process_name "$app")" || fail "missing iOS executable" + start_ios_simulator_log_capture "$device_udid" "$app_process_name" || + fail "failed to start exact-launch iOS unified-log capture" local url url="$(ios_runner_url "$runner")" local launch_output launch_pid @@ -731,45 +576,18 @@ install_and_launch() { fi local logs pass - if [ "$use_maestro_e2e" = "1" ]; then - local maestro_status=0 - run_maestro_installed_smoke "$device_udid" || maestro_status=$? - if [ "$maestro_status" -ne 0 ]; then - stop_ios_simulator_log_capture || true - if [ "$maestro_status" = "2" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "authoritative-smoke-failure" - fail "Expo iOS installed app emitted $failure_tag while Maestro was running" - fi - if [ "$maestro_status" = "3" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "log-capture-failure" - fail "Expo iOS exact-launch unified-log capture ended while Maestro was running" - fi - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "maestro-failure" - fail "Expo iOS installed-app Maestro smoke failed" - fi - + if [ "$runner" = smoke ] && [ "$lifecycle_smoke" != 1 ]; then local receipt_status=0 - pass="$(wait_for_ios_simulator_maestro_receipt)" || receipt_status=$? - stop_ios_simulator_log_capture || true - if [ "$receipt_status" = "0" ]; then - printf '\n%s\n' "$pass" - if ! write_runner_report "$pass"; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "invalid-authoritative-pass-receipt" - fail "Expo iOS app emitted an invalid authoritative OLIPHAUNT_EXPO_SMOKE_PASS receipt" - fi - elif [ "$receipt_status" = "2" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "authoritative-smoke-failure" - fail "Expo iOS installed app emitted $failure_tag after Maestro observed the UI" - elif [ "$receipt_status" = "3" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "log-capture-failure" - fail "Expo iOS exact-launch unified-log capture ended before the app receipt was collected" - elif [ "$receipt_status" = "4" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "invalid-receipt-grace" - fail "Expo iOS receipt grace configuration is invalid" - else - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "missing-authoritative-pass-receipt" - fail "Expo iOS installed-app smoke UI passed without an authoritative OLIPHAUNT_EXPO_SMOKE_PASS receipt" + mobile_app_is_alive() { [ -n "$launch_pid" ] && kill -0 "$launch_pid" 2>/dev/null; } + pass="$(wait_for_mobile_receipt "$ios_simulator_log_file" "$ios_simulator_log_pid")" || receipt_status=$? + stop_ios_simulator_log_capture || receipt_status=3 + mobile_log_has_failure "$ios_simulator_log_file" && receipt_status=2 + mobile_app_is_alive || receipt_status=4 + if [ "$receipt_status" != 0 ]; then + tail -200 "$ios_simulator_log_file" >&2 || true + fail "iOS smoke receipt failed (status $receipt_status: timeout=1, failure=2, capture=3, app=4)" fi + write_runner_report "$pass" || fail "invalid iOS smoke receipt" write_ios_process_metrics "$launch_pid" return fi @@ -777,7 +595,7 @@ install_and_launch() { local deadline=$((SECONDS + timeout_seconds)) local fail_line lifecycle_exercised=0 while [ "$SECONDS" -lt "$deadline" ]; do - logs="$(xcrun simctl spawn "$device_udid" log show --style compact --last 30s --predicate "process == 'reactnativeoliphauntexpo'" 2>/dev/null || true)" + logs="$(cat "$ios_simulator_log_file")" if [ "$lifecycle_smoke" = "1" ] && [ "$lifecycle_exercised" = "0" ]; then if { printf '%s\n' "$logs" diff --git a/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.test.sh b/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.test.sh index eec4544b6..56f85e1fe 100644 --- a/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.test.sh +++ b/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.test.sh @@ -60,52 +60,12 @@ for profile in standard icu; do fi done scratch_root="$test_root/scratch" -maestro_flow="$test_root/installed-smoke.yaml" -app_id="dev.oliphaunt.test" -runner="smoke" -mobile_platform="ios" -timeout_seconds=600 -success_tag="OLIPHAUNT_EXPO_SMOKE_PASS" -failure_tag="OLIPHAUNT_EXPO_SMOKE_FAIL" -ios_simulator_log_pid="" -ios_simulator_log_file="$test_root/simulator.log" +runner=smoke +mobile_platform=ios +success_tag=OLIPHAUNT_EXPO_SMOKE_PASS export CI_HEAD_SHA="$(git rev-parse HEAD)" export OLIPHAUNT_MOBILE_E2E_EXPECT_ICU=0 export OLIPHAUNT_MOBILE_E2E_EXPECT_CATALOG_PROFILE=standard -export FAKE_MAESTRO_STARTED="$test_root/maestro-started" -export FAKE_MAESTRO_TERMINATED="$test_root/maestro-terminated" - -mkdir -p "$scratch_root/reports" -printf 'appId: dev.oliphaunt.test\n---\n- assertVisible: smoke\n' >"$maestro_flow" -fake_maestro="$test_root/maestro" -cat >"$fake_maestro" <<'SH' -#!/usr/bin/env bash -trap 'printf "terminated\n" >"$FAKE_MAESTRO_TERMINATED"; exit 143' TERM INT -printf 'started\n' >"$FAKE_MAESTRO_STARTED" -while :; do sleep 0.1; done -SH -chmod +x "$fake_maestro" - -maestro_binary() { printf '%s\n' "$fake_maestro"; } -ios_simulator_log_capture_is_alive() { return 0; } -latest_ios_simulator_capture_tag() { - [ "$1" = "$failure_tag" ] || return 0 - local attempts=100 - while [ "$attempts" -gt 0 ] && [ ! -f "$FAKE_MAESTRO_STARTED" ]; do - command sleep 0.01 - attempts=$((attempts - 1)) - done - printf '%s fixture\n' "$failure_tag" -} - -set +e -run_maestro_installed_smoke simulator-1 >"$test_root/fail.stdout" 2>"$test_root/fail.stderr" -status=$? -set -e -[ "$status" -eq 2 ] -[ -f "$FAKE_MAESTRO_TERMINATED" ] -grep -Fq "$failure_tag" "$scratch_root/reports/maestro-authoritative-failure.txt" - receipt_json="$( bun "$root/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.fixture.mts" receipt "$root/src/extensions/generated/sdk/extensions.json" )" diff --git a/src/native/sdks/react-native/tools/expo-runner-reporting.mts b/src/native/sdks/react-native/tools/expo-runner-reporting.mts index 16b04a416..491a9541d 100644 --- a/src/native/sdks/react-native/tools/expo-runner-reporting.mts +++ b/src/native/sdks/react-native/tools/expo-runner-reporting.mts @@ -175,27 +175,6 @@ switch (command) { ); break; } - case 'maestro-report': { - const report = { - runner: 'maestro', - platform: process.env.OLIPHAUNT_MAESTRO_PLATFORM, - appId: process.env.OLIPHAUNT_MAESTRO_APP_ID, - flow: process.env.OLIPHAUNT_MAESTRO_FLOW, - passedAt: new Date().toISOString(), - }; - process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); - break; - } - case 'maestro-pass': { - const report = { - runner: 'maestro', - platform: process.env.OLIPHAUNT_MAESTRO_PLATFORM, - appId: process.env.OLIPHAUNT_MAESTRO_APP_ID, - flow: process.env.OLIPHAUNT_MAESTRO_FLOW, - }; - process.stdout.write(`OLIPHAUNT_EXPO_MAESTRO_PASS ${JSON.stringify(report)}\n`); - break; - } case 'package-sizes': { const [report, artifactSizeKey, artifactBytes, rnPackageBytes] = args; const payload = { diff --git a/src/native/sdks/react-native/tools/expo-runner-reporting.sh b/src/native/sdks/react-native/tools/expo-runner-reporting.sh index 198f26f42..655808f0d 100644 --- a/src/native/sdks/react-native/tools/expo-runner-reporting.sh +++ b/src/native/sdks/react-native/tools/expo-runner-reporting.sh @@ -2,7 +2,7 @@ # Shared report helpers for React Native Expo mobile runners. Platform runners # own platform metrics and artifact copying; this file only normalizes runner -# pass/report JSON emitted from Metro logs or Maestro installed-app flows. +# pass/report JSON emitted by the app through Metro or platform logs. configure_mobile_catalog_profile_probe() { local profile="$1" @@ -307,19 +307,6 @@ verify_mobile_e2e_smoke_receipt() { echo "$platform mobile E2E extension receipt postcondition: $receipt" >&2 } -write_maestro_runner_report() { - local platform="$1" - local reports_dir="$scratch_root/reports" - mkdir -p "$reports_dir" - OLIPHAUNT_MAESTRO_PLATFORM="$platform" \ - OLIPHAUNT_MAESTRO_APP_ID="$app_id" \ - OLIPHAUNT_MAESTRO_FLOW="$maestro_flow" \ - bun "$root/src/native/sdks/react-native/tools/expo-runner-reporting.mts" maestro-report >"$reports_dir/$runner-report.json" - OLIPHAUNT_MAESTRO_PLATFORM="$platform" \ - OLIPHAUNT_MAESTRO_APP_ID="$app_id" \ - OLIPHAUNT_MAESTRO_FLOW="$maestro_flow" \ - bun "$root/src/native/sdks/react-native/tools/expo-runner-reporting.mts" maestro-pass >"$reports_dir/$runner-pass.log" -} write_mobile_package_size_report() { local artifact_size_key="$1" diff --git a/src/native/sdks/react-native/tools/mobile-e2e.sh b/src/native/sdks/react-native/tools/mobile-e2e.sh index f2a6d5727..90fc49ffe 100755 --- a/src/native/sdks/react-native/tools/mobile-e2e.sh +++ b/src/native/sdks/react-native/tools/mobile-e2e.sh @@ -36,7 +36,6 @@ case "$platform" in export OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE="${OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE:-release}" export OLIPHAUNT_EXPO_ANDROID_E2E_ONLY=1 export OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE="${OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE:-0}" - export OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER="${OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER:-maestro}" export OLIPHAUNT_EXPO_ANDROID_SCRATCH="$mobile_scratch" if [ "$mobile_runner" = "smoke" ]; then command -v unzip >/dev/null 2>&1 || { @@ -70,7 +69,6 @@ case "$platform" in export OLIPHAUNT_EXPO_IOS_CONFIGURATION="${OLIPHAUNT_EXPO_IOS_CONFIGURATION:-Release}" export OLIPHAUNT_EXPO_IOS_E2E_ONLY=1 export OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE="${OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE:-0}" - export OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER="${OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER:-maestro}" export OLIPHAUNT_EXPO_IOS_SCRATCH="$mobile_scratch" if [ "$mobile_runner" = "smoke" ]; then export_mobile_e2e_icu_expectation_from_ios_app "$app" diff --git a/src/native/sdks/react-native/tools/stage-release-artifacts.mts b/src/native/sdks/react-native/tools/stage-release-artifacts.mts index fc072cd6c..3e25a28e4 100644 --- a/src/native/sdks/react-native/tools/stage-release-artifacts.mts +++ b/src/native/sdks/react-native/tools/stage-release-artifacts.mts @@ -11,13 +11,15 @@ export function stageArtifacts(artifactRoot, workRoot) { const releasePackageDir = path.join(workRoot, 'package'); requireDir(releasePackageDir); const assetDir = process.env.OLIPHAUNT_REACT_NATIVE_IOS_RELEASE_ASSET_DIR; - if (!assetDir) { + const baseCarrierManifest = process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER || undefined; + if (!assetDir && !baseCarrierManifest) { fail( 'oliphaunt-react-native package artifacts require OLIPHAUNT_REACT_NATIVE_IOS_RELEASE_ASSET_DIR', ); } const carrier = buildIosCarrierManifest({ baseAssetDir: assetDir, + baseCarrierManifest, extensionManifests: [], }); writeFileSync( diff --git a/src/native/sdks/rust/moon.yml b/src/native/sdks/rust/moon.yml index 7077e31dc..70acd31e1 100644 --- a/src/native/sdks/rust/moon.yml +++ b/src/native/sdks/rust/moon.yml @@ -94,26 +94,30 @@ tasks: cache: true runFromWorkspaceRoot: true test-integration: - tags: ["regression", "runtime"] + tags: ["ci-native-consumers", "platform-linux-x64-gnu", "regression", "runtime", "requires-rust"] script: | set -e - . src/native/runtime/tools/runtime-preflight.sh - oliphaunt_runtime_native_host_require basic - cargo test -p oliphaunt --locked --test native_smoke --test native_sql_regression -- --test-threads=1 + bash src/native/sdks/tests/with-runtime.sh --tools --broker cargo nextest run -p oliphaunt --locked --test native_smoke --test native_sql_regression --no-tests=fail --test-threads=1 env: CARGO_TARGET_DIR: "target" deps: - target: "~:cargo-sources" cacheStrategy: hash - - "liboliphaunt-native:build-runtime-desktop-target" + - "liboliphaunt-native:package-runtime-desktop-target" + - "postgres-tools-native:package-assets" + - "oliphaunt-broker:build-release-assets" inputs: + - "/src/native/sdks/tests/with-runtime.sh" - "@group(cargo-workspace)" - project: "shared-test-fixtures" group: "fixtures" - - "@group(code)" + - "@group(sources)" + - "tests/native_smoke.rs" + - "tests/native_sql_regression.rs" + - "tests/support/**/*" - "/src/native/runtime/tools/runtime-preflight.sh" options: - cache: local + cache: false runFromWorkspaceRoot: true test-extensions: tags: ["regression", "runtime", "extensions"] @@ -273,8 +277,13 @@ tasks: packaging-unit: tags: ["quality", "unit", "requires-rust"] - command: "bash src/native/sdks/rust/tools/prepare-rust-release-source.test.sh" + script: | + set -e + bash src/native/sdks/rust/tools/prepare-rust-release-source.test.sh + bash src/native/sdks/tests/with-runtime.test.sh inputs: + - "/src/native/sdks/tests/with-runtime*.sh" + - "/src/native/runtime/tools/runtime-preflight.sh" - "**/*" - "/src/query/rust/src/lib.rs" - "/src/native/rust-bindings/**/*" diff --git a/src/native/sdks/swift/moon.yml b/src/native/sdks/swift/moon.yml index ca19832b3..ddbc36c2d 100644 --- a/src/native/sdks/swift/moon.yml +++ b/src/native/sdks/swift/moon.yml @@ -106,20 +106,20 @@ tasks: - "@group(code)" - "/src/native/runtime/include/oliphaunt.h" test-native: - tags: ["runtime", "smoke", "requires-swift"] + tags: ["runtime", "smoke", "requires-swift", "ci-native-consumers", "platform-linux-x64-gnu"] script: | - set -e - . src/native/runtime/tools/runtime-preflight.sh - oliphaunt_runtime_native_host_require basic - env OLIPHAUNT_SWIFT_REQUIRE_NATIVE=1 \ - LIBOLIPHAUNT_PATH="$(oliphaunt_runtime_native_host_lib)" \ - OLIPHAUNT_INSTALL_DIR="$(oliphaunt_runtime_native_host_install_dir)" \ - swift test --package-path src/native/sdks/swift --filter NativeRuntimeTests + set -euo pipefail + log="$(mktemp)" + trap 'rm -f "$log"' EXIT + bash src/native/sdks/tests/with-runtime.sh env OLIPHAUNT_SWIFT_REQUIRE_NATIVE=1 \ + swift test --package-path src/native/sdks/swift --filter NativeRuntimeTests 2>&1 | tee "$log" + grep -Eq 'Test run with [1-9][0-9]* tests? .*passed' "$log" deps: - "oliphaunt-swift:prepare-bindings" - {target: "oliphaunt-mobile-bindings:cargo-sources", cacheStrategy: hash} - - "liboliphaunt-native:build-runtime-desktop-target" + - "liboliphaunt-native:package-runtime-desktop-target" inputs: + - "/src/native/sdks/tests/with-runtime.sh" - project: "shared-test-fixtures" group: "fixtures" - project: "cluster-seed-contract" @@ -127,12 +127,15 @@ tasks: - "/src/native/runtime/include/oliphaunt.h" - project: "liboliphaunt-native" group: "runtime" - - "@group(code)" + - "Sources/**/*" + - "Tests/**/*" + - "Package.swift" + - ".swift-version" - "!/src/native/sdks/swift/.build" - "!/src/native/sdks/swift/.build/**" - "/src/native/runtime/tools/runtime-preflight.sh" options: - cache: local + cache: false runFromWorkspaceRoot: true package-source: tags: ["package"] diff --git a/src/native/sdks/swift/tools/prepare-bindings.sh b/src/native/sdks/swift/tools/prepare-bindings.sh index 6de4f05b4..065701701 100644 --- a/src/native/sdks/swift/tools/prepare-bindings.sh +++ b/src/native/sdks/swift/tools/prepare-bindings.sh @@ -2,7 +2,6 @@ set -euo pipefail root="$(git rev-parse --show-toplevel)" cd "$root" -bash src/native/mobile-bindings/tools/generate.sh stage="$root/src/native/sdks/swift/.build/native-bindings" mkdir -p "$stage/swift" "$stage/ffi" cp target/mobile-bindings/generated/OliphauntNativeBindings.swift "$stage/swift/" diff --git a/src/native/sdks/swift/tools/swift.sh b/src/native/sdks/swift/tools/swift.sh index 4ecd00548..f9caaaffa 100644 --- a/src/native/sdks/swift/tools/swift.sh +++ b/src/native/sdks/swift/tools/swift.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail root="$(git rev-parse --show-toplevel)" +bash "$root/src/native/mobile-bindings/tools/generate.sh" bash "$root/src/native/sdks/swift/tools/prepare-bindings.sh" exec swift "$@" --package-path "$root/src/native/sdks/swift" diff --git a/src/native/sdks/tests/README.md b/src/native/sdks/tests/README.md index 8066ca3bf..1f97d4ed3 100644 --- a/src/native/sdks/tests/README.md +++ b/src/native/sdks/tests/README.md @@ -29,7 +29,7 @@ runtime resources plus a seed. Do not mix a released runtime with changed C APIs The Swift builder expects these repository-relative artifacts: -- `target/mobile-bindings/generated/` (run the Swift `prepare-bindings.sh`). +- `target/mobile-bindings/generated/` (run `moon run oliphaunt-swift:prepare-bindings`). - `target/aarch64-apple-ios/debug/liboliphaunt_mobile_bindings.a` and `target/aarch64-apple-ios-sim/debug/liboliphaunt_mobile_bindings.a`. - `target/liboliphaunt-ios-simulator/out/liboliphaunt.dylib`. diff --git a/src/native/sdks/tests/with-runtime.sh b/src/native/sdks/tests/with-runtime.sh new file mode 100644 index 000000000..872162be9 --- /dev/null +++ b/src/native/sdks/tests/with-runtime.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" +cd "$root" +. src/native/runtime/tools/runtime-preflight.sh +target="$(oliphaunt_runtime_native_host_target_id)" +version="$(bun tools/release/product-version.mts version liboliphaunt-native)" +assets="${OLIPHAUNT_LIBOLIPHAUNT_RELEASE_ASSETS:-$root/target/liboliphaunt/desktop-release-assets/$target}" +stage="$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-sdk-runtime.XXXXXX")" +trap 'rm -rf "$stage"' EXIT +extract() { + mkdir -p "$2" + case "$target" in + windows-*) unzip -q "$1.zip" -d "$2" ;; + *) tar -xzf "$1.tar.gz" -C "$2" ;; + esac +} +extract "$assets/liboliphaunt-$version-$target" "$stage" +suffix= +case "$target" in + windows-*) library=bin/oliphaunt.dll; suffix=.exe ;; + macos-*) library=lib/liboliphaunt.dylib ;; + *) library=lib/liboliphaunt.so ;; +esac +export LIBOLIPHAUNT_PATH="$stage/$library" +export OLIPHAUNT_INSTALL_DIR="$stage/runtime" +export OLIPHAUNT_INITDB="$stage/runtime/bin/initdb$suffix" +export OLIPHAUNT_POSTGRES="$stage/runtime/bin/postgres$suffix" +export OLIPHAUNT_EMBEDDED_MODULE_DIR="$stage/lib/modules" +export LD_LIBRARY_PATH="$stage/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +while [ "${1:-}" = --tools ] || [ "${1:-}" = --broker ]; do + option="$1" + shift + if [ "$option" = --tools ]; then + tools_version="$(bun tools/release/product-version.mts version postgres-tools-native)" + tools_assets="${OLIPHAUNT_POSTGRES_TOOLS_RELEASE_ASSETS:-$root/target/postgres-tools/native/release-assets}" + extract "$tools_assets/oliphaunt-tools-$tools_version-$target" "$stage/tools" + export OLIPHAUNT_TOOLS_DIR="$stage/tools/runtime" + else + broker_version="$(bun tools/release/product-version.mts version oliphaunt-broker)" + broker_assets="${OLIPHAUNT_BROKER_RELEASE_ASSETS:-$root/target/oliphaunt-broker/release-assets}" + extract "$broker_assets/oliphaunt-broker-$broker_version-$target" "$stage/broker" + export OLIPHAUNT_BROKER="$stage/broker/bin/oliphaunt-broker$suffix" + test -x "$OLIPHAUNT_BROKER" + fi +done +[ "$#" -gt 0 ] || { echo "usage: with-runtime.sh [--tools] [--broker] COMMAND [ARGS...]" >&2; exit 2; } +oliphaunt_runtime_native_host_require basic +"$@" diff --git a/src/native/sdks/tests/with-runtime.test.sh b/src/native/sdks/tests/with-runtime.test.sh new file mode 100644 index 000000000..162fee94e --- /dev/null +++ b/src/native/sdks/tests/with-runtime.test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" "$scratch/assets" "$scratch/payload/lib" "$scratch/payload/runtime/bin" +cat > "$scratch/bin/bun" <<'BUN' +#!/usr/bin/env bash +printf '1.0.0\n' +BUN +chmod +x "$scratch/bin/bun" +. src/native/runtime/tools/runtime-preflight.sh +target="$(oliphaunt_runtime_native_host_target_id)" +case "$target" in + linux-*) library=liboliphaunt.so ;; + macos-*) library=liboliphaunt.dylib ;; + *) echo 'archive fixture runs on Unix hosts'; exit 0 ;; +esac +touch "$scratch/payload/lib/$library" +for tool in initdb postgres; do + printf '#!/bin/sh\nexit 0\n' > "$scratch/payload/runtime/bin/$tool" + chmod +x "$scratch/payload/runtime/bin/$tool" +done +archive="$scratch/assets/liboliphaunt-1.0.0-$target.tar.gz" +tar -czf "$archive" -C "$scratch/payload" . +export PATH="$scratch/bin:$PATH" +export OLIPHAUNT_LIBOLIPHAUNT_RELEASE_ASSETS="$scratch/assets" +export OLIPHAUNT_INITDB=/must/not/use/a/previous/runtime +"$BASH" src/native/sdks/tests/with-runtime.sh bash -c ' + test -f "$LIBOLIPHAUNT_PATH" + test -x "$OLIPHAUNT_INITDB" + test ! -e "$OLIPHAUNT_INSTALL_DIR/bin/pg_config" + printf "%s\n" "$OLIPHAUNT_INSTALL_DIR" > "$1" +' -- "$scratch/staged" +test ! -e "$(cat "$scratch/staged")" +tar -czf "$scratch/assets/oliphaunt-tools-1.0.0-$target.tar.gz" -C "$scratch/payload" . +mkdir -p "$scratch/broker/bin" +cp "$scratch/payload/runtime/bin/postgres" "$scratch/broker/bin/oliphaunt-broker" +tar -czf "$scratch/assets/oliphaunt-broker-1.0.0-$target.tar.gz" -C "$scratch/broker" . +export OLIPHAUNT_POSTGRES_TOOLS_RELEASE_ASSETS="$scratch/assets" +export OLIPHAUNT_BROKER_RELEASE_ASSETS="$scratch/assets" +"$BASH" src/native/sdks/tests/with-runtime.sh --tools --broker bash -c ' + test -x "$OLIPHAUNT_TOOLS_DIR/bin/postgres" + test -x "$OLIPHAUNT_BROKER" +' +status=0 +"$BASH" src/native/sdks/tests/with-runtime.sh bash -c 'exit 43' || status=$? +test "$status" = 43 +rm "$scratch/payload/runtime/bin/initdb" +tar -czf "$archive" -C "$scratch/payload" . +if "$BASH" src/native/sdks/tests/with-runtime.sh true > "$scratch/missing.log" 2>&1; then + echo 'incomplete runtime accepted' >&2; exit 1 +fi +grep -q 'missing native Oliphaunt runtime artifacts' "$scratch/missing.log" +echo 'SDK runtime staging: shipped payload, isolated paths, cleanup and failure propagation passed' diff --git a/src/third-party/postgres/fetch-source.sh b/src/third-party/postgres/fetch-source.sh index 34dabe795..24eb62506 100644 --- a/src/third-party/postgres/fetch-source.sh +++ b/src/third-party/postgres/fetch-source.sh @@ -2,7 +2,7 @@ # Shared, fail-closed transport for the pinned PostgreSQL source archive. # Keep this file POSIX-compatible: native and WASIX build scripts source it on -# both macOS and Linux. +# both macOS and Linux. Callers source tools/dev/acquisition.sh first. oliphaunt_postgresql_sha256_file() ( oliphaunt_sha_path="${1:?oliphaunt_postgresql_sha256_file requires a path}" @@ -17,6 +17,7 @@ oliphaunt_postgresql_sha256_file() ( ) oliphaunt_fetch_postgresql_source_archive() ( + oliphaunt_acquisition_start 'PostgreSQL source archive' 180 || return $? if [ "$#" -ne 4 ]; then echo "usage: oliphaunt_fetch_postgresql_source_archive DESTINATION VERSION SHA256 PRIMARY_URL" >&2 return 2 @@ -91,17 +92,12 @@ oliphaunt_fetch_postgresql_source_archive() ( for oliphaunt_url in "$oliphaunt_primary_url" "$oliphaunt_fallback_url"; do rm -f "$oliphaunt_partial" oliphaunt_curl_status=0 - if curl \ + if oliphaunt_acquisition_curl 90 5 3 curl \ --location \ --fail \ --silent \ --show-error \ - --retry 4 \ - --retry-all-errors \ - --retry-delay 3 \ - --retry-max-time 90 \ --connect-timeout 20 \ - --max-time 60 \ --max-filesize 67108864 \ --proto '=https' \ --proto-redir '=https' \ @@ -117,6 +113,7 @@ oliphaunt_fetch_postgresql_source_archive() ( echo "discarding PostgreSQL $oliphaunt_version source from $oliphaunt_url with checksum $oliphaunt_actual_sha instead of $oliphaunt_expected_sha" >&2 else oliphaunt_curl_status=$? + case "$oliphaunt_curl_status" in 126|127|129|130|137|143) return "$oliphaunt_curl_status" ;; esac echo "PostgreSQL $oliphaunt_version source download from $oliphaunt_url failed after bounded retries (curl exit $oliphaunt_curl_status)" >&2 fi done diff --git a/src/third-party/postgres/fetch-source.test.sh b/src/third-party/postgres/fetch-source.test.sh index c72e761f1..91d17eafc 100644 --- a/src/third-party/postgres/fetch-source.test.sh +++ b/src/third-party/postgres/fetch-source.test.sh @@ -2,6 +2,7 @@ set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$script_dir/../../../tools/dev/acquisition.sh" . "$script_dir/fetch-source.sh" fail() { @@ -32,12 +33,9 @@ assert_transport_flags() { for expected in \ '--location' \ '--fail' \ - '--retry 4' \ - '--retry-all-errors' \ - '--retry-delay 3' \ - '--retry-max-time 90' \ + '--retry 0' \ '--connect-timeout 20' \ - '--max-time 60' \ + '--max-time' \ '--max-filesize 67108864' \ '--proto =https' \ '--proto-redir =https' \ @@ -59,6 +57,8 @@ fake_bin="$work_root/fake-bin" mkdir -p "$fake_bin" cp "$script_dir/testdata/curl" "$fake_bin/curl" chmod 0755 "$fake_bin/curl" +printf '#!/bin/sh\nexit 0\n' > "$fake_bin/sleep" +chmod +x "$fake_bin/sleep" fake_path="$fake_bin:$PATH" primary_url="https://primary.invalid/postgresql-18.4.tar.bz2" fallback_url="https://fossies.org/linux/misc/postgresql-18.4.tar.bz2" @@ -86,7 +86,7 @@ OLIPHAUNT_FETCH_TEST_FINAL="$fallback_destination" \ PATH="$fake_path" \ oliphaunt_fetch_postgresql_source_archive "$fallback_destination" 18.4 "$fixture_sha" "$primary_url" assert_verified_file "$fallback_destination" "$fixture_sha" -[[ "$(wc -l < "$fallback_log" | tr -d ' ')" == 2 ]] || fail "transport did not try exactly the primary and fallback URLs" +[[ "$(wc -l < "$fallback_log" | tr -d ' ')" == 6 ]] || fail "transport did not try five primary attempts then the fallback URL" grep -F -- "$primary_url" "$fallback_log" >/dev/null || fail "primary URL was not attempted" grep -F -- "$fallback_url" "$fallback_log" >/dev/null || fail "fallback URL was not attempted" assert_transport_flags "$fallback_log" @@ -115,7 +115,7 @@ if OLIPHAUNT_FETCH_TEST_LOG="$failed_log" \ fail "all-failed transport unexpectedly succeeded" fi [[ ! -e "$failed_destination" ]] || fail "failed transport promoted an unverified final destination" -[[ "$(wc -l < "$failed_log" | tr -d ' ')" == 2 ]] || fail "failed transport exceeded or skipped the two bounded URL attempts" +[[ "$(wc -l < "$failed_log" | tr -d ' ')" == 10 ]] || fail "failed transport exceeded or skipped the bounded URL attempts" assert_no_partials "$work_root/failed" interrupted_destination="$work_root/interrupted/postgresql-18.4.tar.bz2" diff --git a/src/third-party/tools/fetch-sources.sh b/src/third-party/tools/fetch-sources.sh index 1c59728cd..3e1ac4184 100644 --- a/src/third-party/tools/fetch-sources.sh +++ b/src/third-party/tools/fetch-sources.sh @@ -4,6 +4,7 @@ source_tools=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) source_core="$source_tools/source-fetch-core.mts" # shellcheck source=tools/dev/curl-platform-flags.sh source "$source_tools/../../../tools/dev/curl-platform-flags.sh" +source "$source_tools/../../../tools/dev/acquisition.sh" source_git() { local seconds=$1 directory=$2 status=0 @@ -11,7 +12,7 @@ source_git() { # Bound both streams, including diagnostics, without holding them in memory. # pipefail rejects a producer killed by SIGPIPE; the byte count also catches # a producer that completed its last write before head closed the pipe. - "$source_timeout" --kill-after=5 "$seconds" git -C "$directory" \ + oliphaunt_acquisition_run "$seconds" git -C "$directory" \ -c core.fsmonitor=false -c submodule.recurse=false \ -c core.autocrlf=false -c core.eol=lf "$@" \ 2> >(head -c 16777217 > "$source_stage/git-error") | @@ -31,7 +32,7 @@ source_snapshot() { if [[ -d "$checkout" && ! -L "$checkout" && -d "$checkout/.git" && ! -L "$checkout/.git" ]]; then source_git 60 "$checkout" rev-parse --show-toplevel > "$snapshot/worktree" source_git 60 "$checkout" rev-parse --absolute-git-dir > "$snapshot/git-directory" - bun "$source_core" git-identity "$pin" "$checkout" "$snapshot" + oliphaunt_acquisition_run 900 bun "$source_core" git-identity "$pin" "$checkout" "$snapshot" source_git 60 "$checkout" status --porcelain=v1 --untracked-files=all > "$snapshot/status" # Missing pin fields make a clean checkout stale; repository errors above # remain fatal. The data validator compares the complete snapshot. @@ -46,15 +47,16 @@ source_snapshot() { fetch_source() ( set -euo pipefail local pin=$1 checkout_root=$2 archive_root=$3 mode=$4 - local name kind url mirror branch commit archive_name canonical checkout readiness fetched - local source_lock='' lock_deadline + local name kind url mirror branch commit archive_name canonical checkout readiness fetched status + local source_lock='' + oliphaunt_acquisition_start "source $(basename "$pin")" 900 mkdir -p "$checkout_root" "$archive_root" source_stage=$(mktemp -d "$checkout_root/.source-stage-XXXXXX") trap 'rm -rf "$source_stage"; if [[ -n "$source_lock" ]]; then rmdir "$source_lock"; fi' EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM - bun "$source_core" fields "$pin" > "$source_stage/fields" + oliphaunt_acquisition_run 900 bun "$source_core" fields "$pin" > "$source_stage/fields" { IFS= read -r -d '' name; IFS= read -r -d '' kind; IFS= read -r -d '' url IFS= read -r -d '' mirror; IFS= read -r -d '' branch; IFS= read -r -d '' commit IFS= read -r -d '' archive_name; IFS= read -r -d '' canonical @@ -62,11 +64,8 @@ fetch_source() ( checkout="$checkout_root/$name" # Scopes overlap (notably ICU data). Serialize inspection through promotion, # so a waiter reuses the complete checkout instead of replacing it concurrently. - lock_deadline=$((SECONDS + 3600)) until mkdir "$checkout.lock" 2>/dev/null; do - if (( SECONDS >= lock_deadline )); then - echo "timed out waiting for source checkout lock: $checkout.lock" >&2; exit 1 - fi + oliphaunt_acquisition_remaining 900 >/dev/null || exit $? sleep 0.1 done source_lock="$checkout.lock" @@ -77,11 +76,8 @@ fetch_source() ( : > "$source_stage/empty.gitconfig" export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL="$source_stage/empty.gitconfig" export GIT_TERMINAL_PROMPT=0 GCM_INTERACTIVE=Never - source_timeout=$(command -v timeout || command -v gtimeout) || { - echo 'source fetching requires GNU timeout (brew install coreutils on macOS)' >&2; exit 1; - } source_snapshot "$checkout" "$source_stage/durable" - readiness=$(bun "$source_core" inspect "$pin" "$checkout" "$source_stage/durable") + readiness=$(oliphaunt_acquisition_run 900 bun "$source_core" inspect "$pin" "$checkout" "$source_stage/durable") if [[ "$readiness" == ready ]]; then exit 0; fi if [[ "$mode" == verify ]]; then echo "source checkout $checkout is missing or stale" >&2; exit 1; fi @@ -98,10 +94,14 @@ fetch_source() ( if source_git 300 "$candidate" \ -c protocol.allow=never -c protocol.https.allow=always -c credential.helper= \ -c http.followRedirects=false -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=120 \ - fetch --no-tags --depth=1 "$transport" "$commit"; then success=true; break; fi + fetch --no-tags --depth=1 "$transport" "$commit"; then success=true; break + else + status=$? + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac + fi echo "fetch $name from $transport failed on attempt $attempt/5" >&2 if (( attempt < 5 && attempt % ${#transports[@]} == 0 )); then - sleep "$((attempt * 5 / ${#transports[@]}))" + oliphaunt_acquisition_sleep "$((attempt * 5 / ${#transports[@]}))" || exit $? fi done "$success" || exit 1 @@ -111,10 +111,10 @@ fetch_source() ( fi source_git 60 "$candidate" checkout --quiet -B "$branch" "$commit" source_snapshot "$candidate" "$source_stage/candidate" - bun "$source_core" git-candidate "$pin" "$candidate" "$source_stage/candidate" + oliphaunt_acquisition_run 900 bun "$source_core" git-candidate "$pin" "$candidate" "$source_stage/candidate" else local archive="$archive_root/$archive_name" download="$source_stage/$archive_name" - if [[ "$(bun "$source_core" archive-valid "$pin" "$archive")" != valid ]]; then + if [[ "$(oliphaunt_acquisition_run 900 bun "$source_core" archive-valid "$pin" "$archive")" != valid ]]; then local urls=() endpoint success=false if [[ -n "$canonical" ]]; then urls+=("$canonical"); fi urls+=("$url") @@ -122,26 +122,31 @@ fetch_source() ( local tls_flag tls_flag=$(oliphaunt_curl_platform_tls_flag) for endpoint in "${urls[@]}"; do - if "$source_timeout" --kill-after=5 620 curl --disable --fail --location --silent --show-error \ - --retry 2 --retry-all-errors --retry-connrefused --retry-delay 5 --retry-max-time 600 \ - --connect-timeout 20 --max-time 600 --speed-limit 1024 --speed-time 120 \ + if oliphaunt_acquisition_curl 300 3 5 curl --fail --location --silent --show-error \ + --connect-timeout 20 --speed-limit 1024 --speed-time 120 \ --max-filesize 1073741824 --max-redirs 5 --proto-default https \ --proto '=https' --proto-redir '=https' --tlsv1.2 ${tls_flag:+"$tls_flag"} \ - --remove-on-error --url "$endpoint" --output "$download"; then success=true; break; fi + --remove-on-error --url "$endpoint" --output "$download"; then success=true; break + else + status=$? + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac + fi echo "download $name from $endpoint failed" >&2 rm -f "$download" done "$success" || exit 1 # An invalid candidate never replaces even a corrupt existing cache. - [[ "$(bun "$source_core" archive-valid "$pin" "$download")" == valid ]] || exit 1 + [[ "$(oliphaunt_acquisition_run 900 bun "$source_core" archive-valid "$pin" "$download")" == valid ]] || exit 1 + oliphaunt_acquisition_remaining 900 >/dev/null || exit $? bun "$source_core" promote "$download" "$archive" fi - bun "$source_core" unpack "$pin" "$archive" "$candidate" + oliphaunt_acquisition_run 900 bun "$source_core" unpack "$pin" "$archive" "$candidate" fi # Reinspect immediately before replacing an existing checkout, including a # source that changed kind. The promotion helper restores a prior tree on error. source_snapshot "$checkout" "$source_stage/durable" - bun "$source_core" inspect "$pin" "$checkout" "$source_stage/durable" > /dev/null + oliphaunt_acquisition_run 900 bun "$source_core" inspect "$pin" "$checkout" "$source_stage/durable" > /dev/null + oliphaunt_acquisition_remaining 900 >/dev/null || exit $? bun "$source_core" promote "$candidate" "$checkout" ) @@ -153,6 +158,7 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then bun "$source_tools/fetch-sources.mts" plan "$source_work" "$@" mode=$(cat "$source_work/mode") if [[ "$mode" != skip ]]; then + oliphaunt_acquisition_start 'source scope' 1800 while IFS= read -r -d '' pin; do fetch_source "$pin" "$PWD/target/oliphaunt-sources/checkouts" "$PWD/target/oliphaunt-sources/archives" "$mode" # Bash 3.2 does not propagate a failed subshell function through this loop. diff --git a/src/third-party/tools/moon.yml b/src/third-party/tools/moon.yml index 8c4c818cd..003065c9f 100644 --- a/src/third-party/tools/moon.yml +++ b/src/third-party/tools/moon.yml @@ -23,6 +23,7 @@ tasks: - "**/*" - "!**/*.test.*" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - "@group(release-archive-contract)" options: cache: false @@ -34,6 +35,7 @@ tasks: script: "set -e\nbash src/third-party/tools/fetch-sources.sh production-all --validate-only\nbash src/third-party/tools/source-fetch-core.test.sh\nbash tools/dev/bun.sh test ./src/third-party/tools/source-fetch-scopes.test.mts\nbash src/third-party/postgres/fetch-source.test.sh\n" inputs: - "/tools/dev/bun.sh" + - /tools/dev/acquisition.sh - /src/third-party/postgres/**/* - "/src/third-party/{icu,openssl}/source.toml" - /src/native/runtime/sources/*.toml @@ -64,6 +66,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - /src/extensions/tools/extension-upstream-licenses.mts - "@group(release-archive-contract)" options: @@ -83,6 +86,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - "@group(release-archive-contract)" options: cache: false @@ -103,6 +107,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - "@group(release-archive-contract)" options: cache: false @@ -121,6 +126,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh options: cache: false internal: true @@ -138,6 +144,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - /src/extensions/tools/extension-upstream-licenses.mts - "@group(release-archive-contract)" options: diff --git a/src/third-party/tools/source-fetch-core.mts b/src/third-party/tools/source-fetch-core.mts index f0323f0dc..e1f7e894d 100644 --- a/src/third-party/tools/source-fetch-core.mts +++ b/src/third-party/tools/source-fetch-core.mts @@ -297,10 +297,11 @@ export function validateSource(source) { if ( !parsedUrl.pathname.endsWith('.tar.gz') && !parsedUrl.pathname.endsWith('.tgz') && + !parsedUrl.pathname.endsWith('.crate') && !parsedUrl.pathname.endsWith('.zip') ) { throw new Error( - `archive source '${source.name}' URL must identify a .tar.gz, .tgz, or .zip file`, + `archive source '${source.name}' URL must identify a .tar.gz, .tgz, .crate, or .zip file`, ); } if (source.stripPrefix === '.' && !parsedUrl.pathname.endsWith('.zip')) { diff --git a/src/third-party/tools/source-fetch-core.test.mts b/src/third-party/tools/source-fetch-core.test.mts index d5682199b..2fd90be12 100644 --- a/src/third-party/tools/source-fetch-core.test.mts +++ b/src/third-party/tools/source-fetch-core.test.mts @@ -271,6 +271,12 @@ test('source pins reject unsafe URLs, branches, names, and unpinned archives', ( stripPrefix: 'pkg', }; assert.doesNotThrow(() => validateSource(archive)); + assert.doesNotThrow(() => + validateSource({ + ...archive, + url: 'https://static.crates.io/crates/wasmer/wasmer-6.1.0.crate', + }), + ); assert.throws(() => validateSource({ ...archive, mirrorUrl: archive.url })); for (const url of [ 'https://example.invalid/libiconv/libiconv-1.19.tar.gz', diff --git a/src/third-party/tools/source-fetch-core.test.sh b/src/third-party/tools/source-fetch-core.test.sh index b8d378e95..081dd1fa2 100644 --- a/src/third-party/tools/source-fetch-core.test.sh +++ b/src/third-party/tools/source-fetch-core.test.sh @@ -7,6 +7,7 @@ trap 'rm -rf "$scratch"' EXIT test_data="$tools/source-fetch-core.test.mts" archive_tool="$tools/source-archive.mts" export FETCH_TEST_GIT="$(command -v git)" +export FETCH_TEST_DATE="$(command -v date)" export FETCH_TEST_SLEEP="$(command -v sleep)" base_path="$PATH" for scope in icu native-runtime wasix-runtime wasix-postmaster-runtime production-all; do @@ -22,7 +23,7 @@ done bash "$root_dir/tools/dev/bun.sh" test "$test_data" mkdir "$scratch/fixtures" "$scratch/bin" bun "$test_data" prepare "$scratch/fixtures" -for name in git curl sleep; do +for name in git curl sleep date; do cp "$tools/source-fetch-transport.test.sh" "$scratch/bin/$name" chmod +x "$scratch/bin/$name" done @@ -110,7 +111,7 @@ fail_command 'archive sha256: expected' fetch "$root" "$fixtures/valid.json" : > "$root/requests" export FETCH_TEST_ARCHIVE="$fixtures/valid.tar.gz" RUNNER_OS=Windows fetch "$root" "$fixtures/valid.json" -printf '%s\n' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' > "$scratch/expected" +printf '%s\n' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' > "$scratch/expected" cmp "$scratch/expected" "$root/requests" cmp "$cache" "$fixtures/valid.tar.gz" cmp "$scratch/trusted" "$checkout/file.txt" @@ -138,6 +139,10 @@ FETCH_TEST_FAULT=gnu FETCH_TEST_ARCHIVE="$fixtures/updated.tar.gz" \ : > "$root/requests" FETCH_TEST_FAULT=gnu FETCH_TEST_ARCHIVE="$fixtures/valid.tar.gz" fetch "$root" "$root/pin.json" printf '%s\n' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' \ + 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' \ + 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' \ + 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' \ + 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' \ 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' \ 'https://mirror.example.invalid/libiconv.tar.gz' > "$scratch/expected" cmp "$scratch/expected" "$root/requests" @@ -255,3 +260,35 @@ ln -s "$root/upstream/.git" "$checkout/.git" bun "$test_data" git-pin "$root" "$commit" fail_command 'unsupported non-directory \.git metadata' fetch "$root" "$root/pin.json" echo 'Source fetch: verified archives, fallback/retries, exact Git pins, LF repair, symlink containment and local-edit preservation passed' + +# Retries and mirrors share the pin's deadline; expiry preserves existing data +# and releases the checkout lock, including when a transport returns failure. +root="$scratch/archive-deadline" +mkdir -p "$root/archives" +printf '1000000\n' > "$root/clock" +cache="$root/archives/libiconv-$sha.tar.gz" +printf 'prior cache' > "$cache" +FETCH_TEST_EXPIRE=1 FETCH_TEST_CLOCK="$root/clock" fail_command 'deadline' fetch "$root" "$fixtures/valid.json" +[[ "$(wc -l < "$root/requests")" -eq 1 ]] +[[ "$(cat "$cache")" == 'prior cache' ]] +[[ -z "$(ls -A "$root/checkouts")" ]] +root="$scratch/git-deadline" +init_repo "$root/upstream" 'new bytes' upstream +commit="$(git -C "$root/upstream" rev-parse HEAD)" +bun "$test_data" git-pin "$root" "$commit" +init_repo "$root/checkouts/source" prior +prior="$(git -C "$root/checkouts/source" rev-parse HEAD)" +printf '1000000\n' > "$root/clock" +FETCH_TEST_EXPIRE=1 FETCH_TEST_CLOCK="$root/clock" fail_command 'deadline' fetch "$root" "$root/pin.json" +[[ "$(wc -l < "$root/requests")" -eq 1 ]] +[[ "$(git -C "$root/checkouts/source" rev-parse HEAD)" == "$prior" ]] +[[ "$(ls -A "$root/checkouts")" == source ]] +echo 'Source acquisition expiry preserves prior data, stops retries and releases locks' + +# A waiting caller must neither replenish its budget nor remove another owner's lock. +root="$scratch/lock-deadline" +mkdir -p "$root/checkouts/libiconv.lock" +printf '1000000\n' > "$root/clock" +FETCH_TEST_EXPIRE=1 FETCH_TEST_CLOCK="$root/clock" fail_command 'deadline' fetch "$root" "$fixtures/valid.json" +[[ ! -s "$root/requests" ]] +[[ "$(ls -A "$root/checkouts")" == libiconv.lock ]] diff --git a/src/third-party/tools/source-fetch-transport.test.sh b/src/third-party/tools/source-fetch-transport.test.sh index 8355d8bbc..3458f86de 100644 --- a/src/third-party/tools/source-fetch-transport.test.sh +++ b/src/third-party/tools/source-fetch-transport.test.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash set -euo pipefail case "${0##*/}" in + date) + if [[ -n ${FETCH_TEST_CLOCK:-} ]]; then cat "$FETCH_TEST_CLOCK"; else exec "$FETCH_TEST_DATE" "$@"; fi + ;; sleep) printf '%s\n' "$1" >> "$FETCH_TEST_ROOT/sleeps" if [[ "$1" == 0.1 ]]; then touch "$FETCH_TEST_ROOT/lock-waiting" + if [[ ${FETCH_TEST_EXPIRE:-} == 1 ]]; then printf "9999999\n" > "$FETCH_TEST_CLOCK"; exit 0; fi exec "$FETCH_TEST_SLEEP" "$1" fi ;; @@ -14,6 +18,7 @@ case "${0##*/}" in count=${#args[@]} url=${args[$((count-2))]} printf '%s\n' "$url" >> "$FETCH_TEST_ROOT/requests" + if [[ ${FETCH_TEST_EXPIRE:-} == 1 ]]; then printf "9999999\n" > "$FETCH_TEST_CLOCK"; exit 28; fi if [[ ${FETCH_TEST_FAULT:-} == all || (${FETCH_TEST_FAULT:-} == primary && "$url" != https://mirror.example.invalid/source.git) ]]; then echo "transport fault: $url" >&2; exit 1 fi @@ -27,7 +32,7 @@ case "${0##*/}" in curl) [[ "$1" == --disable ]] case " $* " in *' --insecure '*|*' -k '*) exit 90 ;; esac - for required in '--proto =https' '--proto-redir =https' '--max-filesize 1073741824' '--max-time 600' '--tlsv1.2' '--retry 2'; do + for required in '--proto =https' '--proto-redir =https' '--max-filesize 1073741824' '--retry 0' '--tlsv1.2'; do [[ " $* " == *" $required "* ]] || exit 91 done if [[ ${RUNNER_OS:-} == Windows ]]; then [[ " $* " == *' --ssl-revoke-best-effort '* ]]; fi @@ -37,6 +42,7 @@ case "${0##*/}" in shift done printf '%s\n' "$url" >> "$FETCH_TEST_ROOT/requests" + if [[ ${FETCH_TEST_EXPIRE:-} == 1 ]]; then printf "9999999\n" > "$FETCH_TEST_CLOCK"; exit 28; fi if [[ ${FETCH_TEST_BARRIER:-} == 1 ]]; then touch "$FETCH_TEST_ROOT/downloading" while [[ ! -e "$FETCH_TEST_ROOT/release-download" ]]; do "$FETCH_TEST_SLEEP" 0.1; done diff --git a/src/wasix/browser-host/build-provenance.mts b/src/wasix/browser-host/build-provenance.mts index dd34ae0d9..d3621baa0 100644 --- a/src/wasix/browser-host/build-provenance.mts +++ b/src/wasix/browser-host/build-provenance.mts @@ -27,6 +27,11 @@ export async function loadHostBuildContract() { ...patchSeries.map((patch) => `src/wasix/browser-host/patches/${patch}`), buildScriptPath, provenanceScriptPath, + 'tools/dev/curl-platform-flags.sh', + 'tools/dev/acquisition.sh', + 'src/third-party/tools/fetch-sources.sh', + 'src/third-party/tools/source-fetch-core.mts', + 'src/third-party/tools/source-archive.mts', ]); const digests = []; for (const input of inputs) { diff --git a/src/wasix/browser-host/build-sdk.sh b/src/wasix/browser-host/build-sdk.sh index 0db80d0c0..36c9b9620 100755 --- a/src/wasix/browser-host/build-sdk.sh +++ b/src/wasix/browser-host/build-sdk.sh @@ -27,17 +27,11 @@ toml_value() { ' "$source_manifest" } -wasmer_js_url="$(toml_value wasmer-js url)" wasmer_js_version="$(toml_value wasmer-js version)" -wasmer_js_commit="$(toml_value wasmer-js commit)" -wasmer_wasix_url="$(toml_value wasmer-wasix url)" wasmer_wasix_version="$(toml_value wasmer-wasix version)" -wasmer_wasix_sha256="$(toml_value wasmer-wasix sha256)" -wasmer_url="$(toml_value wasmer url)" wasmer_version="$(toml_value wasmer version)" -wasmer_sha256="$(toml_value wasmer sha256)" -for value in "$wasmer_js_url" "$wasmer_js_version" "$wasmer_js_commit" "$wasmer_wasix_url" "$wasmer_wasix_version" "$wasmer_wasix_sha256" "$wasmer_url" "$wasmer_version" "$wasmer_sha256"; do +for value in "$wasmer_js_version" "$wasmer_wasix_version" "$wasmer_version"; do if [[ -z "$value" ]]; then echo "wasix-ts host build: malformed $source_manifest" >&2 exit 1 @@ -48,17 +42,13 @@ if ! command -v bun >/dev/null 2>&1; then echo "wasix-ts host build: required command not found: bun" >&2 exit 1 fi -mapfile -t patch_series < <(bun "$provenance_script" --patch-series) +patch_series="$(bun "$provenance_script" --patch-series)" input_hash="$(bun "$provenance_script" --inputs-sha256)" patch_command="patch" -sha256sum_command="sha256sum" if command -v gpatch >/dev/null 2>&1; then patch_command="gpatch" fi -if command -v gsha256sum >/dev/null 2>&1; then - sha256sum_command="gsha256sum" -fi if [[ -f "$target_dir/.oliphaunt-input-sha256" ]] \ && [[ "$(<"$target_dir/.oliphaunt-input-sha256")" == "$input_hash" ]] \ @@ -71,7 +61,7 @@ fi mkdir -p "$target_parent" -for command_name in awk bun curl git node npm "$patch_command" "$sha256sum_command" tar wasm-pack; do +for command_name in awk bun curl git node npm "$patch_command" wasm-pack; do if ! command -v "$command_name" >/dev/null 2>&1; then echo "wasix-ts host build: required command not found: $command_name" >&2 exit 1 @@ -89,38 +79,36 @@ cleanup() { trap cleanup EXIT wasmer_js_dir="$build_root/wasmer-js" -wasmer_wasix_archive="$build_root/wasmer-wasix.crate" wasmer_wasix_dir="$build_root/wasmer-wasix-$wasmer_wasix_version" -wasmer_archive="$build_root/wasmer.crate" wasmer_dir="$build_root/wasmer-$wasmer_version" -git init --quiet "$wasmer_js_dir" -git -C "$wasmer_js_dir" remote add origin "$wasmer_js_url" -git -C "$wasmer_js_dir" fetch --quiet --depth 1 origin "$wasmer_js_commit" -git -C "$wasmer_js_dir" checkout --quiet --detach FETCH_HEAD -if [[ "$(git -C "$wasmer_js_dir" rev-parse HEAD)" != "$wasmer_js_commit" ]]; then - echo "wasix-ts host build: Wasmer JS checkout did not resolve the pinned commit" >&2 - exit 1 -fi +# Use the same bounded transports, exact pins and safe extraction as the other +# producers. Only temporary source trees are patched; verified archives persist. +source "$repo_root/src/third-party/tools/fetch-sources.sh" +bun - "$source_manifest" "$build_root" <<'JS' +import {readFileSync, writeFileSync} from 'node:fs'; +const pins = Bun.TOML.parse(readFileSync(process.argv[2], 'utf8')); +for (const name of ['wasmer-js', 'wasmer-wasix', 'wasmer']) { + const pin = pins[name]; + const source = name === 'wasmer-js' + ? {name, kind:'git', url:pin.url, branch:'oliphaunt-pinned', commit:pin.commit} + : {name:`${name}-${pin.version}`, kind:'archive', url:pin.url, branch:pin.version, + commit:pin.sha256, sha256:pin.sha256, stripPrefix:`${name}-${pin.version}`}; + writeFileSync(`${process.argv[3]}/${name}.json`, JSON.stringify(source)); +} +JS +oliphaunt_acquisition_start 'browser host sources' 1800 +for source_name in wasmer-js wasmer-wasix wasmer; do + fetch_source "$build_root/$source_name.json" "$build_root" "$target_parent/archives" fetch +done actual_wasmer_js_version="$(bun "$repo_root/tools/dev/node-info.mts" package-version "$wasmer_js_dir/package.json")" if [[ "$actual_wasmer_js_version" != "$wasmer_js_version" ]]; then echo "wasix-ts host build: pinned Wasmer JS version is $actual_wasmer_js_version, expected $wasmer_js_version" >&2 exit 1 fi -curl --fail --location --silent --show-error \ - --user-agent "oliphaunt-wasix-ts-source-build/0.0.0" \ - "$wasmer_wasix_url" --output "$wasmer_wasix_archive" -echo "$wasmer_wasix_sha256 $wasmer_wasix_archive" | "$sha256sum_command" --check --status -tar -xzf "$wasmer_wasix_archive" -C "$build_root" - -curl --fail --location --silent --show-error \ - --user-agent "oliphaunt-wasix-ts-source-build/0.0.0" \ - "$wasmer_url" --output "$wasmer_archive" -echo "$wasmer_sha256 $wasmer_archive" | "$sha256sum_command" --check --status -tar -xzf "$wasmer_archive" -C "$build_root" - -for patch_name in "${patch_series[@]}"; do +while IFS= read -r patch_name; do + [[ -n "$patch_name" ]] || continue patch_file="$host_dir/patches/$patch_name" case "$patch_name" in ????-wasmer-js-*.patch) @@ -138,7 +126,7 @@ for patch_name in "${patch_series[@]}"; do ;; esac "$patch_command" --batch --forward -d "$patch_dir" -p1 < "$patch_file" -done +done <<< "$patch_series" # The pinned source commit's npm lock predates its package metadata. Patch only # the missing root metadata and dependencies, then install the integrity-pinned diff --git a/src/wasix/browser-host/moon.yml b/src/wasix/browser-host/moon.yml index 619315ff2..95d5ed198 100644 --- a/src/wasix/browser-host/moon.yml +++ b/src/wasix/browser-host/moon.yml @@ -14,6 +14,9 @@ tasks: inputs: - "**/*" - "/tools/dev/node-info.mts" + - "/tools/dev/curl-platform-flags.sh" + - "/tools/dev/acquisition.sh" + - "/src/third-party/tools/{fetch-sources.sh,source-fetch-core.mts,source-archive.mts}" - "@group(cargo-workspace)" outputs: - "/target/oliphaunt-wasix-ts/host/wasmer-sdk/**/*" diff --git a/src/wasix/postgres-tools/moon.yml b/src/wasix/postgres-tools/moon.yml index 8299ee666..2d906a6e5 100644 --- a/src/wasix/postgres-tools/moon.yml +++ b/src/wasix/postgres-tools/moon.yml @@ -66,13 +66,13 @@ tasks: test-consumer: tags: ["integration", "consumer", "ci-wasix-ts-sdk-package"] command: "bash tools/ci/with-projects.sh --exec bash src/wasix/postgres-tools/ts/tests/consumer.sh" - deps: ["oliphaunt-wasix-tools-ts:package", "oliphaunt-wasix-ts:package", "oliphaunt-wasix-napi:build-release-assets", "postgres-tools-wasix:package-portable", "postgres-tools-wasix:package-aot"] + deps: ["oliphaunt-query-ts:build", "oliphaunt-wasix-tools-ts:package", {target: "oliphaunt-wasix-ts:package", cacheStrategy: ignored}, "oliphaunt-wasix-ts:build", "wasix-browser-host:build", "oliphaunt-wasix-napi:build-release-assets", "postgres-tools-wasix:package-portable", "postgres-tools-wasix:package-aot"] inputs: ["ts/tests/**/*", "/src/wasix/sdks/ts/tools/integration/packed-node-fixture.mts", "/src/wasix/sdks/ts/tools/pgwire-client.mts", "/src/test-fixtures/postgres/**/*", "tools/*.mts"] options: {cache: false, runFromWorkspaceRoot: true} test-browser: tags: ["integration", "browser", "ci-wasix-ts-sdk-package"] command: "bash tools/ci/with-projects.sh --exec bash src/wasix/sdks/ts/tools/integration/smoke-browser.sh --package-only --tools" - deps: ["oliphaunt-wasix-tools-ts:package", "oliphaunt-wasix-ts:package", "postgres-tools-wasix:package-portable"] + deps: ["oliphaunt-query-ts:build", "oliphaunt-wasix-tools-ts:package", {target: "oliphaunt-wasix-ts:package", cacheStrategy: ignored}, "oliphaunt-wasix-ts:build", "wasix-browser-host:build", "postgres-tools-wasix:package-portable"] inputs: ["ts/tests/**/*", "/src/wasix/sdks/ts/tools/integration/{packed-node-fixture.mts,smoke-browser.*}", "/src/test-fixtures/postgres/**/*", "tools/*.mts"] options: {cache: false, runFromWorkspaceRoot: true} format-check: diff --git a/src/wasix/postgres-tools/ts/tests/smoke-host.sh b/src/wasix/postgres-tools/ts/tests/smoke-host.sh index 147ee146c..55515d68b 100644 --- a/src/wasix/postgres-tools/ts/tests/smoke-host.sh +++ b/src/wasix/postgres-tools/ts/tests/smoke-host.sh @@ -14,7 +14,6 @@ esac deadline="$(command -v gtimeout || command -v timeout)" scratch="$(mktemp -d)" trap 'rm -rf "$scratch"' EXIT -bun run --cwd "$root/src/query/ts" build bun pm --cwd "$root/src/query/ts" pack --filename "$scratch/query.tgz" --quiet bun "$root/src/wasix/sdks/ts/tools/integration/packed-node-fixture.mts" "$scratch" --pgtap --tools cd "$scratch/consumer" diff --git a/src/wasix/postmaster/bin/package-release-assets.sh b/src/wasix/postmaster/bin/package-release-assets.sh index de2b57484..97b418906 100755 --- a/src/wasix/postmaster/bin/package-release-assets.sh +++ b/src/wasix/postmaster/bin/package-release-assets.sh @@ -89,7 +89,7 @@ cp -p "$project_root/bin/run-release-carrier.sh" \ chmod 0555 "$package_root/bin/oliphaunt-wasix-postmaster" cp -p "$repo_root/LICENSE" "$package_root/LICENSE" cp -p "$repo_root/THIRD_PARTY_NOTICES.md" "$package_root/THIRD_PARTY_NOTICES.md" -cp -p "$project_root/README.md" "$package_root/README.md" +cp -p "$repo_root/target/oliphaunt-wasix-postmaster/package-docs/README.md" "$package_root/README.md" bun "$repo_root/tools/packaging/archive-directory.mts" \ --keep-parent "$package_root" "$asset_dir/$asset_name" diff --git a/src/wasix/postmaster/bin/prepare-baseline.sh b/src/wasix/postmaster/bin/prepare-baseline.sh index 286db4319..3234df788 100755 --- a/src/wasix/postmaster/bin/prepare-baseline.sh +++ b/src/wasix/postmaster/bin/prepare-baseline.sh @@ -3,6 +3,7 @@ set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib/common.sh" +source "$REPO_ROOT/tools/dev/acquisition.sh" source "$REPO_ROOT/src/third-party/postgres/fetch-source.sh" print_path=0 diff --git a/src/wasix/postmaster/lib/common.sh b/src/wasix/postmaster/lib/common.sh index f8479fbc3..0d988051d 100644 --- a/src/wasix/postmaster/lib/common.sh +++ b/src/wasix/postmaster/lib/common.sh @@ -873,11 +873,14 @@ fresh_require_canonical_directory() { } fresh_wasix_builder_recipe_sha256() { + local identity_path local file_sha256 local identity_mode local path local recipe_paths=( + "$REPO_ROOT/tools/dev/acquisition.sh" "$WASIX_TOOLCHAIN_ROOT/docker/Dockerfile" + "$WASIX_TOOLCHAIN_ROOT/docker/Dockerfile.dockerignore" "$WASIX_TOOLCHAIN_ROOT/docker/isrg-root-x1.pem" "$WASIX_TOOLCHAIN_ROOT/docker/install-pinned-apt-packages.sh" "$WASIX_TOOLCHAIN_ROOT/docker/install-pinned-wasixcc.sh" @@ -903,7 +906,11 @@ fresh_wasix_builder_recipe_sha256() { else identity_mode=data fi - printf '%s\0%s\0%s\0' "${path#"$WASIX_TOOLCHAIN_ROOT"/}" "$file_sha256" "$identity_mode" + case "$path" in + "$WASIX_TOOLCHAIN_ROOT"/*) identity_path="src/wasix/runtime/assets/build/${path#"$WASIX_TOOLCHAIN_ROOT"/}" ;; + *) identity_path="${path#"$REPO_ROOT"/}" ;; + esac + printf '%s\0%s\0%s\0' "$identity_path" "$file_sha256" "$identity_mode" done } | fresh_sha256_stream } @@ -1705,11 +1712,15 @@ fresh_ensure_docker_image() { if [ "$actual_recipe" = "$expected_recipe" ]; then return fi + [ "$WASIX_TOOLCHAIN_ROOT" = "$REPO_ROOT/src/wasix/runtime/assets/build" ] || { + echo 'WASIX builder context requires WASIX_TOOLCHAIN_ROOT under REPO_ROOT; set both for another checkout' >&2 + return 2 + } "$docker_bin" build \ --label "$label=$expected_recipe" \ -f "$context/Dockerfile" \ -t "$image" \ - "$context" || return + "$REPO_ROOT" || return actual_recipe="$("$docker_bin" image inspect \ --format "{{ index .Config.Labels \"$label\" }}" "$image" 2>/dev/null || true)" [ "$actual_recipe" = "$expected_recipe" ] || { diff --git a/src/wasix/postmaster/lib/common.test.sh b/src/wasix/postmaster/lib/common.test.sh index 9ad944065..2a71aa5ae 100644 --- a/src/wasix/postmaster/lib/common.test.sh +++ b/src/wasix/postmaster/lib/common.test.sh @@ -174,15 +174,14 @@ portable_file_mode() { } builder_recipe_inputs=( docker/Dockerfile + docker/Dockerfile.dockerignore docker/isrg-root-x1.pem docker/install-pinned-apt-packages.sh docker/install-pinned-wasixcc.sh docker/pinned-wasixcc-assets.tsv ) -# The Dockerfile is itself the fifth recipe input. Every other recipe input -# must be a direct COPY source, and the Dockerfile must not consume an input -# that is absent from the recipe identity. +# Dockerfile, context policy and every COPY source enter the recipe identity. dockerfile_copy_sources="$( awk ' toupper($1) != "COPY" { next } @@ -205,11 +204,14 @@ dockerfile_copy_sources="$( exit 1 } expected_builder_context_inputs="$( - printf '%s\n' "${builder_recipe_inputs[@]#docker/}" | LC_ALL=C sort + { + printf 'src/wasix/runtime/assets/build/%s\n' "${builder_recipe_inputs[@]}" + printf 'tools/dev/acquisition.sh\n' + } | LC_ALL=C sort )" actual_builder_context_inputs="$( { - printf 'Dockerfile\n' + printf 'src/wasix/runtime/assets/build/docker/%s\n' Dockerfile Dockerfile.dockerignore printf '%s\n' "$dockerfile_copy_sources" } | LC_ALL=C sort )" @@ -220,6 +222,15 @@ actual_builder_context_inputs="$( exit 1 } +original_repo_root="$REPO_ROOT" +builder_repo="$test_root/builder-repo" +mkdir -p "$builder_repo/tools/dev" +cp "$REPO_ROOT/tools/dev/acquisition.sh" "$builder_repo/tools/dev/acquisition.sh" +REPO_ROOT="$builder_repo" +[ "$(fresh_wasix_builder_recipe_sha256)" = "$live_builder_recipe" ] +printf '\n# acquisition drift probe\n' >> "$builder_repo/tools/dev/acquisition.sh" +[ "$(fresh_wasix_builder_recipe_sha256)" != "$live_builder_recipe" ] +REPO_ROOT="$original_repo_root" builder_fixture="$test_root/builder-fixture" mkdir -p "$builder_fixture" cp -a "$original_toolchain_root/." "$builder_fixture/" @@ -624,7 +635,7 @@ run_fake_docker_case() { if [ "$expected_build_count" -eq 1 ]; then expected_build="$(printf 'build\t--label\t%s=%s\t-f\t%s/Dockerfile\t-t\t%s\t%s' \ - "$label" "$live_builder_recipe" "$context" "$image" "$context")" + "$label" "$live_builder_recipe" "$context" "$image" "$REPO_ROOT")" actual_build="$(awk -F '\t' '$1 == "build"' "$log")" [ "$actual_build" = "$expected_build" ] || { printf 'fake Docker case %s used unexpected build arguments\n' "$name" >&2 @@ -634,6 +645,8 @@ run_fake_docker_case() { fi } +# An external recipe must not build with COPY inputs from a different tree. +WASIX_TOOLCHAIN_ROOT="$builder_fixture" run_fake_docker_case external-context error expected 2 0 run_fake_docker_case matching-label expected expected 0 0 run_fake_docker_case absent-image error expected 0 1 run_fake_docker_case missing-label empty expected 0 1 diff --git a/src/wasix/postmaster/moon.yml b/src/wasix/postmaster/moon.yml index 0377aa763..72e8bab3f 100644 --- a/src/wasix/postmaster/moon.yml +++ b/src/wasix/postmaster/moon.yml @@ -38,6 +38,11 @@ owners: "**/*": - "@oliphaunt/wasix" fileGroups: + production: + - "**/*" + - "!**/*.md" + - "!**/*.test.*" + - "!testdata/**/*" guest-sealing: - bin/apply-wasix-core-overlay.sh - "bin/seal-wasix-{core-exports,linear-memory}.sh" @@ -93,6 +98,7 @@ tasks: inputs: - "/tools/dev/bun.sh" - "**/*" + - "!**/*.md" - /tools/packaging/strict-json.mts options: cache: true @@ -145,7 +151,9 @@ tasks: - liboliphaunt-wasix-postmaster:prepare-runtime inputs: - "@group(cargo-workspace)" + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - "@group(runtime-source)" - /src/wasix/postmaster/lib/common.sh @@ -173,7 +181,9 @@ tasks: - liboliphaunt-wasix-postmaster:prepare-runtime inputs: - "@group(cargo-workspace)" + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - "@group(runtime-source)" - wasmer/tests.sh @@ -191,7 +201,9 @@ tasks: - liboliphaunt-wasix-postmaster:prepare-postgres - liboliphaunt-wasix-postmaster:runtime-build inputs: + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - /src/wasix/postmaster/bin/build-wasix-core.sh - /src/wasix/postmaster/lib/common.sh @@ -211,7 +223,9 @@ tasks: inputs: - "@group(guest-sealing)" - /src/third-party/postgres/source.toml + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - /src/third-party/postgres/**/* - "!/src/third-party/postgres/**/*.test.*" @@ -237,7 +251,9 @@ tasks: deps: - liboliphaunt-wasix-postmaster:runtime-build inputs: + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - "@group(runtime-source)" - /src/wasix/postmaster/lib/common.sh @@ -276,6 +292,8 @@ tasks: deps: - liboliphaunt-wasix-postmaster:runtime-capabilities - liboliphaunt-wasix-postmaster:postgres-build + inputs: + - "@group(production)" options: cache: false internal: true @@ -290,6 +308,8 @@ tasks: command: bash src/wasix/postmaster/bin/stress-wasix-initdb.sh --iterations 20 deps: - liboliphaunt-wasix-postmaster:regression + inputs: + - "@group(production)" options: cache: false internal: true @@ -303,6 +323,8 @@ tasks: command: bash src/wasix/postmaster/bin/smoke-wasix-concurrent-connections.sh deps: - liboliphaunt-wasix-postmaster:initdb-smoke + inputs: + - "@group(production)" options: cache: false runFromWorkspaceRoot: true @@ -317,7 +339,9 @@ tasks: deps: - liboliphaunt-wasix-postmaster:smoke inputs: + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - /src/wasix/postmaster/bin/run-wasix-regress-subset.sh - /src/wasix/postmaster/bin/wasix-make.sh @@ -337,7 +361,7 @@ tasks: inputs: - "@group(legal-files)" - "@group(cargo-workspace)" - - "**/*" + - "@group(production)" - /src/wasix/postmaster/sources/*.toml - "/tools/packaging/*.{mjs,mts}" outputs: @@ -347,6 +371,17 @@ tasks: internal: true runFromWorkspaceRoot: true runInCI: true + package-docs: + tags: ["quality", "static"] + script: | + set -eu + mkdir -p target/oliphaunt-wasix-postmaster/package-docs + cp src/wasix/postmaster/README.md target/oliphaunt-wasix-postmaster/package-docs/README.md + inputs: ["README.md"] + outputs: ["/target/oliphaunt-wasix-postmaster/package-docs/README.md"] + options: + cache: true + runFromWorkspaceRoot: true release-assets: tags: - release @@ -356,9 +391,14 @@ tasks: command: bash src/wasix/postmaster/bin/package-release-assets.sh deps: - liboliphaunt-wasix-postmaster:carrier + # Documentation has its own cheap producer. A release always assembles it, + # but prose changes alone do not requalify unchanged runtime binaries. + - target: liboliphaunt-wasix-postmaster:package-docs + cacheStrategy: ignored inputs: - "@group(legal-files)" - - "**/*" + - "@group(production)" + - /target/oliphaunt-wasix-postmaster/package-docs/README.md - /src/wasix/postmaster/sources/*.toml - "/tools/packaging/*.{mjs,mts}" - /target/oliphaunt-wasix-postmaster/carriers/**/* @@ -378,6 +418,8 @@ tasks: deps: - liboliphaunt-wasix-postmaster:carrier - liboliphaunt-wasix-postmaster:initdb-stress + inputs: + - "@group(production)" options: cache: false internal: true @@ -394,7 +436,7 @@ tasks: - liboliphaunt-wasix-postmaster:carrier - liboliphaunt-wasix-postmaster:backend-wave-stress inputs: - - "**/*" + - "@group(production)" - /target/oliphaunt-wasix-postmaster/carriers/**/* options: cache: false @@ -410,7 +452,7 @@ tasks: - liboliphaunt-wasix-postmaster:postgres-build - liboliphaunt-wasix-postmaster:runtime-capabilities inputs: - - "**/*" + - "@group(production)" - /src/wasix/postmaster/sources/*.toml - "/tools/packaging/*.{mjs,mts}" - /target/oliphaunt-wasix-postmaster/install/wasix-core-release-o3.current diff --git a/src/wasix/postmaster/wasmer/tests.sh b/src/wasix/postmaster/wasmer/tests.sh index 97c9c7e91..1f363834e 100644 --- a/src/wasix/postmaster/wasmer/tests.sh +++ b/src/wasix/postmaster/wasmer/tests.sh @@ -25,13 +25,7 @@ run_tests \ remap_shared_file_fixed_accepts_a_partial_final_file_page \ remap_private_file_fixed_shares_clean_bytes_but_isolates_writes \ immutable_function_tables_are_shared_by_two_instances \ - shared_function_tables_outlive_artifact_owner_and_peer_instance -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$WASMER_ROOT/lib/vm/Cargo.toml" \ - -- \ - --exact \ + shared_function_tables_outlive_artifact_owner_and_peer_instance \ instance::allocator::tests::cached_offsets_produce_the_same_allocator_layout \ trap::traphandlers::tests::tls_stack_reuses_mapping_without_global_queue if [ "$(uname -s)-$(uname -m)" = Linux-x86_64 ]; then @@ -103,15 +97,7 @@ run_tests \ --lib \ --features wasmer/cranelift \ -- \ - --exact \ - state::tests::live_shared_mapping_registry_blocks_backing_file_shrink -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$WASMER_ROOT/lib/wasix/Cargo.toml" \ - --lib \ - --features wasmer/cranelift \ - -- \ + state::tests::live_shared_mapping_registry_blocks_backing_file_shrink \ utils::store::tests if [ "$(uname -s)" = Linux ]; then run_tests \ @@ -130,15 +116,7 @@ run_tests \ --features sys-minimal,wasmer/cranelift,ctrlc \ -- \ --test-threads=1 \ - os::task::task_join_handle::tests -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$WASMER_ROOT/lib/wasix/Cargo.toml" \ - --lib \ - --no-default-features \ - --features sys-minimal,wasmer/cranelift,ctrlc \ - -- \ + os::task::task_join_handle::tests \ runners::wasi:: run_tests \ --locked \ @@ -161,15 +139,6 @@ run_tests \ issues::llvm_rotates_and_atomic_fence_emit_expected_ir \ wast::spec::data_drop0::llvm::llvm \ wast::spec::memory_init::llvm::llvm -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$FRESH_ROOT/executor/Cargo.toml" \ - --lib \ - --no-default-features \ - --features "$FRESH_POSTMASTER_EXECUTOR_FEATURES" \ - -- \ - sealed::tests::runtime_policy_identity_ run_tests \ --locked \ --target-dir "$WASMER_TARGET_DIR" \ @@ -212,14 +181,7 @@ run_tests \ --manifest-path "$WASMER_ROOT/lib/api/Cargo.toml" \ --test module \ -- \ - serialized_artifact_inspector -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$WASMER_ROOT/lib/api/Cargo.toml" \ - --test module \ - -- \ - --exact \ + serialized_artifact_inspector \ detached_module_executes_from_strict_relocated_regular_file_code_memory \ detached_mmapped_module_executes_without_retaining_serializable_state run_tests \ diff --git a/src/wasix/runtime/assets/build/docker/Dockerfile b/src/wasix/runtime/assets/build/docker/Dockerfile index 067871df4..df1aadeda 100644 --- a/src/wasix/runtime/assets/build/docker/Dockerfile +++ b/src/wasix/runtime/assets/build/docker/Dockerfile @@ -12,8 +12,9 @@ ARG OLIPHAUNT_UBUNTU_SNAPSHOT_TLS_ROOT_SHA256=22b557a27055b33606b6559f37703928d3 LABEL dev.oliphaunt.ubuntu-apt-snapshot="${OLIPHAUNT_UBUNTU_APT_SNAPSHOT}" \ dev.oliphaunt.ubuntu-snapshot-tls-root-sha256="${OLIPHAUNT_UBUNTU_SNAPSHOT_TLS_ROOT_SHA256}" -COPY --chmod=0444 isrg-root-x1.pem /usr/local/share/oliphaunt/isrg-root-x1.pem -COPY --chmod=0555 install-pinned-apt-packages.sh /usr/local/libexec/oliphaunt/install-pinned-apt-packages.sh +COPY --chmod=0444 src/wasix/runtime/assets/build/docker/isrg-root-x1.pem /usr/local/share/oliphaunt/isrg-root-x1.pem +COPY --chmod=0555 tools/dev/acquisition.sh /usr/local/libexec/oliphaunt/acquisition.sh +COPY --chmod=0555 src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh /usr/local/libexec/oliphaunt/install-pinned-apt-packages.sh RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ @@ -61,8 +62,8 @@ ENV PATH=/opt/wasixcc-home/.wasixcc/bin:$PATH ARG OLIPHAUNT_WASIXCC_ASSET_MANIFEST_SHA256=9b0ee1aabcfecda1be72c94a9f14a16c9d8a2fc020f3dc471394d5335766c519 LABEL dev.oliphaunt.wasixcc.asset-manifest-sha256="${OLIPHAUNT_WASIXCC_ASSET_MANIFEST_SHA256}" -COPY --chmod=0444 pinned-wasixcc-assets.tsv /usr/local/share/oliphaunt-wasixcc/pinned-wasixcc-assets.tsv -COPY --chmod=0555 install-pinned-wasixcc.sh /usr/local/libexec/oliphaunt/install-pinned-wasixcc.sh +COPY --chmod=0444 src/wasix/runtime/assets/build/docker/pinned-wasixcc-assets.tsv /usr/local/share/oliphaunt-wasixcc/pinned-wasixcc-assets.tsv +COPY --chmod=0555 src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh /usr/local/libexec/oliphaunt/install-pinned-wasixcc.sh RUN printf '%s %s\n' \ "$OLIPHAUNT_WASIXCC_ASSET_MANIFEST_SHA256" \ diff --git a/src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore b/src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore new file mode 100644 index 000000000..2a32c884d --- /dev/null +++ b/src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore @@ -0,0 +1,12 @@ +** +!tools +!tools/dev +!tools/dev/acquisition.sh +!src +!src/wasix +!src/wasix/runtime +!src/wasix/runtime/assets +!src/wasix/runtime/assets/build +!src/wasix/runtime/assets/build/docker +!src/wasix/runtime/assets/build/docker/* +**/*.test.* diff --git a/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh b/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh index 7746945e0..a9cf9f6ff 100755 --- a/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh +++ b/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh @@ -2,6 +2,10 @@ set -eu export LC_ALL=C +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +acquisition_helper="$script_dir/acquisition.sh" +[ -f "$acquisition_helper" ] || acquisition_helper="$script_dir/../../../../../../tools/dev/acquisition.sh" +. "$acquisition_helper" fail() { echo "install-pinned-apt-packages: $*" >&2 @@ -52,7 +56,6 @@ for package in "$@"; do done apt_get="${OLIPHAUNT_APT_GET:-apt-get}" -sleep_command="${OLIPHAUNT_SLEEP:-sleep}" sources_file="${OLIPHAUNT_APT_SOURCES_FILE:-/etc/apt/sources.list.d/ubuntu.sources}" lists_dir="${OLIPHAUNT_APT_LISTS_DIR:-/var/lib/apt/lists}" ca_bundle="${OLIPHAUNT_CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" @@ -78,7 +81,7 @@ for required_path in "$sources_file" "$lists_dir" "$ca_bundle"; do done [ "$lists_dir" != "/" ] || fail "refusing to use / as the APT lists directory" command -v "$apt_get" >/dev/null 2>&1 || fail "missing APT command: $apt_get" -command -v "$sleep_command" >/dev/null 2>&1 || fail "missing sleep command: $sleep_command" +oliphaunt_acquisition_start 'WASIX builder APT update/install' 900 mkdir -p "$(dirname "$sources_file")" cat >"$sources_file" <&2 - "$sleep_command" "$delay" + oliphaunt_acquisition_sleep "$delay" || exit $? attempt=$((attempt + 1)) done fail "$label transaction failed after $max_attempts attempts" diff --git a/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh b/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh index 88617887e..49b9e84eb 100755 --- a/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh +++ b/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh @@ -44,6 +44,9 @@ if [ "$operation" = update ]; then update="$(( $(cat "$update_file" 2>/dev/null || echo 0) + 1 ))" printf '%s\n' "$update" >"$update_file" case "${OLIPHAUNT_FAKE_APT_MODE:-success}" in + expired-update) + printf '9999999\n' > "$OLIPHAUNT_FAKE_APT_CLOCK" + ;; transient-update) [ "$update" -gt 1 ] || exit 100 ;; @@ -69,6 +72,12 @@ printf 'sleep=%s\n' "$*" >>"${OLIPHAUNT_FAKE_APT_LOG:?}" SLEEP chmod 0755 "$fake_bin/sleep" +cat >"$fake_bin/date" <<'DATE' +#!/usr/bin/env bash +cat "$OLIPHAUNT_FAKE_APT_CLOCK" +DATE +chmod +x "$fake_bin/date" + run_case() { local name="$1" local mode="$2" @@ -76,18 +85,19 @@ run_case() { local case_root="$work_root/$name" mkdir -p "$case_root" : >"$case_root/apt.log" + printf '1000000\n' > "$case_root/clock" if [ "$seed_ca" = true ]; then mkdir -p "$case_root/certs" printf '%s\n' fixture-pinned-snapshot-root >"$case_root/certs/ca-certificates.crt" fi PATH="$fake_bin:$PATH" \ OLIPHAUNT_APT_GET="$fake_bin/apt-get" \ - OLIPHAUNT_SLEEP="$fake_bin/sleep" \ OLIPHAUNT_APT_SOURCES_FILE="$case_root/ubuntu.sources" \ OLIPHAUNT_APT_LISTS_DIR="$case_root/lists" \ OLIPHAUNT_CA_BUNDLE="$case_root/certs/ca-certificates.crt" \ OLIPHAUNT_APT_MAX_ATTEMPTS=3 \ OLIPHAUNT_APT_RETRY_DELAY_SECONDS=1 \ + OLIPHAUNT_FAKE_APT_CLOCK="$case_root/clock" \ OLIPHAUNT_FAKE_APT_MODE="$mode" \ OLIPHAUNT_FAKE_APT_STATE="$case_root/state" \ OLIPHAUNT_FAKE_APT_LOG="$case_root/apt.log" \ @@ -156,6 +166,11 @@ if grep -q 'operation=install' "$permanent_log"; then fi [ "$(grep -c '^sleep=' "$permanent_log")" -eq 2 ] || fail "permanent update failure slept an unexpected number of times" +# A successful update cannot reset the budget before install or the next retry. +expect_failure deadline expired-update "deadline" +[ "$(cat "$work_root/deadline/state/calls")" = 1 ] || fail "deadline admitted another APT command" +[ ! -e "$work_root/deadline/state/installs" ] || fail "expired update reached installation" + expect_failure missing-ca success "pinned snapshot TLS root is missing" false missing_ca_root="$work_root/missing-ca" [ ! -s "$missing_ca_root/apt.log" ] || fail "missing TLS root reached APT" @@ -171,7 +186,6 @@ fi set +e invalid_output="$( OLIPHAUNT_APT_GET="$fake_bin/apt-get" \ - OLIPHAUNT_SLEEP="$fake_bin/sleep" \ "$installer" --snapshot latest -- bash 2>&1 )" invalid_status=$? diff --git a/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh b/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh index 82942e036..42cdf23f0 100755 --- a/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh +++ b/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh @@ -2,6 +2,10 @@ set -euo pipefail export LC_ALL=C +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +acquisition_helper="$script_dir/acquisition.sh" +[ -f "$acquisition_helper" ] || acquisition_helper="$script_dir/../../../../../../tools/dev/acquisition.sh" +. "$acquisition_helper" fail() { echo "install-pinned-wasixcc: $*" >&2 @@ -71,6 +75,7 @@ trap cleanup EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM +oliphaunt_acquisition_start 'WASIX compiler assets' 1800 validate_archive_members() { local archive="$1" @@ -258,17 +263,12 @@ while IFS=$'\t' read -r kind asset_name expected_bytes expected_sha256 url extra partial="$download_root/$asset_name.partial" archive="$download_root/$asset_name" rm -f "$partial" "$archive" - curl \ + oliphaunt_acquisition_curl 900 9 5 curl \ --fail \ --location \ --silent \ --show-error \ - --retry 8 \ - --retry-all-errors \ - --retry-delay 5 \ - --retry-max-time 900 \ --connect-timeout 20 \ - --max-time 900 \ --speed-limit 1024 \ --speed-time 120 \ --max-filesize "$expected_bytes" \ diff --git a/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh b/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh index b602f013c..def5bb69b 100755 --- a/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh +++ b/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh @@ -12,6 +12,10 @@ fail() { } work_root="$(mktemp -d)" +mkdir -p "$work_root/no-delay" +printf '#!/bin/sh\n[ "$1" = 5 ] && exit 0\nexec "%s" "$@"\n' "$(command -v sleep)" > "$work_root/no-delay/sleep" +chmod +x "$work_root/no-delay/sleep" +export PATH="$work_root/no-delay:$PATH" trap 'rm -rf "$work_root"' EXIT fixtures="$work_root/fixtures" fake_bin="$work_root/fake-bin" @@ -249,10 +253,9 @@ expected_success_compiler_version="$(printf '%s\n' \ (cd "$success_root" && sha256sum --check --strict .oliphaunt-toolchain-assets.sha256 >/dev/null) || fail "installed identity manifest does not verify" for required_flag in \ - '--retry 8' \ - '--retry-all-errors' \ + '--retry 0' \ '--connect-timeout 20' \ - '--max-time 900' \ + '--max-time' \ '--proto =https' \ '--proto-redir =https' \ '--remove-on-error'; do diff --git a/src/wasix/runtime/assets/build/docker_contrib_extensions.sh b/src/wasix/runtime/assets/build/docker_contrib_extensions.sh index 078df839e..4a26058de 100755 --- a/src/wasix/runtime/assets/build/docker_contrib_extensions.sh +++ b/src/wasix/runtime/assets/build/docker_contrib_extensions.sh @@ -41,7 +41,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_initdb.sh b/src/wasix/runtime/assets/build/docker_initdb.sh index c279dd208..39e4aee0c 100755 --- a/src/wasix/runtime/assets/build/docker_initdb.sh +++ b/src/wasix/runtime/assets/build/docker_initdb.sh @@ -40,7 +40,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_oliphaunt.sh b/src/wasix/runtime/assets/build/docker_oliphaunt.sh index 6726aa1cd..82c21a6aa 100755 --- a/src/wasix/runtime/assets/build/docker_oliphaunt.sh +++ b/src/wasix/runtime/assets/build/docker_oliphaunt.sh @@ -41,7 +41,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_pgdump.sh b/src/wasix/runtime/assets/build/docker_pgdump.sh index b888a6f0e..5f77e7609 100755 --- a/src/wasix/runtime/assets/build/docker_pgdump.sh +++ b/src/wasix/runtime/assets/build/docker_pgdump.sh @@ -40,7 +40,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_pgxs_extensions.sh b/src/wasix/runtime/assets/build/docker_pgxs_extensions.sh index 1bbcfb188..110799144 100755 --- a/src/wasix/runtime/assets/build/docker_pgxs_extensions.sh +++ b/src/wasix/runtime/assets/build/docker_pgxs_extensions.sh @@ -41,7 +41,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_psql.sh b/src/wasix/runtime/assets/build/docker_psql.sh index 131f027ab..b634585c8 100755 --- a/src/wasix/runtime/assets/build/docker_psql.sh +++ b/src/wasix/runtime/assets/build/docker_psql.sh @@ -40,7 +40,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_runtime_support.sh b/src/wasix/runtime/assets/build/docker_runtime_support.sh index 4ead94638..2115f3585 100755 --- a/src/wasix/runtime/assets/build/docker_runtime_support.sh +++ b/src/wasix/runtime/assets/build/docker_runtime_support.sh @@ -40,7 +40,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/prepare_postgres_source.sh b/src/wasix/runtime/assets/build/prepare_postgres_source.sh index 9e725f2fc..f215338da 100755 --- a/src/wasix/runtime/assets/build/prepare_postgres_source.sh +++ b/src/wasix/runtime/assets/build/prepare_postgres_source.sh @@ -4,6 +4,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$SCRIPT_DIR/wasix_third_party.sh" REPO_ROOT="$(oliphaunt_wasix_repo_root "$SCRIPT_DIR")" +. "$REPO_ROOT/tools/dev/acquisition.sh" . "$REPO_ROOT/src/third-party/postgres/fetch-source.sh" SOURCE_TOML="$REPO_ROOT/src/third-party/postgres/source.toml" PATCH_DIR="$REPO_ROOT" diff --git a/src/wasix/runtime/assets/build/wasix_third_party.sh b/src/wasix/runtime/assets/build/wasix_third_party.sh index b581d73d8..2b7b0c9e9 100755 --- a/src/wasix/runtime/assets/build/wasix_third_party.sh +++ b/src/wasix/runtime/assets/build/wasix_third_party.sh @@ -69,7 +69,7 @@ oliphaunt_wasix_run_extension_build_in_docker_if_needed() { "$docker" build \ -t "$image" \ -f "$root/docker/Dockerfile" \ - "$root/docker" + "$(cd "$root/../../../../.." && pwd)" else echo "reusing Docker image $image" fi diff --git a/src/wasix/runtime/moon.yml b/src/wasix/runtime/moon.yml index 1235f5395..445fe19dc 100644 --- a/src/wasix/runtime/moon.yml +++ b/src/wasix/runtime/moon.yml @@ -109,6 +109,7 @@ tasks: - unit script: "set -e\nbash src/wasix/runtime/tools/check-shim-abi.sh\nbash src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh\nbash src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh\n" inputs: + - /tools/dev/acquisition.sh - assets/build/docker/**/* - assets/build/wasix_shim/**/* - tools/check-shim-abi.sh @@ -134,6 +135,23 @@ tasks: deps: - source-inputs:source-fetch-wasix-runtime inputs: + - "$ASSET_PROFILE" + - "$IMAGE" + - "$OLIPHAUNT_WASM_SOURCE_LANE" + - "$CONTAINER_ROOT" + - "$CONTAINER_GENERATED_ROOT" + - "$OLIPHAUNT_WASM_WASIX_COPT" + - "$OLIPHAUNT_WASM_WASIX_LOPT" + - "$OLIPHAUNT_WASM_WASIX_CONFIGURE_WASM_OPT" + - "$OLIPHAUNT_WASM_WASIX_BUILD_WASM_OPT" + - "$OLIPHAUNT_WASM_WASM_OPT_FLAGS" + - "$OLIPHAUNT_WASM_WASM_OPT_SUPPRESS_DEFAULT" + - "$OLIPHAUNT_WASM_WASM_OPT_PRESERVE_UNOPTIMIZED" + - "$OLIPHAUNT_WASM_WASIX_COMPILER_FLAGS" + - "$OLIPHAUNT_WASM_WASIX_LINKER_FLAGS" + - "$OLIPHAUNT_WASM_PG18_DISABLE_SPINLOCKS" + - "$CONTAINER_BUILD_DIR" + - "$CONTAINER_PGSRC" - "@group(legal-files)" - "@group(cargo-workspace)" - /src/wasix/sdks/rust/THIRD_PARTY_NOTICES.md @@ -149,7 +167,9 @@ tasks: group: build - project: extension-runtime-contract group: contract + - /tools/dev/acquisition.sh - assets/build/**/* + - "!assets/build/**/*.test.*" - postgres/**/* - /src/third-party/postgres/**/* - "!/src/third-party/postgres/**/*.test.*" diff --git a/src/wasix/runtime/tools/build-compiler-output.sh b/src/wasix/runtime/tools/build-compiler-output.sh index f0d0e3d9f..952cd9451 100755 --- a/src/wasix/runtime/tools/build-compiler-output.sh +++ b/src/wasix/runtime/tools/build-compiler-output.sh @@ -34,7 +34,24 @@ export OLIPHAUNT_WASM_BUILD_PROFILE="$asset_profile" bash src/third-party/tools/fetch-sources.sh wasix-runtime --verify-only bash src/wasix/runtime/assets/build/prepare_postgres_source.sh >/dev/null build=src/wasix/runtime/assets/build -if [ "${OLIPHAUNT_SKIP_BUILD:-0}" != "1" ]; then +# Moon owns the complete source/toolchain input hash. Keep the absolute-path +# Make tree in the existing compilation cache, and verify its recorded bytes +# before bypassing compilation. Raw script calls keep normal incremental builds. +compiler_tree=target/oliphaunt-wasix/wasix-build/work/docker-oliphaunt +receipt=target/oliphaunt-wasix/wasix-build/build/compiler-input-hash +checksums=target/oliphaunt-wasix/wasix-build/build/compiler-output-sha256 +reuse=0 +if [ "${MOON_TARGET:-}" = liboliphaunt-wasix:compiler-output ] && + [[ "${MOON_TASK_HASH:-}" =~ ^[0-9a-f]{64}$ ]] && + [ "${FORCE_RECONFIGURE:-0}" != 1 ] && [ "${FORCE_IMAGE_BUILD:-0}" != 1 ] && + [ -s "$receipt" ] && [ -s "$checksums" ] && + [ "$(cat "$receipt")" = "$MOON_TASK_HASH" ] && + sha256sum --check --status "$checksums"; then + reuse=1 + echo 'Reusing verified WASIX compiler output for the current Moon input hash' +fi +if [ "${OLIPHAUNT_SKIP_BUILD:-0}" != "1" ] && [ "$reuse" = 0 ]; then + rm -f "$receipt" "$checksums" for script in docker_oliphaunt docker_runtime_support docker_initdb; do bash "$build/$script.sh" done @@ -42,3 +59,11 @@ fi awk -v profile="$asset_profile" '$0 == "profile=" profile {found=1} END {exit !found}' \ target/oliphaunt-wasix/wasix-build/work/docker-oliphaunt/.oliphaunt-wasix-build-profile cargo run -p xtask -- assets stage-runtime + +if [ "${MOON_TARGET:-}" = liboliphaunt-wasix:compiler-output ] && + [[ "${MOON_TASK_HASH:-}" =~ ^[0-9a-f]{64}$ ]] && [ "$reuse" = 0 ] && + [ "${OLIPHAUNT_SKIP_BUILD:-0}" != 1 ]; then + mkdir -p "$(dirname "$receipt")" + find "$compiler_tree" -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum > "$checksums" + printf '%s\n' "$MOON_TASK_HASH" > "$receipt" +fi diff --git a/src/wasix/runtime/tools/build-runtime-portable.test.sh b/src/wasix/runtime/tools/build-runtime-portable.test.sh index fdaa88647..40465e068 100755 --- a/src/wasix/runtime/tools/build-runtime-portable.test.sh +++ b/src/wasix/runtime/tools/build-runtime-portable.test.sh @@ -52,4 +52,24 @@ bash "$fixture/$owner/tools/build-runtime-portable.sh" --package-only echo profile=debug > "$receipt" if OLIPHAUNT_SKIP_BUILD=1 bash "$fixture/$owner/tools/build-runtime-portable.sh"; then exit 1; fi [ "$(tail -1 "$BUILD_LOG")" = prepare_postgres_source ] +# Only a complete matching input/output receipt skips the compiler. Corrupt +# bytes and a changed input hash must both return to the build path. +echo profile=release > "$receipt" +export MOON_TARGET=liboliphaunt-wasix:compiler-output +export MOON_TASK_HASH="$(printf '%064d' 1)" +: > "$BUILD_LOG" +bash "$fixture/$owner/tools/build-compiler-output.sh" +: > "$BUILD_LOG" +bash "$fixture/$owner/tools/build-compiler-output.sh" +! grep -q '^docker_' "$BUILD_LOG" +echo corrupted >> "$receipt" +: > "$BUILD_LOG" +# The fixture build scripts do not repair the profile marker; its release line +# remains present, so the assertion still allows the fresh compiler path. +bash "$fixture/$owner/tools/build-compiler-output.sh" +grep -q '^docker_oliphaunt$' "$BUILD_LOG" +export MOON_TASK_HASH="$(printf '%064d' 2)" +: > "$BUILD_LOG" +bash "$fixture/$owner/tools/build-compiler-output.sh" +grep -q '^docker_oliphaunt$' "$BUILD_LOG" echo 'WASIX build order, core-only selection, failure propagation, and stale-profile refusal passed.' diff --git a/src/wasix/sdks/rust/moon.yml b/src/wasix/sdks/rust/moon.yml index 72f258d86..a806e4c9e 100644 --- a/src/wasix/sdks/rust/moon.yml +++ b/src/wasix/sdks/rust/moon.yml @@ -46,13 +46,13 @@ tasks: - project: "database-resources" group: "cargo-carrier-sources" test-regression: - tags: ["regression", "runtime", "requires-rust"] + tags: ["regression", "runtime", "requires-rust", "ci-wasix-release-regression"] command: "bash src/wasix/runtime/tools/runtime-smoke.sh regression" - deps: [{target: "cargo-sources", cacheStrategy: hash}, "liboliphaunt-wasix:runtime-aot", "extension-artifacts-wasix:build-aot", "postgres-tools-wasix:build-aot"] + deps: [{target: "cargo-sources", cacheStrategy: hash}, "liboliphaunt-wasix:runtime-aot", "extension-artifacts-wasix:build-target", "extension-artifacts-wasix:build-aot", "postgres-tools-wasix:build-aot"] inputs: ["@group(code)", "@group(cargo-workspace)", "/src/wasix/runtime/tools/{runtime-smoke,runtime-preflight,cargo-test-filter}.sh", "/src/wasix/runtime/tools/wasix-extension-features.mts", "/src/extensions/artifacts/packages/tools/**/*"] - options: {runFromWorkspaceRoot: true, cache: local} + options: {runFromWorkspaceRoot: true, cache: false} test-integration: - tags: ["runtime", "integration", "requires-rust"] + tags: ["runtime", "integration", "requires-rust", "ci-wasix-release-regression", "platform-linux-x64-gnu"] command: "bash src/wasix/sdks/rust/tools/test-resources.sh" deps: - target: cargo-sources @@ -62,6 +62,7 @@ tasks: - "database-resources:build-wasix-icu" - "database-resources:package-icu" inputs: + - "/src/wasix/runtime/tools/runtime-preflight.sh" - "@group(code)" - "@group(cargo-workspace)" options: diff --git a/src/wasix/sdks/rust/tools/test-resources.sh b/src/wasix/sdks/rust/tools/test-resources.sh index 923ed5dbe..796dc9b12 100644 --- a/src/wasix/sdks/rust/tools/test-resources.sh +++ b/src/wasix/sdks/rust/tools/test-resources.sh @@ -2,6 +2,11 @@ set -euo pipefail root="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" cd "$root" +. src/wasix/runtime/tools/runtime-preflight.sh +oliphaunt_runtime_wasm_require +export OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD=1 +export OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR="$root/target/oliphaunt-wasix/assets" +export OLIPHAUNT_WASM_GENERATED_AOT_DIR="$root/target/oliphaunt-wasix/aot" version="$(cat src/database-resources/VERSION)" export OLIPHAUNT_TEST_STANDARD_SEED="$root/target/database-resources/release-assets/database-resources-$version-seed-wasix-standard.tar.zst" export OLIPHAUNT_TEST_ICU_SEED="$root/target/database-resources/release-assets/database-resources-$version-seed-wasix-icu.tar.zst" @@ -9,4 +14,9 @@ OLIPHAUNT_TEST_ICU_ROOT="$(mktemp -d)" export OLIPHAUNT_TEST_ICU_ROOT trap 'rm -rf "$OLIPHAUNT_TEST_ICU_ROOT"' EXIT tar -xzf "$root/target/database-resources/release-assets/database-resources-$version-icu-data.tar.gz" -C "$OLIPHAUNT_TEST_ICU_ROOT" -cargo test -p oliphaunt-wasix --locked --test resources -- --ignored --test-threads=1 +cargo test -p oliphaunt-wasix --locked --test resources --color never -- --ignored --test-threads=1 \ + 2>&1 | tee "$OLIPHAUNT_TEST_ICU_ROOT/tests.log" +grep -Eq '^test result: ok\. [1-9][0-9]* passed;' "$OLIPHAUNT_TEST_ICU_ROOT/tests.log" || { + echo 'WASIX resource tests must execute and pass' >&2 + exit 1 +} diff --git a/src/wasix/sdks/ts/moon.yml b/src/wasix/sdks/ts/moon.yml index 679ca0493..298d4986e 100644 --- a/src/wasix/sdks/ts/moon.yml +++ b/src/wasix/sdks/ts/moon.yml @@ -78,7 +78,7 @@ tasks: - package - release - artifact-package - - ci-wasix-ts-sdk-package + - ci-wasix-ts-package script: bun run --cwd src/wasix/sdks/ts package deps: - wasix-browser-host:build @@ -155,9 +155,13 @@ tasks: - consumer - ci-wasix-ts-sdk-package deps: + - oliphaunt-query-ts:build - extension-artifacts-wasix:compiler-output - liboliphaunt-wasix:runtime-portable - - oliphaunt-wasix-ts:package + - target: oliphaunt-wasix-ts:package + cacheStrategy: ignored + - oliphaunt-wasix-ts:build + - wasix-browser-host:build - oliphaunt-wasix-napi:build-release-assets inputs: &inputs @@ -166,7 +170,7 @@ tasks: - /src/examples/wasix/browser/**/* - project: shared-test-fixtures group: fixtures - - "**/*" + - "@group(code)" - /tools/dev/bun.sh - /tools/dev/deno.sh - /src/wasix/runtime/tools/wasix-*.mts @@ -182,11 +186,15 @@ tasks: - browser - ci-wasix-ts-sdk-package deps: + - oliphaunt-query-ts:build - extension-artifacts-wasix:compiler-output - database-resources:package-wasix - database-resources:package-icu - liboliphaunt-wasix:runtime-portable - - oliphaunt-wasix-ts:package + - target: oliphaunt-wasix-ts:package + cacheStrategy: ignored + - oliphaunt-wasix-ts:build + - wasix-browser-host:build inputs: *inputs options: diff --git a/src/wasix/sdks/ts/tools/integration/smoke-browser.sh b/src/wasix/sdks/ts/tools/integration/smoke-browser.sh index cc4268b73..1f68be834 100644 --- a/src/wasix/sdks/ts/tools/integration/smoke-browser.sh +++ b/src/wasix/sdks/ts/tools/integration/smoke-browser.sh @@ -37,7 +37,6 @@ cleanup() { exit "$status" } trap cleanup EXIT -bun run --cwd "$root/src/query/ts" build bun pm --cwd "$root/src/query/ts" pack --filename "$scratch/query.tgz" --quiet bun "$tool" --prepare "$scratch" "$@" configuration="$scratch/browser.json" diff --git a/src/wasix/sdks/ts/tools/integration/smoke-node.sh b/src/wasix/sdks/ts/tools/integration/smoke-node.sh index cb034245e..47f87377a 100644 --- a/src/wasix/sdks/ts/tools/integration/smoke-node.sh +++ b/src/wasix/sdks/ts/tools/integration/smoke-node.sh @@ -13,7 +13,6 @@ done deadline="$(command -v gtimeout || command -v timeout)" scratch="$(mktemp -d)" trap 'rm -rf "$scratch"' EXIT -bun run --cwd "$root/src/query/ts" build bun pm --cwd "$root/src/query/ts" pack --filename "$scratch/query.tgz" --quiet bun "$root/src/wasix/sdks/ts/tools/integration/smoke-node.mts" "$scratch" "${args[@]}" cd "$scratch/consumer" diff --git a/tools/ci/capture-ci-test-observations.sh b/tools/ci/capture-ci-test-observations.sh index 6cb9ae224..71ae53b29 100644 --- a/tools/ci/capture-ci-test-observations.sh +++ b/tools/ci/capture-ci-test-observations.sh @@ -6,12 +6,18 @@ directory="${1:?expected observation directory}" bash tools/dev/bun.sh tools/ci/ci-plan-test-observations.mts "$directory" unset MOON_BASE MOON_HEAD export MOON_CACHE=off +# Each request writes its own file. Four workers bound memory and preserve all +# graph/affected proofs while avoiding serial workspace startup for every case. +# NUL records preserve paths and the empty target of an affected query on BSD/GNU xargs. while IFS=$'\t' read -r kind id target; do + printf '%s\0%s\0%s\0' "$kind" "$id" "$target" +done < "$directory/requests.tsv" | xargs -0 -n 3 -P 4 bash -eu -c ' + directory=$1 moon_bin=$2 kind=$3 id=$4 target=$5 case "$kind" in affected) - "${MOON_BIN:-moon}" query affected stdin --upstream none --downstream deep \ + "$moon_bin" query affected stdin --upstream none --downstream deep \ < "$directory/affected-$id.input" > "$directory/affected-$id.json" ;; - task) "${MOON_BIN:-moon}" task-graph "$target" --json > "$directory/task-$id.json" ;; + task) "$moon_bin" task-graph "$target" --json > "$directory/task-$id.json" ;; *) echo "unexpected test observation kind: $kind" >&2; exit 1 ;; esac -done < "$directory/requests.tsv" +' ci-observation "$directory" "${MOON_BIN:-moon}" diff --git a/tools/ci/capture-ci-test-observations.test.sh b/tools/ci/capture-ci-test-observations.test.sh new file mode 100644 index 000000000..ce3d3bdb4 --- /dev/null +++ b/tools/ci/capture-ci-test-observations.test.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(git rev-parse --show-toplevel)" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +cat > "$work/moon" <<'MOON' +#!/bin/sh +# Fail graph queries after successful affected queries have already completed. +if [ "$1" = task-graph ]; then exit 7; fi +printf '{}\n' +MOON +chmod +x "$work/moon" +if MOON_BIN="$work/moon" bash "$root/tools/ci/capture-ci-test-observations.sh" "$work/observations with spaces"; then + echo 'parallel observation capture swallowed a failed query' >&2 + exit 1 +fi +[ -n "$(find "$work/observations with spaces" -name 'affected-*.json' -print -quit)" ] +echo 'Parallel Moon observation failures propagate after completed queries' diff --git a/tools/ci/check-workflows.sh b/tools/ci/check-workflows.sh index 267eb4247..3e8559a17 100755 --- a/tools/ci/check-workflows.sh +++ b/tools/ci/check-workflows.sh @@ -28,6 +28,7 @@ require zizmor run actionlint -ignore 'unexpected key "queue" for "concurrency" section' run zizmor --config .github/zizmor.yml --min-severity medium --persona auditor .github/workflows .github/actions run bash tools/dev/bun.sh test ./tools/ci/workflow-security.test.mts +run bash tools/dev/bun.sh test ./tools/ci/workflow-caches.test.mts run tools/dev/bun.sh tools/ci/workflow-security.mts run bash .github/scripts/check-ci-gate.test.sh run bash tools/dev/bun.sh test ./.github/scripts/resolve-mobile-e2e.test.mts @@ -42,6 +43,7 @@ export OLIPHAUNT_CI_TEST_OBSERVATIONS="$observations" run bash tools/dev/bun.sh test ./.github/scripts/moon-task-capabilities.test.mts run bash .github/scripts/write-affected-moon-target-matrices.test.sh run bash .github/scripts/resolve-planned-moon-execution.test.sh +run bash tools/ci/capture-ci-test-observations.test.sh run bash tools/ci/with-projects.sh --exec bash tools/ci/capture-ci-test-observations.sh "$observations" run bash tools/ci/ci-release-scope.test.sh run bash tools/ci/with-projects.sh test \ diff --git a/tools/ci/ci-plan-node-products.test.mts b/tools/ci/ci-plan-node-products.test.mts index 73cec77f9..f81d93be0 100644 --- a/tools/ci/ci-plan-node-products.test.mts +++ b/tools/ci/ci-plan-node-products.test.mts @@ -1,22 +1,24 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; +import { loadExtensionTargetProfiles } from '../../src/extensions/contracts/extension-target-profiles.mts'; +import { publishedConsumerDependencies } from '../../src/native/sdks/ts/tools/published-consumer.mts'; +import { + contribCarrierDescriptor, + extensionProductForSqlName, +} from '../release/release-artifact-targets.mts'; import { buildPlan, loadGraph, normalizeFiles } from '../release/release-graph.mts'; import { affectedNames, triggeringProjectNames, triggeringTaskNames } from './affected.mts'; import { dependencyPlatformTargets, + extensionArtifactsNativeMatrixForPlan, jobTargetsForJobs, planForReleaseProducts, planJobsForAffected, + renderPlanForFullRun, requiredTasksForAffected, } from './ci_plan.mts'; import { combinedNativeWasix, paths, taskRoots } from './ci-plan-test-inputs.mts'; import { affectedObservation, taskObservation } from './ci-plan-test-observations.mts'; -import { loadExtensionTargetProfiles } from '../../src/extensions/contracts/extension-target-profiles.mts'; -import { - contribCarrierDescriptor, - extensionProductForSqlName, -} from '../release/release-artifact-targets.mts'; -import { publishedConsumerDependencies } from '../../src/native/sdks/ts/tools/published-consumer.mts'; const GRAPH = loadGraph('ci-plan-node-products.test.mts'); const NATIVE_TS_CONSUMER_JOBS = [ @@ -161,8 +163,10 @@ test('an empty Moon selection requires no product tasks or releases', () => { assert.deepEqual(buildPlan(GRAPH, [], 'ci-plan-node-products.test.mts').releaseProducts, []); }); -test('WASIX qualification selects all same-run release regression inputs', () => { +test('WASIX qualification and Linux diagnostics select all same-run regression inputs', () => { const required = [ + 'oliphaunt-wasix-rust:test-regression', + 'oliphaunt-wasix-rust:test-integration', 'liboliphaunt-wasix:runtime-portable', 'liboliphaunt-wasix:runtime-aot', 'extension-artifacts-wasix:compiler-output', @@ -170,23 +174,37 @@ test('WASIX qualification selects all same-run release regression inputs', () => 'extension-artifacts-wasix:build-aot', 'postgres-tools-wasix:compiler-output', 'postgres-tools-wasix:build-aot', + 'database-resources:build-wasix-standard', + 'database-resources:build-wasix-icu', + 'database-resources:package-icu', ]; const roots = new Set(['liboliphaunt-wasix:release-assets']); const jobs = planJobsForAffected(roots); const affected = jobTargetsForJobs(jobs, requiredTasksForAffected(roots)); const release = planForReleaseProducts(['liboliphaunt-wasix'], 'a'.repeat(40)); - for (const targets of [affected, release.job_targets]) { + const full = renderPlanForFullRun(); + const linux = renderPlanForFullRun({ wasmTarget: 'linux-x64-gnu' }); + for (const plan of [release, full, linux]) { + assert(plan.jobs.includes('wasix-release-regression')); + assert(!plan.builder_jobs.includes('wasix-release-regression')); + assert(plan.jobs.includes('js-sdk-package')); + assert(plan.extension_artifacts_wasix_matrix.include.length > 0); + assert(plan.liboliphaunt_wasix_aot_runtime_matrix_linux.include.length > 0); + } + for (const targets of [affected, release.job_targets, full.job_targets, linux.job_targets]) { const selected = new Set(Object.values(targets).flat()); for (const target of required) assert(selected.has(target), `missing evidence input ${target}`); } assert(jobs.has('extension-artifacts-wasix')); + assert( + !renderPlanForFullRun({ wasmTarget: 'macos-arm64' }).jobs.includes('wasix-release-regression'), + ); }); test('shared Rust query changes stage the native and WASIX consumer artifacts', () => { const result = effects(paths.sdksRustQuerySrcLibRs); - for (const consumer of ['oliphaunt-wasix-ts:package', 'oliphaunt-wasix-ts:test-consumer']) { - assert(result.jobTargets['wasix-ts-sdk-package'].includes(consumer), consumer); - } + assert(result.jobTargets['wasix-ts-package'].includes('oliphaunt-wasix-ts:package')); + assert(result.jobTargets['wasix-ts-sdk-package'].includes('oliphaunt-wasix-ts:test-consumer')); for (const producer of [ 'oliphaunt-rust:package', 'oliphaunt-query:package', @@ -467,6 +485,8 @@ test('combined JavaScript SDK and WASIX N-API changes release only changed produ 'node-direct', 'wasix-napi', 'wasix-napi-release-assets', + 'wasix-release-regression', + 'wasix-ts-package', 'wasix-ts-sdk-package', ]); assert.deepEqual(result.releaseProducts, ['oliphaunt-js', 'oliphaunt-wasix-napi']); @@ -540,6 +560,8 @@ test('WASIX N-API source selects only its real WASIX artifact inputs', () => { 'liboliphaunt-wasix-runtime', 'wasix-napi', 'wasix-napi-release-assets', + 'wasix-release-regression', + 'wasix-ts-package', 'wasix-ts-sdk-package', ]); assert.deepEqual(result.releaseProducts, ['oliphaunt-wasix-napi']); @@ -887,3 +909,96 @@ test('mixed SDK and query releases retain the Linux native consumer alongside mo ), ); }); + +test('frozen iOS metadata only replaces the unchanged package input in affected runs', () => { + const old = process.env.OLIPHAUNT_REUSE_IOS_CARRIER; + try { + assert( + !Object.hasOwn( + affectedObservation(paths.sdksKotlinToolsStageReleaseArtifactsMts).tasks, + 'liboliphaunt-native:finalize-runtime-ios-abi', + ), + ); + assert( + Object.hasOwn( + affectedObservation(paths.runtimesLiboliphauntNativeSrcLiboliphauntProcessC).tasks, + 'liboliphaunt-native:finalize-runtime-ios-abi', + ), + ); + process.env.OLIPHAUNT_REUSE_IOS_CARRIER = 'true'; + const roots = new Set(['integration-examples:react-native-android-build']); + const warm = requiredTasksForAffected(roots); + assert(!warm.has('liboliphaunt-native:finalize-runtime-ios-abi')); + assert( + requiredTasksForAffected(roots, undefined, false).has( + 'liboliphaunt-native:finalize-runtime-ios-abi', + ), + ); + roots.add('liboliphaunt-native:finalize-runtime-ios-abi'); + assert(requiredTasksForAffected(roots).has('liboliphaunt-native:finalize-runtime-ios-abi')); + const release = planForReleaseProducts(['oliphaunt-react-native'], 'a'.repeat(40)); + assert(release.jobs.includes('liboliphaunt-native-ios-abi')); + assert.equal(release.reuse_ios_carrier, false); + } finally { + if (old === undefined) delete process.env.OLIPHAUNT_REUSE_IOS_CARRIER; + else process.env.OLIPHAUNT_REUSE_IOS_CARRIER = old; + } +}); + +test('Android extension packaging selects matching dependency-complete Linux support', () => { + for (const target of ['android-arm64-v8a', 'android-x86_64']) { + const matrix = extensionArtifactsNativeMatrixForPlan( + new Set(['extension-artifacts-native']), + new Set([target]), + new Set([extensionProductForSqlName('earthdistance')]), + 'all', + ); + assert.deepEqual( + matrix.include.map((row) => row.target), + [target, 'linux-x64-gnu'], + ); + const android = matrix.include.find((row) => row.target === target); + const linux = matrix.include.find((row) => row.target === 'linux-x64-gnu'); + for (const product of android.extensions_csv.split(',')) { + assert(linux.extensions_csv.split(',').includes(product), product); + } + assert(linux.sql_names_csv.split(',').includes('cube')); + } +}); + +test('WASIX package README and compiler unit fixtures do not select runtime compilers', () => { + for (const file of [paths.wasixSdkReadme, paths.wasixDockerTest, paths.nativeExtensionFixture]) { + const observation = affectedObservation([file]); + const roots = new Set(triggeringTaskNames(observation.tasks)); + const required = requiredTasksForAffected(roots); + assert(!required.has('liboliphaunt-wasix:compiler-output'), file); + assert(!required.has('extension-artifacts-native:build-target'), file); + assert(!required.has('liboliphaunt-wasix-postmaster:postgres-build'), file); + } + const jobs = planJobsForAffected( + new Set(triggeringTaskNames(affectedObservation([paths.wasixSdkReadme]).tasks)), + ); + assert(jobs.has('wasix-ts-package')); + assert(!jobs.has('wasix-ts-sdk-package')); +}); + +test('SDK runtime test edits select their final hosted execution roots', () => { + for (const [file, job, target] of [ + [paths.nativeRustRuntimeTests, 'native-consumers', 'oliphaunt-rust:test-integration'], + [paths.mobileBrokerRuntimeTests, 'native-consumers', 'oliphaunt-mobile-bindings:test-native'], + [paths.nativeSwiftRuntimeTests, 'native-consumers', 'oliphaunt-swift:test-native'], + [paths.nativeKotlinRuntimeTests, 'native-consumers', 'oliphaunt-kotlin:test-native-bindings'], + [ + paths.wasixResourceRuntimeTests, + 'wasix-release-regression', + 'oliphaunt-wasix-rust:test-integration', + ], + ]) { + const selected = effects(file); + assert(selected.jobs.includes(job), `${file} must schedule ${job}`); + const roots = new Set(selected.directTasks); + const executable = requiredTasksForAffected(roots); + assert(executable.has(target), `${target} is missing from executable closure`); + assert(jobTargetsForJobs(new Set(selected.jobs), executable)[job].includes(target)); + } +}); diff --git a/tools/ci/ci-plan-test-inputs.mts b/tools/ci/ci-plan-test-inputs.mts index 7000c063e..fa2dca846 100644 --- a/tools/ci/ci-plan-test-inputs.mts +++ b/tools/ci/ci-plan-test-inputs.mts @@ -1,5 +1,15 @@ // Sample changes exercised by CI planning tests; never used for production affectedness. export const paths = { + postmasterReadme: 'src/wasix/postmaster/README.md', + nativeRustRuntimeTests: 'src/native/sdks/rust/tests/native_sql_regression.rs', + nativeSwiftRuntimeTests: 'src/native/sdks/swift/Tests/OliphauntTests/NativeRuntimeTests.swift', + nativeKotlinRuntimeTests: + 'src/native/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/NativeBindingsTest.kt', + mobileBrokerRuntimeTests: 'src/native/sdks/rust/tests/mobile_broker.rs', + wasixResourceRuntimeTests: 'src/wasix/sdks/rust/tests/resources.rs', + wasixSdkReadme: 'src/wasix/sdks/ts/README.md', + wasixDockerTest: 'src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh', + nativeExtensionFixture: 'src/extensions/artifacts/native/tools/create-artifact.test.mts', windowsVcRuntimePolicy: 'tools/packaging/windows-vc-runtime-policy.json', wasixRuntimeCarrierSource: 'src/wasix/runtime/crates/assets/src/lib.rs', wasixToolsCarrierSource: 'src/wasix/postgres-tools/crates/tools/src/lib.rs', @@ -77,7 +87,6 @@ export const paths = { runtimesLiboliphauntWasixPostmasterWasmerBinVerifyPostmasterConcurrencyContractTestMts: 'src/wasix/postmaster/wasmer/bin/verify-postmaster-concurrency-contract.test.mts', srcSourcesThirdPartyNativeREADMEMd: 'src/sources/third-party/native/README.md', - toolsDevMaestroToml: 'tools/dev/maestro.toml', postgresToolsWasixCratesToolsSrcLibRs: 'src/wasix/postgres-tools/crates/tools/src/lib.rs', runtimesLiboliphauntWasixToolsXtaskSrcMainRs: 'src/wasix/runtime/tools/xtask/src/main.rs', runtimesLiboliphauntWasixAssetsBuildDockerInstallPinnedWasixccSh: diff --git a/tools/ci/ci-plan-wasix-postmaster-release.test.mts b/tools/ci/ci-plan-wasix-postmaster-release.test.mts index 533e6d777..b24228464 100644 --- a/tools/ci/ci-plan-wasix-postmaster-release.test.mts +++ b/tools/ci/ci-plan-wasix-postmaster-release.test.mts @@ -1,5 +1,3 @@ -import { affectedObservation, taskObservation } from './ci-plan-test-observations.mts'; -import { paths, taskRoots } from './ci-plan-test-inputs.mts'; import assert from 'node:assert/strict'; import { test } from 'node:test'; import { buildPlan, loadGraph, normalizeFiles } from '../release/release-graph.mts'; @@ -10,6 +8,8 @@ import { renderPlanWithSelection, selectedExtensionProductsForPlan, } from './ci_plan.mts'; +import { paths, taskRoots } from './ci-plan-test-inputs.mts'; +import { affectedObservation, taskObservation } from './ci-plan-test-observations.mts'; const taskGraph = taskObservation; @@ -170,7 +170,6 @@ test('source prose, transport tests, and unrelated toolchains do not rebuild run 'liboliphaunt-wasix-postmaster:test', ], [paths.srcSourcesThirdPartyNativeREADMEMd, null], - [paths.toolsDevMaestroToml, 'ci-workflows:check'], ]; for (const [relativePath, expectedTask] of cases) { const effects = directEffects(relativePath); @@ -244,3 +243,13 @@ test('native lifecycle supervisor changes select its exact hosted proof', () => paths.runtimesLiboliphauntWasixPostmasterLibProcessSupervisionSh, ); }); + +test('shipped Postmaster documentation has a cheap producer without losing release ownership', () => { + const effects = directEffects(paths.postmasterReadme); + assert(effects.directTasks.includes('liboliphaunt-wasix-postmaster:package-docs')); + assert(!effects.jobs.includes('wasix-postmaster')); + const release = buildPlan(loadGraph(), normalizeFiles([paths.postmasterReadme])); + assert(release.releaseProducts.includes('liboliphaunt-wasix-postmaster')); + const closure = taskGraph(taskRoots.liboliphauntWasixPostmasterReleaseAssets); + assert(closure.some(({ target }) => target === 'liboliphaunt-wasix-postmaster:package-docs')); +}); diff --git a/tools/ci/ci-plan.sh b/tools/ci/ci-plan.sh index b30d16a0a..c94d9781d 100644 --- a/tools/ci/ci-plan.sh +++ b/tools/ci/ci-plan.sh @@ -30,6 +30,25 @@ if [[ $# == 0 && (${CI_GENERATED_RELEASE_PR:-false} == true || (${GITHUB_EVENT_N export CI_RELEASE_PRODUCTS_JSON fi fi +# A frozen carrier replaces only the React Native package's unchanged Apple +# metadata input. Missing/stale state falls back to the normal producers. +unset OLIPHAUNT_REUSE_IOS_CARRIER +carrier_cache=target/ci/ios-carrier +if [[ $# == 0 && ${GITHUB_EVENT_NAME:-} != workflow_dispatch && ${CI_RELEASE_PRODUCTS_JSON:-[]} == '[]' && -s "$carrier_cache/source-sha" && -s "$carrier_cache/manifest.json" ]]; then + producer_sha="$(cat "$carrier_cache/source-sha")" + if [[ "$producer_sha" =~ ^[0-9a-f]{40}$ ]] && git cat-file -e "$producer_sha^{commit}" 2>/dev/null; then + git diff --name-only "$producer_sha" HEAD > "$plan_dir/carrier-changes" + if [[ -s "$plan_dir/carrier-changes" ]]; then + (unset MOON_BASE MOON_HEAD; "$moon_bin" query affected stdin --upstream none --downstream deep < "$plan_dir/carrier-changes") > "$plan_dir/carrier-affected.json" + else + printf '{"tasks":{}}\n' > "$plan_dir/carrier-affected.json" + fi + if bun -e 'const data=await Bun.file(process.argv[1]).json(); process.exit(Object.hasOwn(data.tasks ?? {}, "liboliphaunt-native:finalize-runtime-ios-abi") ? 1 : 0)' "$plan_dir/carrier-affected.json" && + bun src/native/sdks/swift/tools/ios-carrier-manifest.mts --base-carrier "$carrier_cache/manifest.json" --output "$plan_dir/carrier.json"; then + export OLIPHAUNT_REUSE_IOS_CARRIER=true + fi + fi +fi "$moon_bin" task-graph --json >"$OLIPHAUNT_MOON_TASK_GRAPH_FILE" if [[ $# == 0 && ${GITHUB_EVENT_NAME:-} != workflow_dispatch && (-z ${CI_RELEASE_PRODUCTS_JSON:-} || ${CI_RELEASE_PRODUCTS_JSON:-} == '[]') ]]; then : "${MOON_BASE:?MOON_BASE is required for affected CI planning}" diff --git a/tools/ci/ci-release-scope.test.sh b/tools/ci/ci-release-scope.test.sh index 972dfe7d3..ed75a22b0 100644 --- a/tools/ci/ci-release-scope.test.sh +++ b/tools/ci/ci-release-scope.test.sh @@ -25,7 +25,18 @@ set -eu case "$1" in --version) echo "moon $FIXTURE_MOON_VERSION" ;; task-graph) cat "$FIXTURE_TASK_GRAPH" ;; - query) [[ "$2" == projects ]]; cat "$FIXTURE_PROJECTS" ;; + query) + if [[ "$2" == projects ]]; then cat "$FIXTURE_PROJECTS"; + elif [[ ${FIXTURE_AFFECTED:-false} == true && "$2" == affected ]]; then + if [[ ${3:-} == stdin ]]; then + [[ -n "$(cat)" ]] || { echo 'empty stdin must not be queried as a Git ref' >&2; exit 82; } + if [[ ${FIXTURE_IOS_CHANGED:-false} == true ]]; then + printf '{"tasks":{"liboliphaunt-native:finalize-runtime-ios-abi":{}}}\n' + else printf '{"tasks":{}}\n'; fi + else + printf '{"projects":{"integration-examples":{"other":true}},"tasks":{"integration-examples:react-native-android-build":{"other":true}}}\n' + fi + else echo 'unexpected affected query' >&2; exit 82; fi ;; *) echo 'unexpected affected query' >&2; exit 82 ;; esac SH @@ -56,3 +67,44 @@ if bash "$root/tools/ci/ci-plan.sh" > "$scratch/invalid.out" 2> "$scratch/invali echo 'unknown release owner was accepted' >&2; exit 1 fi bash "$root/tools/dev/bun.sh" "$root/tools/ci/ci-release-scope.test.mts" assert "$scratch" "$head" + +# A missing or unusable optimization cache must keep the normal producer path. +# Same-SHA reuse has an empty diff, which Moon otherwise interprets as a ref. +git commit --allow-empty -qm 'fix: Android app' +export MOON_BASE="$head" GITHUB_REF=refs/heads/fixture GITHUB_EVENT_NAME=pull_request +MOON_HEAD="$(git rev-parse HEAD)"; export MOON_HEAD +export FIXTURE_AFFECTED=true +ln -s "$root/src" "$repo/src" +mkdir -p target/ci/ios-carrier +bun - "$root" "$scratch/carrier.json" <<'JS' +import {writeFileSync, readFileSync} from 'node:fs'; +const root = process.argv[2]; +const {iosBaseLegalMetadata} = await import(`${root}/src/native/sdks/swift/tools/ios-carrier-manifest.mts`); +const version = readFileSync(`${root}/src/native/runtime/VERSION`, 'utf8').trim(); +const tag = `liboliphaunt-native-v${version}`; +const assets = [ + ['base-xcframework', `liboliphaunt-${version}-apple-spm-xcframework.zip`, 'zip', 'liboliphaunt.xcframework'], + ['runtime-resources', `liboliphaunt-${version}-runtime-resources-ios-datum64.tar.gz`, 'tar.gz', 'oliphaunt'], +].map(([role, name, format, member]) => ({role, name, format, member, bytes:1, sha256:'a'.repeat(64), + url:`https://github.com/f0rr0/oliphaunt/releases/download/${tag}/${name}`})); +writeFileSync(process.argv[3], JSON.stringify({schema:'oliphaunt-react-native-ios-carrier-v1', + base:{product:'liboliphaunt-native', version, tag, assets}, carriers:[], extensions:[], + legal:{base:iosBaseLegalMetadata(), extensions:[]}})); +JS +for scenario in missing same-sha invalid-sha corrupt unchanged-ios changed-ios; do + expected=false + cp "$scratch/carrier.json" target/ci/ios-carrier/manifest.json + printf '%s\n' "$MOON_HEAD" > target/ci/ios-carrier/source-sha + export FIXTURE_IOS_CHANGED=false + case "$scenario" in + missing) rm target/ci/ios-carrier/manifest.json ;; + same-sha) expected=true ;; + invalid-sha) printf 'invalid\n' > target/ci/ios-carrier/source-sha ;; + corrupt) printf '{invalid\n' > target/ci/ios-carrier/manifest.json ;; + unchanged-ios) printf '%s\n' "$before" > target/ci/ios-carrier/source-sha; expected=true ;; + changed-ios) printf '%s\n' "$before" > target/ci/ios-carrier/source-sha; export FIXTURE_IOS_CHANGED=true ;; + esac + bash "$root/tools/ci/ci-plan.sh" > "$scratch/$scenario.out" 2> "$scratch/$scenario.err" + bun -e 'import assert from "node:assert/strict"; const output=await Bun.file(process.argv[2]).text(); assert(output.includes(`reuse_ios_carrier=${process.argv[1]}`)); const jobs=JSON.parse(output.match(/^jobs=(.+)$/m)[1]); assert.equal(jobs.includes("liboliphaunt-native-ios-abi"), process.argv[1] !== "true")' "$expected" "$scratch/$scenario.out" +done +echo 'Frozen iOS metadata: same-SHA reuse and cold, corrupt, changed-input fallbacks passed' diff --git a/tools/ci/ci_plan.mts b/tools/ci/ci_plan.mts index 31c62ddfb..992bc58ac 100644 --- a/tools/ci/ci_plan.mts +++ b/tools/ci/ci_plan.mts @@ -7,6 +7,11 @@ // targets are CI execution details, not source projects. import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; +import { qualificationRequestKey } from '../../.github/scripts/release-candidate-lib.mts'; +import { + publishedConsumerInventory, + validPublishedConsumerInventory, +} from '../../src/native/sdks/ts/tools/published-consumer.mts'; import { brokerRuntimeMatrix, extensionArtifactsNativeMatrix, @@ -29,13 +34,8 @@ import { extensionSqlNames, extensionSqlNamesForProducts, } from '../release/release-artifact-targets.mts'; -import { affectedNames, triggeringProjectNames, triggeringTaskNames } from './affected.mts'; import { loadProducts, moonProjectsById } from '../release/release-graph.mts'; -import { qualificationRequestKey } from '../../.github/scripts/release-candidate-lib.mts'; -import { - publishedConsumerInventory, - validPublishedConsumerInventory, -} from '../../src/native/sdks/ts/tools/published-consumer.mts'; +import { affectedNames, triggeringProjectNames, triggeringTaskNames } from './affected.mts'; const ROOT = path.resolve(import.meta.dir, '../..'); const PREFIX = 'ci_plan.mts'; @@ -166,7 +166,9 @@ const RELEASE_ONLY_TARGETS = new Set( .map((task) => task.target), ); export const BUILDER_JOBS = new Set( - Object.keys(CI_JOB_TARGETS).filter((job) => job !== NATIVE_EXTENSION_LIFECYCLE_JOB), + Object.keys(CI_JOB_TARGETS).filter( + (job) => ![NATIVE_EXTENSION_LIFECYCLE_JOB, 'wasix-release-regression'].includes(job), + ), ); const JOBS_BY_TARGET = (() => { const jobs = new Map(); @@ -179,6 +181,7 @@ const DEPENDENTS_BY_TARGET = (() => { const dependents = new Map(); for (const task of TASKS_BY_TARGET.values()) { for (const dependency of task.deps ?? []) { + if (dependency.cacheStrategy === 'ignored') continue; const target = typeof dependency === 'string' ? dependency : dependency.target; if (typeof target === 'string') { dependents.set(target, [...(dependents.get(target) ?? []), task.target]); @@ -272,18 +275,29 @@ export function addRequiredJobs(jobs) { return jobs; } -export function planJobsForAffected(tasks, excludedTargets = RELEASE_ONLY_TARGETS) { +export function planJobsForAffected( + tasks, + excludedTargets = RELEASE_ONLY_TARGETS, + reuseIosCarrier = process.env.OLIPHAUNT_REUSE_IOS_CARRIER === 'true', +) { const jobs = new Set(ALWAYS_JOBS); - const directlySelectedJobs = jobsForTargets(requiredTasksForAffected(tasks, excludedTargets), { - allowedJobs: ALL_BUILDER_JOBS, - }); + const directlySelectedJobs = jobsForTargets( + requiredTasksForAffected(tasks, excludedTargets, reuseIosCarrier), + { + allowedJobs: ALL_BUILDER_JOBS, + }, + ); for (const job of directlySelectedJobs) { jobs.add(job); } return jobs; } -export function requiredTasksForAffected(tasks, excludedTargets = RELEASE_ONLY_TARGETS) { +export function requiredTasksForAffected( + tasks, + excludedTargets = RELEASE_ONLY_TARGETS, + reuseIosCarrier = process.env.OLIPHAUNT_REUSE_IOS_CARRIER === 'true', +) { const selected = new Set( [...downstreamTaskClosure(tasks, excludedTargets)].filter((target) => JOBS_BY_TARGET.has(target), @@ -294,16 +308,18 @@ export function requiredTasksForAffected(tasks, excludedTargets = RELEASE_ONLY_T const target = pending.pop(); const task = TASKS_BY_TARGET.get(target); if (!task) fail(`affected Moon selection references missing target ${target}`); - const dependencies = taskDependencyTargets(task); - // Selecting the portable WASIX job also requires full same-run lifecycle - // evidence. Include every producer downloaded by wasix-release-regression. + const dependencies = taskDependencyTargets(task).filter( + (dependency) => + !( + reuseIosCarrier && + target === 'oliphaunt-react-native:package' && + dependency === 'liboliphaunt-native:finalize-runtime-ios-abi' + ), + ); + // Every planned portable runtime requires fresh lifecycle qualification. + // Its owner task declares the producer closure; do not repeat it here. if (task.tags?.includes('ci-liboliphaunt-wasix-runtime')) { - dependencies.push( - 'liboliphaunt-wasix:runtime-aot', - 'extension-artifacts-wasix:build-target', - 'extension-artifacts-wasix:build-aot', - 'postgres-tools-wasix:build-aot', - ); + dependencies.push(...CI_JOB_TARGETS['wasix-release-regression']); } for (const dependency of dependencies) { if (!selected.has(dependency)) { @@ -514,7 +530,7 @@ export function planForReleaseProducts( excludedTargets.delete('oliphaunt-js:test-consumer-published'); roots.add('oliphaunt-js:test-consumer-published'); } - const tasks = requiredTasksForAffected(roots, excludedTargets); + const tasks = requiredTasksForAffected(roots, excludedTargets, false); for (const target of downstreamTaskClosure(roots, excludedTargets)) { const task = TASKS_BY_TARGET.get(target); if ( @@ -533,7 +549,7 @@ export function planForReleaseProducts( } } } - const jobs = planJobsForAffected(roots, excludedTargets); + const jobs = planJobsForAffected(roots, excludedTargets, false); const selectedExtensionProducts = selectedExtensionProductsForPlan(projects, roots, jobs); const plan = renderPlanWithSelection({ jobs, @@ -700,8 +716,8 @@ export function planForFullRun({ new Set(['liboliphaunt-wasix-runtime', 'liboliphaunt-wasix-aot']), ); if (wasmTarget === 'linux-x64-gnu') { - // The workflow selects release regression for the Linux host target. - focusedJobs.add('extension-artifacts-wasix'); + focusedJobs.add('wasix-release-regression'); + addRequiredJobs(focusedJobs); } return { jobs: focusedJobs, @@ -716,7 +732,7 @@ export function planForFullRun({ BASE_JOBS, BUILDER_JOBS, WASM_RUNTIME_JOBS, - new Set([NATIVE_EXTENSION_LIFECYCLE_JOB]), + new Set([NATIVE_EXTENSION_LIFECYCLE_JOB, 'wasix-release-regression']), ); addRequiredJobs(jobs); return { @@ -791,9 +807,8 @@ export function extensionArtifactsNativeMatrixForPlan( jobs.has('extension-packages') ? undefined : (selectedTargets ?? undefined), selectedExtensionProducts ?? undefined, ); - if (!jobs.has(NATIVE_EXTENSION_LIFECYCLE_JOB)) { - return matrix; - } + const android = matrix.include.some((row) => row.target.startsWith('android-')); + if (!jobs.has(NATIVE_EXTENSION_LIFECYCLE_JOB) && !android) return matrix; const exactProducts = new Set(exactExtensionProducts()); const requiredTargets = new Set(['linux-x64-gnu']); @@ -897,6 +912,11 @@ export function renderPlanWithSelection({ jobs: sorted(jobs), builder_jobs: sorted(new Set([...jobs].filter((job) => BUILDER_JOBS.has(job)))), e2e_jobs: mobileE2eJobsForPlan(jobs), + reuse_ios_carrier: + qualificationMode === AFFECTED_QUALIFICATION_MODE && + process.env.OLIPHAUNT_REUSE_IOS_CARRIER === 'true' && + jobs.has('react-native-sdk-package') && + !jobs.has('liboliphaunt-native-ios-abi'), job_targets: jobTargetsForJobs( jobs, qualificationMode === PRODUCT_QUALIFICATION_MODE @@ -997,12 +1017,13 @@ export function renderPlanWithSelection({ ['extension_artifacts_native_matrix', ['linux', 'android', 'ios', 'other']], ['liboliphaunt_native_desktop_runtime_matrix', ['linux', 'other']], ['broker_runtime_matrix', ['linux', 'other']], + ['liboliphaunt_wasix_aot_runtime_matrix', ['linux', 'other']], ]) { for (const group of groups) { plan[`${matrix}_${group}`] = { include: plan[matrix].include.filter((row) => { const family = - row.target === 'linux-x64-gnu' + (row.target_id ?? row.target) === 'linux-x64-gnu' ? 'linux' : groups.includes('android') && row.target.startsWith('android-') ? 'android' @@ -1015,6 +1036,8 @@ export function renderPlanWithSelection({ } } + plan.wasix_napi_targets = plan.wasix_napi_runtime_matrix.include.map((row) => row.target); + for (const family of ['android', 'ios']) { plan[`mobile_extension_package_native_targets_${family}_csv`] = plan.mobile_extension_package_native_targets diff --git a/tools/ci/moon.yml b/tools/ci/moon.yml index 2a3a7d10f..ad7d22997 100644 --- a/tools/ci/moon.yml +++ b/tools/ci/moon.yml @@ -11,6 +11,6 @@ tasks: test: tags: ["quality", "unit"] command: "bash tools/ci/start-android-emulator-ci.test.sh" - inputs: ["start-android-emulator-ci.sh", "start-android-emulator-ci.test.sh"] + inputs: ["start-android-emulator-ci.sh", "start-android-emulator-ci.test.sh", "/tools/dev/acquisition.sh"] options: runFromWorkspaceRoot: true diff --git a/tools/ci/start-android-emulator-ci.sh b/tools/ci/start-android-emulator-ci.sh index cdb84dc38..2d8cbcaa7 100755 --- a/tools/ci/start-android-emulator-ci.sh +++ b/tools/ci/start-android-emulator-ci.sh @@ -58,8 +58,11 @@ need_cmd avdmanager need_cmd adb ensure_kvm_access -yes | sdkmanager --licenses >/dev/null || true -sdkmanager --install "emulator" "$image" +# Package acquisition and emulator boot have separate budgets. +. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../dev" && pwd)/acquisition.sh" +oliphaunt_acquisition_start 'Android emulator packages' 1800 +yes | oliphaunt_acquisition_run 60 sdkmanager --licenses >/dev/null || true +oliphaunt_acquisition_run 1800 sdkmanager --install "emulator" "$image" need_cmd emulator mkdir -p "$ANDROID_AVD_HOME" diff --git a/tools/ci/workflow-caches.test.mts b/tools/ci/workflow-caches.test.mts new file mode 100644 index 000000000..76f444bfa --- /dev/null +++ b/tools/ci/workflow-caches.test.mts @@ -0,0 +1,155 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(import.meta.dir, '../..'); +const yaml = (file) => Bun.YAML.parse(readFileSync(path.join(root, file), 'utf8')); +const workflow = yaml('.github/workflows/ci.yml'); +const common = 'source src/wasix/postmaster/lib/common.sh\n'; + +test('the shared builder label satisfies Postmaster without a second Docker build', () => { + const scratch = mkdtempSync(path.join(tmpdir(), 'oliphaunt-builder-cache-')); + try { + const action = yaml('.github/actions/setup-wasix-builder/action.yml'); + const output = path.join(scratch, 'output'); + execFileSync('bash', ['-eu', '-c', action.runs.steps[0].run], { + cwd: root, + env: { ...process.env, GITHUB_OUTPUT: output }, + }); + const recipe = readFileSync(output, 'utf8').trim().split('=')[1]; + assert.match(recipe, /^[a-f0-9]{64}$/u); + const build = action.runs.steps.find(({ uses }) => + uses?.startsWith('docker/build-push-action@'), + ); + const label = build.with.labels.replace('${{ steps.recipe.outputs.sha256 }}', recipe); + const docker = path.join(scratch, 'docker'); + // A cache hit may inspect the image, but must not invoke Docker build. + writeFileSync( + docker, + `#!/usr/bin/env bash +set -eu +test "$1 $2" = 'image inspect' +test "$4" = '{{ index .Config.Labels "dev.oliphaunt.wasix-builder.recipe-sha256" }}' +test "$5" = "$EXPECTED_IMAGE" +test "\${BUILDER_LABEL%%=*}" = dev.oliphaunt.wasix-builder.recipe-sha256 +printf '%s\\n' "\${BUILDER_LABEL#*=}" +`, + { mode: 0o755 }, + ); + const image = execFileSync( + 'bash', + ['-eu', '-c', `${common}printf '%s' "$FRESH_WASIX_DOCKER_IMAGE"`], + { + cwd: root, + encoding: 'utf8', + }, + ); + assert.ok(build.with.tags.trim().split('\n').includes(image)); + execFileSync( + 'bash', + [ + '-eu', + '-c', + `${common}fresh_docker_bin() { printf '%s' "$FAKE_DOCKER"; }; fresh_ensure_docker_image`, + ], + { + cwd: root, + env: { ...process.env, FAKE_DOCKER: docker, BUILDER_LABEL: label, EXPECTED_IMAGE: image }, + }, + ); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +}); + +test('Postmaster caches Cargo target directories without cleaning their shared parent', () => { + const targets = execFileSync( + 'bash', + [ + '-eu', + '-c', + `${common}printf '%s\\n' "$FRESH_WORK_ROOT/runtime/wasmer/target" "$FRESH_POSTMASTER_EXECUTOR_TARGET_DIR" "$FRESH_POSTMASTER_COMPILER_TARGET_DIR"`, + ], + { cwd: root, encoding: 'utf8' }, + ) + .trim() + .split('\n'); + for (const id of ['wasix-postmaster-portable', 'wasix-postmaster-target']) { + const steps = workflow.jobs[id].steps; + const rustIndex = steps.findIndex(({ uses }) => uses === './.github/actions/setup-rust'); + const mappings = steps[rustIndex].with['cache-workspaces'] + .trim() + .split('\n') + .map((line) => { + const [workspace, target] = line.split('->').map((part) => part.trim()); + return { + workspace: path.resolve(root, workspace), + target: path.resolve(root, workspace, target), + }; + }); + for (const target of targets) + assert.ok( + mappings.some((mapping) => mapping.target === target), + `${id}: missing ${target}`, + ); + for (const a of mappings) { + for (const b of mappings) { + if (a !== b) + assert.ok( + !b.target.startsWith(`${a.target}${path.sep}`), + 'a parent Cargo cleanup would delete the nested cache', + ); + } + } + for (const { workspace } of mappings) + assert.ok(readFileSync(path.join(workspace, 'Cargo.toml'), 'utf8')); + } +}); + +test('Cargo cache metadata needs no generated Wasmer checkout at restore time', () => { + const scratch = mkdtempSync(path.join(tmpdir(), 'oliphaunt-cache-workspace-')); + try { + const executor = path.join(root, 'src/wasix/postmaster/executor'); + for (const entry of ['Cargo.toml', 'Cargo.lock', 'src']) + cpSync(path.join(executor, entry), path.join(scratch, entry), { recursive: true }); + const lock = readFileSync(path.join(scratch, 'Cargo.lock'), 'utf8'); + // rust-cache collects workspace members without dependency resolution on restore. + const metadata = JSON.parse( + execFileSync('cargo', ['metadata', '--all-features', '--no-deps', '--format-version', '1'], { + cwd: scratch, + encoding: 'utf8', + }), + ); + assert.equal(metadata.workspace_members.length, 1); + assert.equal(readFileSync(path.join(scratch, 'Cargo.lock'), 'utf8'), lock); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +}); + +test('Rust setup follows the repository pin and rejects an unpinned override', () => { + const scratch = mkdtempSync(path.join(tmpdir(), 'oliphaunt-rust-pin-')); + try { + const step = yaml('.github/actions/setup-rust-tools/action.yml').runs.steps.find( + ({ id }) => id === 'toolchain', + ); + const output = path.join(scratch, 'output'); + writeFileSync(path.join(scratch, 'rust-toolchain.toml'), '[toolchain]\nchannel = "9.8.7"\n'); + const run = (version) => + execFileSync('bash', ['-eu', '-c', step.run], { + cwd: scratch, + env: { ...process.env, TOOLCHAIN_INPUT: version, GITHUB_OUTPUT: output }, + stdio: 'pipe', + }); + run(''); + assert.equal(readFileSync(output, 'utf8'), 'version=9.8.7\n'); + run('1.99.0'); + assert.match(readFileSync(output, 'utf8'), /version=1[.]99[.]0/u); + assert.throws(() => run('stable'), /expected a pinned Rust version/u); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +}); diff --git a/tools/ci/workflow-moon-transfers.test.mts b/tools/ci/workflow-moon-transfers.test.mts index 742dd29fe..b80c7a170 100644 --- a/tools/ci/workflow-moon-transfers.test.mts +++ b/tools/ci/workflow-moon-transfers.test.mts @@ -13,8 +13,8 @@ import { } from 'node:fs'; import path from 'node:path'; import test from 'node:test'; -import { CI_JOB_TARGETS } from './ci_plan.mts'; import { resolveExecution } from '../../.github/scripts/resolve-planned-moon-execution.mts'; +import { CI_JOB_TARGETS } from './ci_plan.mts'; const ROOT = path.resolve(import.meta.dir, '../..'); const workflow = Bun.YAML.parse(readFileSync(path.join(ROOT, '.github/workflows/ci.yml'), 'utf8')); @@ -27,6 +27,51 @@ const tasks = new Map( ]), ); +if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) + test('jobs without a Moon cache have no cacheable local task subtree', () => { + for (const [id, job] of Object.entries(workflow.jobs)) { + if ( + !job.steps?.some( + (step) => + step.uses === './.github/actions/setup-moon' && step.with?.['task-cache'] === 'false', + ) + ) + continue; + const roots = CI_JOB_TARGETS[id] ?? []; + if (!roots.length) { + assert( + !job.steps.some((step) => + /run-(?:planned-moon-job|moon-targets)[.]sh/u.test(step.run ?? ''), + ), + `${id} executes unclassified Moon work`, + ); + continue; + } + const transferred = [ + ...new Set( + job.steps.flatMap((step) => + JSON.parse(step.env?.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON ?? '[]'), + ), + ), + ]; + const execution = resolveExecution(roots, transferred, tasks); + assert.deepEqual( + execution.localDependencies, + [], + `${id} has local work that could reuse the cache`, + ); + // The adapter always executes paths consuming transferred artifacts with + // MOON_CACHE=off; only localDependencies use normal Moon caching. + assert.ok(execution.transferred.length > 0, `${id} runs without an artifact boundary`); + } + const action = Bun.YAML.parse( + readFileSync(path.join(ROOT, '.github/actions/setup-moon/action.yml'), 'utf8'), + ); + assert.equal(action.inputs['task-cache'].default, 'true'); + const cache = action.runs.steps.find((step) => step.uses?.startsWith('actions/cache@')); + assert.equal(cache.if, `\${{ inputs.task-cache == 'true' }}`); + }); + if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) test('artifact production waits for source check and test results without a dependency cycle', () => { const jobs = workflow.jobs; @@ -102,6 +147,34 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) assert.deepEqual(execution.targets, ['extension-packages:package']); }); +if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) + test('WASIX consumers build query once before running against transferred SDK artifacts', () => { + const step = workflow.jobs['wasix-ts-sdk-package'].steps.find( + (step) => step.name === 'Test WASIX TypeScript consumers', + ); + const execution = resolveExecution( + CI_JOB_TARGETS['wasix-ts-sdk-package'], + JSON.parse(step.env.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON), + tasks, + ); + // SDK transfers cut off their upstream query build, so every independently + // selectable consumer must retain its own edge to the shared producer. + for (const target of CI_JOB_TARGETS['wasix-ts-sdk-package']) { + assert(dependencies(target).includes('oliphaunt-query-ts:build'), target); + } + assert.deepEqual(execution.localDependencies, ['oliphaunt-query-ts:build']); + for (const target of [...execution.localDependencies, ...execution.targets]) { + assert( + ![ + 'wasix-browser-host:build', + 'oliphaunt-wasix-ts:build', + 'oliphaunt-wasix-ts:package', + ].includes(target), + target, + ); + } + }); + function dependencies(target) { const task = tasks.get(target); assert.ok(task, `workflow root ${target} must exist in Moon`); @@ -119,7 +192,13 @@ function dependencies(target) { if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) test('downloaded Moon dependencies are explicit reachable handoffs', () => { - for (const [workflowJob, job] of Object.entries(workflow.jobs)) { + const host = Bun.YAML.parse( + readFileSync(path.join(ROOT, '.github/workflows/wasix-host.yml'), 'utf8'), + ); + for (const [workflowJob, job] of Object.entries({ + ...workflow.jobs, + 'wasix-host': { ...host.jobs.build, needs: ['liboliphaunt-wasix-runtime'] }, + })) { const steps = job.steps ?? []; for (const [index, step] of steps.entries()) { const run = String(step.run ?? ''); @@ -130,7 +209,9 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) const rawTransfers = step.env?.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON; if (rawTransfers === undefined) continue; - const plannedJob = run.match(/run-planned-moon-job[.]sh ([a-z0-9-]+)/u)?.[1]; + const plannedJob = + run.match(/run-planned-moon-job[.]sh ([a-z0-9-]+)/u)?.[1] ?? + (run.includes('collect-wasix-evidence.sh') ? 'wasix-release-regression' : undefined); assert.ok(plannedJob, `${workflowJob} transferred handoff must use the planned-job runner`); const transfers = JSON.parse(rawTransfers); assert.ok(Array.isArray(transfers) && transfers.length > 0); @@ -180,6 +261,59 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) } }); +if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) + test('SDK runtime suites execute in artifact-consuming jobs without rebuilding their producers', () => { + for (const [job, roots, forbidden] of [ + [ + 'native-consumers', + [ + 'oliphaunt-rust:test-integration', + 'oliphaunt-mobile-bindings:test-native', + 'oliphaunt-swift:test-native', + 'oliphaunt-kotlin:test-native-bindings', + ], + [ + 'liboliphaunt-native:build-runtime-desktop-target', + 'oliphaunt-broker:build-release-assets', + ], + ], + [ + 'wasix-release-regression', + ['oliphaunt-wasix-rust:test-integration'], + [ + 'liboliphaunt-wasix:compiler-output', + 'liboliphaunt-wasix:runtime-aot', + 'database-resources:build-icu-data', + ], + ], + ]) { + const step = workflow.jobs[job].steps.find( + (step) => step.env?.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON, + ); + const available = JSON.parse(step.env.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON); + for (const root of roots) { + assert(CI_JOB_TARGETS[job].includes(root), `${root} has no executable hosted owner`); + const reachable = new Set(dependencies(root)); + for (const dependency of reachable) + for (const parent of dependencies(dependency)) reachable.add(parent); + const execution = resolveExecution( + [root], + available.filter((target) => reachable.has(target)), + tasks, + ); + assert(execution.targets.includes(root)); + const executed = [...execution.targets, ...execution.localDependencies]; + for (const target of forbidden) + assert(!executed.includes(target), `${root} rebuilds ${target}`); + assert.equal( + tasks.get(root).options.cache, + false, + `${root} must execute against this run's artifacts`, + ); + } + } + }); + const phase = process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE; const scratch = process.argv[2]; if (phase) { diff --git a/tools/dev/acquisition.sh b/tools/dev/acquisition.sh new file mode 100644 index 000000000..1e9e02c1b --- /dev/null +++ b/tools/dev/acquisition.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env sh +# One budget per acquisition, shared by retries, mirrors and lock waits. +# POSIX shell and curl suffice for bootstrap downloads; other subprocesses use +# GNU timeout (already required for source Git operations). + +oliphaunt_acquisition_start() { + oliphaunt_acquisition_label=$1 + oliphaunt_acquisition_seconds=${OLIPHAUNT_ACQUISITION_TIMEOUT_SECONDS:-$2} + case "$oliphaunt_acquisition_seconds" in + ''|0*|*[!0-9]*|?????*) echo 'acquisition timeout must be an integer from 1 to 7200 seconds' >&2; return 2 ;; + esac + if [ "$oliphaunt_acquisition_seconds" -lt 1 ] || [ "$oliphaunt_acquisition_seconds" -gt 7200 ]; then + echo 'acquisition timeout must be an integer from 1 to 7200 seconds' >&2 + return 2 + fi + oliphaunt_acquisition_end=$(( $(date +%s) + oliphaunt_acquisition_seconds )) + if [ -n "${oliphaunt_acquisition_deadline:-}" ] && + [ "$oliphaunt_acquisition_deadline" -lt "$oliphaunt_acquisition_end" ]; then + oliphaunt_acquisition_end=$oliphaunt_acquisition_deadline + fi + oliphaunt_acquisition_deadline=$oliphaunt_acquisition_end +} + +oliphaunt_acquisition_remaining() ( + remaining=$(( ${oliphaunt_acquisition_deadline:?start an acquisition first} - $(date +%s) )) + if [ "$remaining" -le 0 ]; then + echo "acquisition deadline exhausted: $oliphaunt_acquisition_label" >&2 + exit 124 + fi + [ "$remaining" -le "$1" ] || remaining=$1 + printf '%s\n' "$remaining" +) + +oliphaunt_acquisition_sleep() ( + remaining=$(oliphaunt_acquisition_remaining 7200) || exit $? + if [ "$1" -ge "$remaining" ]; then + echo "acquisition retry would exceed deadline: $oliphaunt_acquisition_label" >&2 + exit 124 + fi + sleep "$1" +) + +# Git for Windows can have System32/timeout.exe ahead of GNU timeout in PATH. +# Prefer Coreutils and fall back to the shell's bundled /usr/bin copy. +oliphaunt_acquisition_timeout() ( + for candidate in gtimeout timeout /usr/bin/timeout; do + timer=$(command -v "$candidate") || continue + case "$("$timer" --version 2>/dev/null)" in + *'GNU coreutils'*) printf '%s\n' "$timer"; exit 0 ;; + esac + done + echo 'acquisition requires GNU timeout (brew install coreutils on macOS; use Git Bash on Windows)' >&2 + exit 127 +) + +oliphaunt_acquisition_run() ( + timer=$(oliphaunt_acquisition_timeout) || exit $? + seconds=$(oliphaunt_acquisition_remaining "$1") || exit $? + shift + status=0 + "$timer" --kill-after=5 "${seconds}s" "$@" || status=$? + if [ "$status" = 124 ] || [ "$status" = 137 ]; then + echo "acquisition command timed out after ${seconds}s: $oliphaunt_acquisition_label" >&2 + fi + exit "$status" +) + +# CAP ATTEMPTS DELAY CURL ARGS...; CAP bounds this endpoint, not each retry. +# curl's retry-max-time allows a final transfer to overrun its timer. Run single +# attempts with the remaining time instead, without needing a bootstrap runtime. +oliphaunt_acquisition_curl() ( + cap=$1 attempts=$2 delay=$3 curl_command=$4 + shift 4 + endpoint_deadline=$(( $(date +%s) + cap )) + if [ "$endpoint_deadline" -lt "$oliphaunt_acquisition_deadline" ]; then + oliphaunt_acquisition_deadline=$endpoint_deadline + fi + attempt=1 status=0 + while [ "$attempt" -le "$attempts" ]; do + seconds=$(oliphaunt_acquisition_remaining "$cap") || exit $? + if "$curl_command" --disable --retry 0 --max-time "$seconds" "$@"; then + exit 0 + else + status=$? + fi + # Cancellation is not a transient transport failure. + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac + oliphaunt_acquisition_remaining "$cap" >/dev/null || exit $? + [ "$attempt" -lt "$attempts" ] || break + oliphaunt_acquisition_sleep "$delay" || exit $? + attempt=$((attempt + 1)) + done + exit "$status" +) diff --git a/tools/dev/acquisition.test.sh b/tools/dev/acquisition.test.sh new file mode 100644 index 000000000..012b9dc7a --- /dev/null +++ b/tools/dev/acquisition.test.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(git rev-parse --show-toplevel)" +. "$root/tools/dev/acquisition.sh" +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT + +# A fake clock exercises shared budgets without waiting through retry delays. +printf '100\n' > "$scratch/clock" +date() { cat "$scratch/clock"; } +sleep() { printf '%s\n' "$(( $(date +%s) + $1 ))" > "$scratch/clock"; } +unset OLIPHAUNT_ACQUISITION_TIMEOUT_SECONDS oliphaunt_acquisition_deadline +oliphaunt_acquisition_start fixture 20 +for invalid in '' 0 -1 08 abc 7201 99999999999999999999; do + if OLIPHAUNT_ACQUISITION_TIMEOUT_SECONDS="$invalid" oliphaunt_acquisition_start invalid 20 2>/dev/null; then + # An empty override deliberately selects the caller's default. + [ -z "$invalid" ] || exit 1 + fi +done +oliphaunt_acquisition_start fixture 20 +curl_attempt() { + local seconds="" argument + while [ "$#" -gt 0 ]; do + argument=$1; shift + case "$argument" in + --max-time) seconds=$1; shift ;; + --retry) [ "$1" = 0 ]; shift ;; + esac + done + printf '%s\n' "$seconds" >> "$scratch/attempts" + # Model an attempt that consumes seven seconds then loses its connection. + sleep 7 + return 7 +} +status=0 +oliphaunt_acquisition_curl 20 10 3 curl_attempt || status=$? +[ "$status" = 124 ] +printf '20\n10\n' > "$scratch/expected" +cmp "$scratch/expected" "$scratch/attempts" +[ "$(date +%s)" = 117 ] +# An interrupted transfer must not be retried. +curl_interrupted() { printf 'called\n' >> "$scratch/cancelled"; return 143; } +status=0 +oliphaunt_acquisition_curl 20 10 0 curl_interrupted || status=$? +[ "$status" = 143 ] && [ "$(wc -l < "$scratch/cancelled")" -eq 1 ] +# Starting a mirror/sub-operation cannot replenish its parent's remaining time. +oliphaunt_acquisition_start mirror 100 +[ "$(oliphaunt_acquisition_remaining 100)" = 3 ] +printf '121\n' > "$scratch/clock" +status=0 +oliphaunt_acquisition_run 60 touch "$scratch/late-command" || status=$? +[ "$status" = 124 ] && [ ! -e "$scratch/late-command" ] + +# Exhaustion on the final attempt is still a deadline, not a fallback-triggering +# transport error. Endpoint expiry leaves the parent's budget for another mirror. +printf '100\n' > "$scratch/clock" +unset oliphaunt_acquisition_deadline +oliphaunt_acquisition_start endpoint 20 +status=0 +oliphaunt_acquisition_curl 7 1 0 curl_attempt || status=$? +[ "$status" = 124 ] +[ "$(oliphaunt_acquisition_remaining 20)" = 13 ] + +# A non-GNU program named timeout (Windows System32) must not mask Coreutils. +# Intercept discovery so this also tests a Mac with only gtimeout installed. +real_timeout=$(oliphaunt_acquisition_timeout) +for available in gtimeout /usr/bin/timeout missing; do + command() { + [ "$1" = -v ] || return 2 + if [ "$2" = "$available" ]; then printf '%s\n' "$real_timeout" + elif [ "$2" = timeout ]; then printf '%s\n' false + else return 1; fi + } + if [ "$available" = missing ]; then + status=0 + oliphaunt_acquisition_timeout 2>/dev/null || status=$? + [ "$status" = 127 ] + else + [ "$(oliphaunt_acquisition_timeout)" = "$real_timeout" ] + fi + unset -f command +done + +# Real GNU timeout must terminate descendants, preserving ordinary exit codes. +unset -f date sleep +unset oliphaunt_acquisition_deadline +oliphaunt_acquisition_start process 30 +status=0 +oliphaunt_acquisition_run 10 sh -c 'exit 7' || status=$? +[ "$status" = 7 ] +status=0 +oliphaunt_acquisition_run 1 sh -c 'sleep 3; touch "$1"' sh "$scratch/orphan" || status=$? +[ "$status" = 124 ] +sleep 3 +[ ! -e "$scratch/orphan" ] +echo 'Acquisition deadlines: shared retries, nested budgets, expired admission and process cleanup passed' diff --git a/tools/dev/android-sdk.toml b/tools/dev/android-sdk.toml index ce8f41c36..3a5fea936 100644 --- a/tools/dev/android-sdk.toml +++ b/tools/dev/android-sdk.toml @@ -2,6 +2,8 @@ command_line_tools_build = "14742923" command_line_tools_revision = "20.0" ndk = "27.0.12077973" +# React Native / Expo app toolchain; native runtime artifacts retain their own NDK. +expo_ndk = "27.1.12297006" cmake = "3.22.1" compile_sdk = "36" build_tools = "36.0.0" diff --git a/tools/dev/extract-maestro.mts b/tools/dev/extract-maestro.mts deleted file mode 100644 index 50f6a4456..000000000 --- a/tools/dev/extract-maestro.mts +++ /dev/null @@ -1,27 +0,0 @@ -import { mkdirSync, writeFileSync } from 'node:fs'; -import path from 'node:path'; -import { readPortableArchiveEntries } from '../packaging/portable-archive.mts'; -const [archive, destination, version] = process.argv.slice(2); -try { - const entries = readPortableArchiveEntries(archive, { - maxArchiveBytes: 400_000_000, - maxExpandedBytes: 800_000_000, - maxEntries: 4096, - }); - for (const name of ['maestro/bin/maestro', 'maestro/lib/maestro-cli-' + version + '.jar']) { - if (!entries.get(name)?.isFile || !entries.get(name)?.size) - throw new Error('missing expected archive entry: ' + name); - } - for (const entry of entries.values()) { - if (entry.name !== 'maestro' && !entry.name.startsWith('maestro/')) - throw new Error('unsafe Maestro archive path: ' + entry.name); - } - for (const entry of entries.values()) { - const output = path.join(destination, entry.name); - mkdirSync(entry.isDirectory ? output : path.dirname(output), { recursive: true }); - if (entry.isFile) writeFileSync(output, entry.data(), { flag: 'wx', mode: 0o644 }); - } -} catch (error) { - console.error(`invalid Maestro archive: ${error.message}`); - process.exitCode = 1; -} diff --git a/tools/dev/install-pinned-js-runtime.sh b/tools/dev/install-pinned-js-runtime.sh index 3be389cf7..c90f4044c 100755 --- a/tools/dev/install-pinned-js-runtime.sh +++ b/tools/dev/install-pinned-js-runtime.sh @@ -48,6 +48,8 @@ if [ ! -f "$curl_platform_flags" ] || [ -L "$curl_platform_flags" ]; then fi # shellcheck source=tools/dev/curl-platform-flags.sh disable=SC1091 . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "$tool bootstrap" 300 manifest_value() { local section="$1" @@ -284,20 +286,22 @@ for candidate_url in "$url" ${mirror_url:+"$mirror_url"}; do curl_args=( --fail --location --silent --show-error --proto '=https' --proto-redir '=https' - --retry 6 --retry-all-errors --retry-max-time 120 - --connect-timeout 20 --max-time 180 --max-filesize 200000000 + --connect-timeout 20 --max-filesize 200000000 ) if [ -n "$curl_platform_tls_flag" ]; then curl_args+=("$curl_platform_tls_flag") fi curl_args+=(--remove-on-error --output "$archive" "$candidate_url") - if "${OLIPHAUNT_PINNED_TOOL_CURL:-curl}" "${curl_args[@]}"; then + if oliphaunt_acquisition_curl 120 7 2 "${OLIPHAUNT_PINNED_TOOL_CURL:-curl}" "${curl_args[@]}"; then actual_archive_sha256="$(sha256_file "$archive")" if [ "$actual_archive_sha256" = "$archive_sha256" ]; then downloaded=1 break fi echo "$tool archive checksum mismatch from $candidate_url; trying the next pinned origin" >&2 + else + status=$? + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac fi done [ "$downloaded" = "1" ] || fail "could not download the verified $tool $version $target archive" diff --git a/tools/dev/install-pinned-js-runtime.test.sh b/tools/dev/install-pinned-js-runtime.test.sh index a91a6839b..de0cbf5da 100755 --- a/tools/dev/install-pinned-js-runtime.test.sh +++ b/tools/dev/install-pinned-js-runtime.test.sh @@ -4,6 +4,10 @@ set -euo pipefail root="$(git rev-parse --show-toplevel)" installer="$root/tools/dev/install-pinned-js-runtime.sh" tmp="$(mktemp -d)" +mkdir -p "$tmp/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$tmp/no-delay/sleep" +chmod +x "$tmp/no-delay/sleep" +export PATH="$tmp/no-delay:$PATH" trap 'rm -rf "$tmp"' EXIT HUP INT TERM diff --git a/tools/dev/install-pinned-maintainer-tool.sh b/tools/dev/install-pinned-maintainer-tool.sh index ac899f21a..36b7ccacd 100755 --- a/tools/dev/install-pinned-maintainer-tool.sh +++ b/tools/dev/install-pinned-maintainer-tool.sh @@ -272,6 +272,8 @@ if cache_valid_release; then exit 0 fi +. "$(dirname "${BASH_SOURCE[0]}")/acquisition.sh" +oliphaunt_acquisition_start "$tool bootstrap" 180 curl_command="${OLIPHAUNT_MAINTAINER_TOOLS_CURL:-curl}" command -v "$curl_command" >/dev/null 2>&1 || { echo "missing required download command: $curl_command" >&2; exit 1; } command -v mktemp >/dev/null 2>&1 || { echo "missing required command: mktemp" >&2; exit 1; } @@ -282,17 +284,12 @@ archive="$temporary_root/archive.partial" extract_root="$temporary_root/extracted" mkdir -p "$extract_root" set +e -"$curl_command" \ +oliphaunt_acquisition_curl 180 5 2 "$curl_command" \ --fail \ --location \ --silent \ --show-error \ - --retry 4 \ - --retry-all-errors \ - --retry-delay 2 \ - --retry-max-time 120 \ --connect-timeout 20 \ - --max-time 180 \ --max-filesize "$max_archive_bytes" \ --proto '=https' \ --proto-redir '=https' \ diff --git a/tools/dev/install-pinned-winflexbison.sh b/tools/dev/install-pinned-winflexbison.sh index fe1c39460..eb04b41d4 100755 --- a/tools/dev/install-pinned-winflexbison.sh +++ b/tools/dev/install-pinned-winflexbison.sh @@ -28,6 +28,8 @@ esac fail "missing regular curl platform policy: $curl_platform_flags" # shellcheck source=tools/dev/curl-platform-flags.sh disable=SC1091 . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "winflexbison bootstrap" 180 manifest_value() { local section="$1" @@ -176,13 +178,12 @@ trap 'exit 143' TERM curl_args=( --fail --location --silent --show-error --proto '=https' --proto-redir '=https' - --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 120 - --connect-timeout 20 --max-time 180 --max-filesize 2000000 + --connect-timeout 20 --max-filesize 2000000 ) curl_platform_tls_flag="$(oliphaunt_curl_platform_tls_flag)" if [ -n "$curl_platform_tls_flag" ]; then curl_args+=("$curl_platform_tls_flag"); fi curl_args+=(--remove-on-error --output "$archive" "$url") -"$curl_command" "${curl_args[@]}" || fail "could not download pinned winflexbison $version" +oliphaunt_acquisition_curl 180 6 2 "$curl_command" "${curl_args[@]}" || fail "could not download pinned winflexbison $version" actual_bytes="$(wc -c <"$archive" | tr -d '[:space:]')" [ "$actual_bytes" = "$archive_bytes" ] || fail "winflexbison archive size mismatch: expected $archive_bytes, got $actual_bytes" diff --git a/tools/dev/install-pinned-winflexbison.test.sh b/tools/dev/install-pinned-winflexbison.test.sh index 7aef06433..3cd06fd0f 100755 --- a/tools/dev/install-pinned-winflexbison.test.sh +++ b/tools/dev/install-pinned-winflexbison.test.sh @@ -5,6 +5,10 @@ root="$(git rev-parse --show-toplevel)" installer="$root/tools/dev/install-pinned-winflexbison.sh" extractor="$root/tools/dev/extract-pinned-zip.sh" tmp="$(mktemp -d)" +mkdir -p "$tmp/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$tmp/no-delay/sleep" +chmod +x "$tmp/no-delay/sleep" +export PATH="$tmp/no-delay:$PATH" trap 'rm -rf "$tmp"' EXIT HUP INT TERM mkdir -p "$tmp/fixtures" "$tmp/config" "$tmp/bin" @@ -70,7 +74,7 @@ payload="$(run_installer)" [ -x "$payload/win_flex.exe" ] [ -x "$payload/win_bison.exe" ] [ -f "$payload/data/README.md" ] -grep -Fxq -- "--retry-all-errors" "$tmp/curl-args.log" +grep -Fxq -- "--retry" "$tmp/curl-args.log" grep -Fxq -- "=https" "$tmp/curl-args.log" # A complete verified cache is network-independent. diff --git a/tools/dev/maestro.toml b/tools/dev/maestro.toml deleted file mode 100644 index 320d6448a..000000000 --- a/tools/dev/maestro.toml +++ /dev/null @@ -1,15 +0,0 @@ -[toolchain] -maestro = "2.6.0" -install_url = "https://github.com/mobile-dev-inc/Maestro/releases/download/cli-2.6.0/maestro.zip" -sha256 = "80185105a5d7e227e3b3fbcf225f45b312508ea676a9fc8e1b1aa1cac8b9ff6e" -license = "Apache-2.0" -cloud_required = false - -[decision] -status = "accepted" -date = "2026-06-08" -scope = "react-native-installed-app-e2e" -runner = "open-source-maestro-cli" -reopen_only_if = "Replacing the mobile E2E implementation or proving an actual requirement cannot be met by the pinned open-source CLI." -stop_rule = "Do not re-check Maestro, Detox, Appium, EAS-only flows, or hosted mobile E2E providers during routine implementation/review work." -default_path = "free, public-checkout reproducible, GitHub-hosted emulator/simulator E2E" diff --git a/tools/dev/maintainer-tool-install.test.sh b/tools/dev/maintainer-tool-install.test.sh index 8d998f70e..7b14e7dae 100644 --- a/tools/dev/maintainer-tool-install.test.sh +++ b/tools/dev/maintainer-tool-install.test.sh @@ -3,6 +3,10 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." repo="$PWD" scratch=$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-maintainer-tools-XXXXXX") +mkdir -p "$scratch/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$scratch/no-delay/sleep" +chmod +x "$scratch/no-delay/sleep" +export PATH="$scratch/no-delay:$PATH" trap 'rm -rf "$scratch"' EXIT FAKE_REAL_MV="$(command -v mv)" export FAKE_REAL_MV diff --git a/tools/dev/moon.yml b/tools/dev/moon.yml index e693443aa..5d0ef0128 100644 --- a/tools/dev/moon.yml +++ b/tools/dev/moon.yml @@ -21,24 +21,23 @@ tasks: tags: - quality - unit - script: "set -e\nbash tools/dev/setup-android-sdk.test.sh\nbash tools/dev/setup-maestro.test.sh\n" + script: "set -e\nbash tools/dev/setup-android-sdk.test.sh\n" inputs: + - acquisition* - setup-android-sdk* - - setup-maestro* - - extract-maestro.mts - bun.sh - node-info.mts - extract-pinned-zip.* - /tools/packaging/testdata/zip-fixture.mts - /tools/packaging/portable-archive.mts - - "/tools/dev/{android-sdk,maestro}.toml" + - "/tools/dev/android-sdk.toml" options: runFromWorkspaceRoot: true test: tags: - quality - unit - script: "set -eu\nfor test in tools/dev/bun.test.sh tools/dev/extract-pinned-zip.test.sh tools/dev/install-pinned-js-runtime.test.sh tools/dev/install-pinned-winflexbison.test.sh .github/actions/setup-moon/install-pinned-node.test.sh .github/actions/setup-moon/install-pinned-toolchain.test.sh .github/actions/setup-npm-publisher/install.test.sh .github/scripts/setup-native-build-tools.test.sh; do\n bash \"$test\"\ndone\nbash tools/dev/maintainer-tool-install.test.sh\n" + script: "set -eu\nfor test in tools/dev/acquisition.test.sh tools/dev/bun.test.sh tools/dev/extract-pinned-zip.test.sh tools/dev/install-pinned-js-runtime.test.sh tools/dev/install-pinned-winflexbison.test.sh .github/actions/setup-moon/install-pinned-node.test.sh .github/actions/setup-moon/install-pinned-toolchain.test.sh .github/actions/setup-npm-publisher/install.test.sh .github/scripts/setup-native-build-tools.test.sh; do\n bash \"$test\"\ndone\nbash tools/dev/maintainer-tool-install.test.sh\n" inputs: - install-pinned* - install-actionlint.sh @@ -50,6 +49,7 @@ tasks: - bun.test.sh - node-info.mts - curl-platform-flags.sh + - acquisition* - /.prototools - /.moon/toolchains.yml - /.github/actions/setup-moon/** diff --git a/tools/dev/setup-android-sdk.sh b/tools/dev/setup-android-sdk.sh index e83ecec7e..1e47b68fd 100755 --- a/tools/dev/setup-android-sdk.sh +++ b/tools/dev/setup-android-sdk.sh @@ -22,6 +22,9 @@ Options: --ndk-version Must match the authoritative toolchain manifest. --cmake-version Must match the authoritative toolchain manifest. --compile-sdk Must match the authoritative toolchain manifest. + --native-tools Install NDK and CMake (default true). + --expo Also install the manifest-pinned Expo app NDK. + --expo-ndk-version Assert the generated app uses the pinned Expo NDK. -h, --help Show this help. The command-line-tools URLs and SHA-256 checksums are intentionally not @@ -32,6 +35,8 @@ EOF root="$(git rev-parse --show-toplevel 2>/dev/null)" || fail "must run inside the Oliphaunt git checkout" cd "$root" +. "$root/tools/dev/acquisition.sh" +oliphaunt_acquisition_start "Android SDK setup" 1800 manifest="${OLIPHAUNT_ANDROID_TOOLCHAIN_MANIFEST:-$root/tools/dev/android-sdk.toml}" extractor="${OLIPHAUNT_ANDROID_ZIP_EXTRACTOR:-$root/tools/dev/extract-pinned-zip.sh}" @@ -78,6 +83,9 @@ sdk_root="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/android-sdk}}" ndk_version="$pinned_ndk" cmake_version="$pinned_cmake" compile_sdk="$pinned_compile_sdk" +native_tools=true +expo=false +expo_ndk_input="" sdkmanager_install_attempts="${ANDROID_SDKMANAGER_INSTALL_ATTEMPTS:-4}" sdkmanager_retry_delay="${ANDROID_SDKMANAGER_RETRY_DELAY:-5}" @@ -103,6 +111,18 @@ while [ "$#" -gt 0 ]; do compile_sdk="$2" shift 2 ;; + --native-tools) + [ "$#" -ge 2 ] || fail "--native-tools requires a value" + native_tools="$2" + shift 2 + ;; + --expo) expo=true; shift ;; + --expo-ndk-version) + [ "$#" -ge 2 ] && [ -n "$2" ] || fail "--expo-ndk-version requires a value" + expo=true + expo_ndk_input="$2" + shift 2 + ;; -h | --help) usage exit 0 @@ -113,6 +133,16 @@ while [ "$#" -gt 0 ]; do esac done +case "$native_tools" in true|false) ;; *) fail "--native-tools must be true or false" ;; esac +[ "$expo" = false ] || [ "$native_tools" = true ] || fail "--expo requires native tools" +expo_ndk="" +if [ "$expo" = true ]; then + expo_ndk="$(manifest_package expo_ndk)" + [ -z "$expo_ndk_input" ] || [ "$expo_ndk_input" = "$expo_ndk" ] || + fail "Expo app NDK $expo_ndk_input differs from manifest pin $expo_ndk; update the pin before building" + case "$expo_ndk" in ''|.*|*.|*..*|*[!0-9.]*) fail "invalid Expo NDK pin" ;; esac +fi + [ -n "$sdk_root" ] || fail "Android SDK root is empty" case "$sdk_root" in /|.|..) fail "unsafe Android SDK root: $sdk_root" ;; esac [ "$ndk_version" = "$pinned_ndk" ] || @@ -165,6 +195,7 @@ case "$cmdline_entry_count" in *[!0-9]*|'') fail "$manifest $section.entry_count must be numeric" ;; esac +oliphaunt_acquisition_timeout >/dev/null require java require mktemp command -v "$curl_bin" >/dev/null 2>&1 || fail "missing required command: $curl_bin" @@ -201,7 +232,7 @@ sdkmanager_version() { local binary="$1" local output [ -f "$binary" ] && [ ! -L "$binary" ] && [ -x "$binary" ] || return 1 - output="$("$binary" --sdk_root="$sdk_root" --version 2>/dev/null)" || return 1 + output="$(oliphaunt_acquisition_run 30 "$binary" --sdk_root="$sdk_root" --version 2>/dev/null)" || return 1 printf '%s\n' "$output" | awk ' { sub(/\r$/, "") @@ -267,11 +298,10 @@ install_cmdline_tools() { for candidate_url in "$cmdline_url" "$cmdline_mirror_url"; do rm -f "$archive" echo "Downloading pinned Android command-line tools: $candidate_url" - if "$curl_bin" \ + if oliphaunt_acquisition_curl 240 6 2 "$curl_bin" \ --fail --location --silent --show-error \ --proto '=https' --proto-redir '=https' \ - --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 180 \ - --connect-timeout 20 --max-time 300 --max-filesize 220000000 \ + --connect-timeout 20 --max-filesize 220000000 \ --remove-on-error --output "$archive" "$candidate_url"; then actual="$(sha256_file "$archive")" if [ "$actual" = "$cmdline_sha256" ]; then @@ -279,6 +309,9 @@ install_cmdline_tools() { break fi echo "Android command-line-tools checksum mismatch from $candidate_url; trying the next pinned origin" >&2 + else + status=$? + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac fi done [ "$downloaded" = "1" ] || @@ -382,9 +415,10 @@ cmake_valid() { } ndk_valid() { - local directory="$sdk_root/ndk/$pinned_ndk" + local version="${1:-$pinned_ndk}" + local directory="$sdk_root/ndk/$version" local clang count=0 - package_revision_valid "$directory" "$pinned_ndk" || return 1 + package_revision_valid "$directory" "$version" || return 1 for clang in "$directory"/toolchains/llvm/prebuilt/*/bin/clang; do [ -e "$clang" ] || continue usable_executable "$clang" || return 1 @@ -393,12 +427,15 @@ ndk_valid() { [ "$count" -eq 1 ] } + sdk_packages_valid() { platform_tools_valid && platform_valid && - build_tools_valid && - cmake_valid && - ndk_valid + build_tools_valid || return 1 + if [ "$native_tools" = true ]; then + cmake_valid && ndk_valid || return 1 + [ "$expo" = false ] || ndk_valid "$expo_ndk" || return 1 + fi } cleanup_invalid_sdk_packages() { @@ -407,28 +444,33 @@ cleanup_invalid_sdk_packages() { platform_valid || rm -rf "$sdk_root/platforms/android-$pinned_compile_sdk" build_tools_valid || rm -rf "$sdk_root/build-tools/$pinned_build_tools" - cmake_valid || - rm -rf "$sdk_root/cmake/$pinned_cmake" - ndk_valid || - rm -rf "$sdk_root/ndk/$pinned_ndk" + if [ "$native_tools" = true ]; then + cmake_valid || rm -rf "$sdk_root/cmake/$pinned_cmake" + ndk_valid || rm -rf "$sdk_root/ndk/$pinned_ndk" + if [ "$expo" = true ]; then + ndk_valid "$expo_ndk" || rm -rf "$sdk_root/ndk/$expo_ndk" + fi + fi } install_sdk_packages() { local attempt=1 + local packages=(platform-tools "platforms;android-$pinned_compile_sdk" "build-tools;$pinned_build_tools") + if [ "$native_tools" = true ]; then + packages+=("cmake;$pinned_cmake" "ndk;$pinned_ndk") + [ "$expo" = false ] || packages+=("ndk;$expo_ndk") + fi while [ "$attempt" -le "$sdkmanager_install_attempts" ]; do cleanup_invalid_sdk_packages echo "Installing exact Android SDK package identities (attempt $attempt/$sdkmanager_install_attempts)" - if "$sdkmanager_bin" --sdk_root="$sdk_root" --install \ - "platform-tools" \ - "platforms;android-$pinned_compile_sdk" \ - "build-tools;$pinned_build_tools" \ - "cmake;$pinned_cmake" \ - "ndk;$pinned_ndk" && sdk_packages_valid; then + if oliphaunt_acquisition_run 1800 "$sdkmanager_bin" --sdk_root="$sdk_root" --install \ + "${packages[@]}" && sdk_packages_valid; then return 0 fi + cleanup_invalid_sdk_packages if [ "$attempt" -lt "$sdkmanager_install_attempts" ]; then echo "Android SDK package install or validation failed; repairing before retry" >&2 - sleep "$sdkmanager_retry_delay" + oliphaunt_acquisition_sleep "$sdkmanager_retry_delay" || exit $? fi attempt=$((attempt + 1)) done @@ -437,10 +479,15 @@ install_sdk_packages() { if ! sdk_packages_valid; then echo "Accepting Android SDK licenses" - yes | "$sdkmanager_bin" --sdk_root="$sdk_root" --licenses >/dev/null || true + yes | oliphaunt_acquisition_run 1800 "$sdkmanager_bin" --sdk_root="$sdk_root" --licenses >/dev/null || true install_sdk_packages fi sdk_packages_valid || fail "Android SDK package cache is invalid after repair" echo "ANDROID_HOME=$sdk_root" -echo "ANDROID_NDK_HOME=$sdk_root/ndk/$pinned_ndk" +if [ "$native_tools" = true ]; then + echo "ANDROID_NDK_HOME=$sdk_root/ndk/$pinned_ndk" + if [ -n "${GITHUB_ENV:-}" ]; then + echo "ANDROID_NDK_HOME=$sdk_root/ndk/$pinned_ndk" >> "$GITHUB_ENV" + fi +fi diff --git a/tools/dev/setup-android-sdk.test.sh b/tools/dev/setup-android-sdk.test.sh index f89784513..95be4a183 100755 --- a/tools/dev/setup-android-sdk.test.sh +++ b/tools/dev/setup-android-sdk.test.sh @@ -5,6 +5,10 @@ root="$(git rev-parse --show-toplevel)" installer="$root/tools/dev/setup-android-sdk.sh" extractor="$root/tools/dev/extract-pinned-zip.sh" tmp="$(mktemp -d)" +mkdir -p "$tmp/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$tmp/no-delay/sleep" +chmod +x "$tmp/no-delay/sleep" +export PATH="$tmp/no-delay:$PATH" trap 'rm -rf "$tmp"' EXIT HUP INT TERM @@ -19,7 +23,7 @@ const write = (name, data) => writeFileSync(root + '/' + name, data); function archive(name, version, layout = 'cmdline-tools') { const entries = { - [layout + '/bin/sdkmanager']: "#!/usr/bin/env bash\nset -euo pipefail\nsdk_root=\"\"\noperation=\"\"\npackages=()\nfor argument in \"$@\"; do\n case \"$argument\" in\n --sdk_root=*) sdk_root=\"${argument#--sdk_root=}\" ;;\n --version) operation=version ;;\n --licenses) operation=licenses ;;\n --install) operation=install ;;\n *) packages+=(\"$argument\") ;;\n esac\ndone\n[ -n \"$sdk_root\" ]\ncase \"$operation\" in\n version)\n printf '{version}\\n'\n ;;\n licenses)\n exit 0\n ;;\n install)\n expected=(\n platform-tools\n 'platforms;android-36'\n 'build-tools;36.0.0'\n 'cmake;3.22.1'\n 'ndk;27.0.12077973'\n )\n [ \"${#packages[@]}\" = \"${#expected[@]}\" ]\n for index in \"${!expected[@]}\"; do\n [ \"${packages[$index]}\" = \"${expected[$index]}\" ]\n done\n mkdir -p \\\n \"$sdk_root/platform-tools\" \\\n \"$sdk_root/platforms/android-36\" \\\n \"$sdk_root/build-tools/36.0.0\" \\\n \"$sdk_root/cmake/3.22.1/bin\" \\\n \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/platform-tools/adb\"\n chmod +x \"$sdk_root/platform-tools/adb\"\n printf 'AndroidVersion.ApiLevel=36\\n' > \"$sdk_root/platforms/android-36/source.properties\"\n printf 'fake-android-jar\\n' > \"$sdk_root/platforms/android-36/android.jar\"\n printf 'Pkg.Revision=36.0.0\\n' > \"$sdk_root/build-tools/36.0.0/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/aapt2\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/zipalign\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/apksigner\"\n chmod +x \\\n \"$sdk_root/build-tools/36.0.0/aapt2\" \\\n \"$sdk_root/build-tools/36.0.0/zipalign\" \\\n \"$sdk_root/build-tools/36.0.0/apksigner\"\n printf 'Pkg.Revision = 3.22.1\\n' > \"$sdk_root/cmake/3.22.1/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/cmake/3.22.1/bin/cmake\"\n chmod +x \"$sdk_root/cmake/3.22.1/bin/cmake\"\n printf 'Pkg.Revision = 27.0.12077973\\n' > \"$sdk_root/ndk/27.0.12077973/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\"\n chmod +x \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\"\n count=0\n [ ! -f \"$sdk_root/fake-install-count\" ] || count=\"$(cat \"$sdk_root/fake-install-count\")\"\n printf '%s\\n' \"$((count + 1))\" > \"$sdk_root/fake-install-count\"\n ;;\n *)\n exit 2\n ;;\nesac\n".replace('{version}', version), + [layout + '/bin/sdkmanager']: "#!/usr/bin/env bash\nset -euo pipefail\nsdk_root=\"\"\noperation=\"\"\npackages=()\nfor argument in \"$@\"; do\n case \"$argument\" in\n --sdk_root=*) sdk_root=\"${argument#--sdk_root=}\" ;;\n --version) operation=version ;;\n --licenses) operation=licenses ;;\n --install) operation=install ;;\n *) packages+=(\"$argument\") ;;\n esac\ndone\n[ -n \"$sdk_root\" ]\ncase \"$operation\" in\n version)\n printf '{version}\\n'\n ;;\n licenses)\n exit 0\n ;;\n install)\n expected=(\n platform-tools\n 'platforms;android-36'\n 'build-tools;36.0.0'\n 'cmake;3.22.1'\n 'ndk;27.0.12077973'\n )\n case \"${#packages[@]}\" in\n 3) expected=(\"${expected[@]:0:3}\") ;;\n 6) expected+=('ndk;27.1.12297006') ;;\n esac\n [ \"${#packages[@]}\" = \"${#expected[@]}\" ]\n for index in \"${!expected[@]}\"; do\n [ \"${packages[$index]}\" = \"${expected[$index]}\" ]\n done\n mkdir -p \\\n \"$sdk_root/platform-tools\" \\\n \"$sdk_root/platforms/android-36\" \\\n \"$sdk_root/build-tools/36.0.0\" \\\n \"$sdk_root/cmake/3.22.1/bin\" \\\n \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/platform-tools/adb\"\n chmod +x \"$sdk_root/platform-tools/adb\"\n printf 'AndroidVersion.ApiLevel=36\\n' > \"$sdk_root/platforms/android-36/source.properties\"\n printf 'fake-android-jar\\n' > \"$sdk_root/platforms/android-36/android.jar\"\n printf 'Pkg.Revision=36.0.0\\n' > \"$sdk_root/build-tools/36.0.0/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/aapt2\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/zipalign\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/apksigner\"\n chmod +x \\\n \"$sdk_root/build-tools/36.0.0/aapt2\" \\\n \"$sdk_root/build-tools/36.0.0/zipalign\" \\\n \"$sdk_root/build-tools/36.0.0/apksigner\"\n printf 'Pkg.Revision = 3.22.1\\n' > \"$sdk_root/cmake/3.22.1/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/cmake/3.22.1/bin/cmake\"\n chmod +x \"$sdk_root/cmake/3.22.1/bin/cmake\"\n printf 'Pkg.Revision = 27.0.12077973\\n' > \"$sdk_root/ndk/27.0.12077973/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\"\n chmod +x \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\"\n if [ \"${#packages[@]}\" = 3 ]; then\n rm -rf \"$sdk_root/ndk\" \"$sdk_root/cmake\"\n elif [ \"${#packages[@]}\" = 6 ]; then\n cp -R \"$sdk_root/ndk/27.0.12077973\" \"$sdk_root/ndk/27.1.12297006\"\n printf 'Pkg.Revision = 27.1.12297006\\n' > \"$sdk_root/ndk/27.1.12297006/source.properties\"\n fi\n count=0\n [ ! -f \"$sdk_root/fake-install-count\" ] || count=\"$(cat \"$sdk_root/fake-install-count\")\"\n printf '%s\\n' \"$((count + 1))\" > \"$sdk_root/fake-install-count\"\n ;;\n *)\n exit 2\n ;;\nesac\n".replace('{version}', version), [layout + '/bin/avdmanager']: '#!/usr/bin/env bash\nset -euo pipefail\nexit 0\n', [layout + '/bin/apkanalyzer']: '#!/usr/bin/env bash\nset -euo pipefail\nexit 0\n', [layout + '/source.properties']: 'Pkg.Revision=20.0\n', @@ -34,7 +38,7 @@ for (const [name, digest] of [ ['android-bad-sha.toml','0'.repeat(64)], ['android-wrong-version.toml',archive('android-wrong-version.zip','19.0')], ['android-wrong-layout.toml',archive('android-wrong-layout.zip','20.0','not-cmdline-tools')], -]) write('config/' + name, "[packages]\ncommand_line_tools_build = \"14742923\"\ncommand_line_tools_revision = \"20.0\"\nndk = \"27.0.12077973\"\ncmake = \"3.22.1\"\ncompile_sdk = \"36\"\nbuild_tools = \"36.0.0\"\n\n[command_line_tools.linux]\nurl = \"https://dl.google.com/android/repository/commandlinetools-linux-14742923_latest.zip\"\nmirror_url = \"https://edgedl.me.gvt1.com/edgedl/android/repository/commandlinetools-linux-14742923_latest.zip\"\nsha256 = \"{digest}\"\nentry_count = \"5\"\n\n[command_line_tools.mac]\nurl = \"https://dl.google.com/android/repository/commandlinetools-mac-14742923_latest.zip\"\nmirror_url = \"https://edgedl.me.gvt1.com/edgedl/android/repository/commandlinetools-mac-14742923_latest.zip\"\nsha256 = \"{digest}\"\nentry_count = \"5\"\n".replaceAll('{digest}', digest)); +]) write('config/' + name, "[packages]\ncommand_line_tools_build = \"14742923\"\ncommand_line_tools_revision = \"20.0\"\nndk = \"27.0.12077973\"\nexpo_ndk = \"27.1.12297006\"\ncmake = \"3.22.1\"\ncompile_sdk = \"36\"\nbuild_tools = \"36.0.0\"\n\n[command_line_tools.linux]\nurl = \"https://dl.google.com/android/repository/commandlinetools-linux-14742923_latest.zip\"\nmirror_url = \"https://edgedl.me.gvt1.com/edgedl/android/repository/commandlinetools-linux-14742923_latest.zip\"\nsha256 = \"{digest}\"\nentry_count = \"5\"\n\n[command_line_tools.mac]\nurl = \"https://dl.google.com/android/repository/commandlinetools-mac-14742923_latest.zip\"\nmirror_url = \"https://edgedl.me.gvt1.com/edgedl/android/repository/commandlinetools-mac-14742923_latest.zip\"\nsha256 = \"{digest}\"\nentry_count = \"5\"\n".replaceAll('{digest}', digest)); TS @@ -94,7 +98,7 @@ run_android() { --sdk-root "${SDK_ROOT:-$tmp/sdk}" \ --ndk-version 27.0.12077973 \ --cmake-version 3.22.1 \ - --compile-sdk 36 + --compile-sdk 36 "$@" } # The official mirror is a bounded fallback, and installed identities are exact. @@ -220,4 +224,39 @@ for retry_case in attempts delay; do [ ! -s "$tmp/curl.log" ] done +# Replay installs no native compilers; Expo explicitly installs both pinned NDKs. +SDK_ROOT="$tmp/replay-sdk" run_android --native-tools false > "$tmp/replay.out" +[ ! -d "$tmp/replay-sdk/ndk" ] && [ ! -d "$tmp/replay-sdk/cmake" ] +[ -x "$tmp/replay-sdk/platform-tools/adb" ] +SDK_ROOT="$tmp/expo-sdk" run_android --expo-ndk-version 27.1.12297006 > "$tmp/expo.out" +for ndk in 27.0.12077973 27.1.12297006; do + grep -qx "Pkg.Revision = $ndk" "$tmp/expo-sdk/ndk/$ndk/source.properties" +done +SDK_ROOT="$tmp/expo-sdk" CURL_MODE=fail-all run_android --expo > "$tmp/expo-cached.out" +grep -qx 1 "$tmp/expo-sdk/fake-install-count" +if SDK_ROOT="$tmp/replay-sdk" run_android --native-tools false --expo >/dev/null 2>&1; then + echo 'accepted Expo without native tools' >&2; exit 1 +fi +if run_android --expo-ndk-version 99.0.0 > "$tmp/skew.out" 2>&1; then + echo 'accepted unexpected Expo NDK' >&2; exit 1 +fi +grep -q 'differs from manifest pin' "$tmp/skew.out" +# A license request that exhausts the budget must not start package installation. +cp -R "$tmp/sdk" "$tmp/deadline-sdk" +sed 's/ exit 0/ printf "9999999\\n" > "$DEADLINE_CLOCK"; exit 0/' \ + "$tmp/sdk/cmdline-tools/latest/bin/sdkmanager" > "$tmp/deadline-sdk/cmdline-tools/latest/bin/sdkmanager" +rm "$tmp/deadline-sdk/build-tools/36.0.0/apksigner" +cat > "$tmp/bin/date" <<'DATE' +#!/bin/sh +cat "$DEADLINE_CLOCK" +DATE +chmod +x "$tmp/bin/date" +printf '1000000\n' > "$tmp/clock" +if PATH="$tmp/bin:$PATH" DEADLINE_CLOCK="$tmp/clock" SDK_ROOT="$tmp/deadline-sdk" run_android > "$tmp/deadline.out" 2>&1; then + echo 'expired SDK setup succeeded' >&2; exit 1 +fi +grep -q 'deadline' "$tmp/deadline.out" +cmp "$tmp/sdk/fake-install-count" "$tmp/deadline-sdk/fake-install-count" +[ -x "$tmp/deadline-sdk/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" ] +[ ! -e "$tmp/deadline-sdk/build-tools/36.0.0" ] echo "Android SDK bootstrap fault tests passed" diff --git a/tools/dev/setup-maestro.sh b/tools/dev/setup-maestro.sh deleted file mode 100755 index c722e0ab9..000000000 --- a/tools/dev/setup-maestro.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "must run inside the Oliphaunt git checkout" >&2 - exit 1 -} -cd "$root" - -need_cmd() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required command: $1" >&2 - exit 1 - } -} - -need_cmd curl -need_cmd java -need_cmd mktemp - -export MAESTRO_CLI_NO_ANALYTICS=true -export MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED=true -maestro_bin="$HOME/.maestro/bin/maestro" -maestro_manifest="tools/dev/maestro.toml" - -manifest_value() { - local key="$1" - local count - local value - count="$(grep -Ec "^[[:space:]]*${key}[[:space:]]*=" "$maestro_manifest" || true)" - [ "$count" = "1" ] || return 1 - value="$(sed -n "s/^[[:space:]]*${key}[[:space:]]*=[[:space:]]*\"\([^\"]*\)\"[[:space:]]*$/\\1/p" "$maestro_manifest")" - [ -n "$value" ] || return 1 - printf '%s\n' "$value" -} - -version="$(manifest_value maestro || true)" -maestro_url="$(manifest_value install_url || true)" -maestro_sha256="$(manifest_value sha256 || true)" -if [ -z "$version" ] || [ -z "$maestro_url" ] || [ -z "$maestro_sha256" ]; then - echo "$maestro_manifest must contain exactly one quoted maestro version, install_url, and sha256 pin" >&2 - exit 1 -fi - -normalized_version="${version#cli-}" -stable_maestro_version_re='^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z]+)*$' -if ! [[ "$normalized_version" =~ $stable_maestro_version_re ]] || { - [ "$version" != "$normalized_version" ] && [ "$version" != "cli-$normalized_version" ]; -}; then - echo "$maestro_manifest contains an invalid Maestro version: $version" >&2 - exit 1 -fi -expected_url="https://github.com/mobile-dev-inc/Maestro/releases/download/cli-$normalized_version/maestro.zip" -if [ "$maestro_url" != "$expected_url" ]; then - echo "$maestro_manifest install_url must be the exact release asset for cli-$normalized_version" >&2 - exit 1 -fi -if [ "${#maestro_sha256}" -ne 64 ] || [[ "$maestro_sha256" =~ [^0-9A-Fa-f] ]]; then - echo "$maestro_manifest sha256 must be exactly 64 hexadecimal characters" >&2 - exit 1 -fi -maestro_sha256="$(printf '%s' "$maestro_sha256" | tr '[:upper:]' '[:lower:]')" - -maestro_version() { - local binary="$1" - local output - local detected - output="$("$binary" --version 2>/dev/null)" || return 1 - detected="$(printf '%s\n' "$output" | awk 'NF { value = $NF } END { print value }')" - detected="${detected#cli-}" - [ -n "$detected" ] || return 1 - printf '%s\n' "$detected" -} - -maestro_sha256_file() { - local path="$1" - if command -v shasum >/dev/null 2>&1; then - shasum -a 256 "$path" | awk '{print tolower($1)}' - elif command -v sha256sum >/dev/null 2>&1; then - sha256sum "$path" | awk '{print tolower($1)}' - else - echo "Maestro installation requires shasum or sha256sum" >&2 - return 127 - fi -} - -export MAESTRO_VERSION="$normalized_version" -if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then - echo "Maestro installation requires shasum or sha256sum" >&2 - exit 127 -fi -umask 077 -mkdir -p "$HOME" -install_root="$HOME/.maestro" -temporary_root="$(mktemp -d "$HOME/.maestro.install.XXXXXX")" -previous_root="$temporary_root/previous" -had_previous=0 -promotion_started=0 -cleanup() { - local cleanup_status="$?" - trap - EXIT HUP INT TERM - if [ "$promotion_started" = "1" ] && [ "$had_previous" = "1" ] && \ - [ ! -e "$install_root" ] && [ ! -L "$install_root" ] && \ - { [ -e "$previous_root" ] || [ -L "$previous_root" ]; }; then - if ! mv "$previous_root" "$install_root"; then - echo "could not restore the previous Maestro installation; it remains at $previous_root" >&2 - [ "$cleanup_status" -ne 0 ] || cleanup_status=1 - exit "$cleanup_status" - fi - fi - rm -rf "$temporary_root" - exit "$cleanup_status" -} -trap cleanup EXIT -trap 'exit 129' HUP -trap 'exit 130' INT -trap 'exit 143' TERM - -archive="$temporary_root/maestro.zip" -extract_root="$temporary_root/extracted" -mkdir -p "$extract_root" -curl \ - --fail \ - --location \ - --silent \ - --show-error \ - --retry 4 \ - --retry-all-errors \ - --retry-delay 3 \ - --retry-max-time 300 \ - --connect-timeout 20 \ - --max-time 300 \ - --max-filesize 400000000 \ - --proto '=https' \ - --proto-redir '=https' \ - --tlsv1.2 \ - --remove-on-error \ - --output "$archive" \ - "$maestro_url" - -actual_sha256="$(maestro_sha256_file "$archive")" -if [ "$actual_sha256" != "$maestro_sha256" ]; then - echo "Maestro $normalized_version archive checksum mismatch: expected $maestro_sha256, got $actual_sha256" >&2 - exit 1 -fi - -bash tools/dev/bun.sh tools/dev/extract-maestro.mts "$archive" "$extract_root" "$normalized_version" - -candidate_root="$extract_root/maestro" -candidate_bin="$candidate_root/bin/maestro" -candidate_jar="$candidate_root/lib/maestro-cli-$normalized_version.jar" -if [ ! -f "$candidate_bin" ] || [ -L "$candidate_bin" ]; then - echo "Maestro archive did not produce a regular launcher at $candidate_bin" >&2 - exit 1 -fi -if [ ! -f "$candidate_jar" ] || [ -L "$candidate_jar" ]; then - echo "Maestro archive did not produce the pinned CLI jar at $candidate_jar" >&2 - exit 1 -fi -chmod 0755 "$candidate_bin" -candidate_version="$(maestro_version "$candidate_bin" || true)" -if [ "$candidate_version" != "$normalized_version" ]; then - echo "Maestro archive launcher version mismatch: expected $normalized_version, got ${candidate_version:-}" >&2 - exit 1 -fi - -if [ -e "$install_root" ] || [ -L "$install_root" ]; then - promotion_started=1 - had_previous=1 - mv "$install_root" "$previous_root" -fi -if mv "$candidate_root" "$install_root"; then - : -else - promotion_status=$? - if [ "$had_previous" = "1" ] && ! mv "$previous_root" "$install_root"; then - trap - EXIT - echo "Maestro promotion failed and rollback failed; the previous installation remains at $previous_root" >&2 - exit "$promotion_status" - fi - echo "Maestro promotion failed; the previous installation was restored" >&2 - exit "$promotion_status" -fi - -[ -x "$maestro_bin" ] || { - echo "maestro install did not produce $maestro_bin" >&2 - exit 1 -} - -if [ -n "${GITHUB_PATH:-}" ]; then - printf '%s\n' "$HOME/.maestro/bin" >>"$GITHUB_PATH" -fi - -"$maestro_bin" --version diff --git a/tools/dev/setup-maestro.test.sh b/tools/dev/setup-maestro.test.sh deleted file mode 100755 index 0ec1e810c..000000000 --- a/tools/dev/setup-maestro.test.sh +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root="$(git rev-parse --show-toplevel)" -installer="$root/tools/dev/setup-maestro.sh" -manifest="$root/tools/dev/maestro.toml" -configured_version="$(sed -n 's/^[[:space:]]*maestro[[:space:]]*=[[:space:]]*"\([^"]*\)"[[:space:]]*$/\1/p' "$manifest")" -expected_version="${configured_version#cli-}" -expected_sha256="$(sed -n 's/^[[:space:]]*sha256[[:space:]]*=[[:space:]]*"\([^"]*\)"[[:space:]]*$/\1/p' "$manifest")" -expected_url="$(sed -n 's/^[[:space:]]*install_url[[:space:]]*=[[:space:]]*"\([^"]*\)"[[:space:]]*$/\1/p' "$manifest")" -if [ -z "$expected_version" ] || [ -z "$expected_sha256" ] || [ -z "$expected_url" ]; then - echo "setup-maestro.test.sh: missing Maestro release metadata in $manifest" >&2 - exit 1 -fi -test_root="$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-maestro-test.XXXXXX")" -trap 'rm -rf "$test_root"' EXIT - -fail() { - echo "setup-maestro.test.sh: $*" >&2 - exit 1 -} - -assert_contains() { - local path="$1" - local expected="$2" - grep -F -- "$expected" "$path" >/dev/null || fail "$path did not contain: $expected" -} - -assert_curl_arg() { - local path="$1" - local expected="$2" - grep -Fx -- "$expected" "$path" >/dev/null || fail "curl did not receive argument: $expected" -} - -fake_bin="$test_root/fake-bin" -mkdir -p "$fake_bin" - -cat >"$fake_bin/curl" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -printf '%s\n' "$@" >"$MAESTRO_TEST_CURL_ARGS" -if [ "${MAESTRO_TEST_CURL_EXIT:-0}" != "0" ]; then - exit "$MAESTRO_TEST_CURL_EXIT" -fi -output="" -while [ "$#" -gt 0 ]; do - case "$1" in - --output) - output="$2" - shift 2 - ;; - *) shift ;; - esac -done -[ -n "$output" ] || exit 64 -cp "$MAESTRO_TEST_ARCHIVE" "$output" -SH - -cat >"$fake_bin/shasum" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -last="${!#}" -printf '%s %s\n' "$MAESTRO_TEST_SHA256" "$last" -SH - -cat >"$fake_bin/java" <<'SH' -#!/usr/bin/env bash -exit 0 -SH - -cat >"$fake_bin/maestro" <<'SH' -#!/usr/bin/env bash -printf '%s\n' "${MAESTRO_TEST_AMBIENT_VERSION:-cli-0.0.0}" -SH - -cat >"$fake_bin/mv" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -count=0 -if [ -f "$MAESTRO_TEST_MV_COUNTER" ]; then - count="$(cat "$MAESTRO_TEST_MV_COUNTER")" -fi -count=$((count + 1)) -printf '%s\n' "$count" >"$MAESTRO_TEST_MV_COUNTER" -if [ "${MAESTRO_TEST_MV_FAIL_SECOND:-0}" = "1" ] && [ "$count" = "2" ]; then - exit 73 -fi -exec /bin/mv "$@" -SH - -pinned_bun="$(bash "$root/tools/dev/bun.sh" "$root/tools/dev/node-info.mts" executable)" -ln -s "$pinned_bun" "$fake_bin/bun" -chmod 0755 "$fake_bin"/* - -fallback_bin="$test_root/fallback-bin" -no_hash_bin="$test_root/no-hash-bin" -mkdir -p "$fallback_bin" "$no_hash_bin" -for command_name in bash git grep sed tr awk mkdir mktemp dirname chmod rm cp cat; do - command_path="$(command -v "$command_name")" - ln -s "$command_path" "$fallback_bin/$command_name" - ln -s "$command_path" "$no_hash_bin/$command_name" -done -for helper in bun curl java maestro mv; do - cp "$fake_bin/$helper" "$fallback_bin/$helper" - cp "$fake_bin/$helper" "$no_hash_bin/$helper" -done -cp "$fake_bin/shasum" "$fallback_bin/sha256sum" - -make_archive() { - local output="$1" - local launcher_version="$2" - local shape="$3" - bash "$root/tools/dev/bun.sh" - "$output" "$launcher_version" "$shape" "$expected_version" <<'TS' -import {writeFileSync} from 'node:fs'; -import {zipArchive} from './tools/packaging/testdata/zip-fixture.mts'; -const [output,version,shape,archiveVersion] = process.argv.slice(2); -const rows = [{name:'maestro/bin/maestro',data:"#!/usr/bin/env bash\nprintf '" + version + "\\n'\n",externalAttributes:0o100755<<16}]; -if (shape !== 'missing-jar') rows.push({name:'maestro/lib/maestro-cli-'+archiveVersion+'.jar',data:'mock jar'}); -if (shape === 'traversal') rows.push({name:'maestro/../escape',data:'escape'}); -writeFileSync(output,zipArchive(rows)); -TS -} - -valid_archive="$test_root/valid.zip" -wrong_version_archive="$test_root/wrong-version.zip" -missing_jar_archive="$test_root/missing-jar.zip" -traversal_archive="$test_root/traversal.zip" -corrupt_archive="$test_root/corrupt.zip" -make_archive "$valid_archive" "$expected_version" valid -make_archive "$wrong_version_archive" "999.999.999" valid -make_archive "$missing_jar_archive" "$expected_version" missing-jar -make_archive "$traversal_archive" "$expected_version" traversal -printf 'not a zip archive\n' >"$corrupt_archive" - -run_case() { - local name="$1" - local configured_version="$2" - local archive="$3" - local reported_sha256="$4" - local fail_second_mv="${5:-0}" - local curl_exit="${6:-0}" - local manifest_mode="${7:-pinned}" - local hash_mode="${8:-shasum}" - local ambient_version="${9:-cli-0.0.0}" - local case_path - case "$hash_mode" in - shasum) case_path="$fake_bin:$PATH" ;; - sha256sum) case_path="$fallback_bin" ;; - none) case_path="$no_hash_bin" ;; - *) fail "unknown hash mode: $hash_mode" ;; - esac - - CASE_ROOT="$test_root/cases/$name" - CASE_REPO="$CASE_ROOT/repo" - CASE_HOME="$CASE_ROOT/home" - CASE_LOG="$CASE_ROOT/setup.log" - CASE_CURL_ARGS="$CASE_ROOT/curl-args" - CASE_GITHUB_PATH="$CASE_ROOT/github-path" - mkdir -p "$CASE_REPO/tools/dev" "$CASE_REPO/tools/dev" "$CASE_HOME/.maestro" - cp "$installer" "$CASE_REPO/tools/dev/setup-maestro.sh" - cp "$root/tools/dev/extract-maestro.mts" "$CASE_REPO/tools/dev/extract-maestro.mts" - cp "$root/tools/dev/bun.sh" "$CASE_REPO/tools/dev/bun.sh" - cp "$root/.prototools" "$CASE_REPO/.prototools" - mkdir -p "$CASE_REPO/tools/packaging" - cp "$root/tools/packaging/portable-archive.mts" "$CASE_REPO/tools/packaging/portable-archive.mts" - case "$manifest_mode" in - pinned) - printf '[toolchain]\nmaestro = "%s"\ninstall_url = "%s"\nsha256 = "%s"\n' \ - "$configured_version" "$expected_url" "$expected_sha256" \ - >"$CASE_REPO/tools/dev/maestro.toml" - ;; - unpinned) - printf '[toolchain]\nmaestro = "%s"\n' \ - "$configured_version" >"$CASE_REPO/tools/dev/maestro.toml" - ;; - wrong-url) - printf '[toolchain]\nmaestro = "%s"\ninstall_url = "https://example.invalid/maestro.zip"\nsha256 = "%s"\n' \ - "$configured_version" "$expected_sha256" \ - >"$CASE_REPO/tools/dev/maestro.toml" - ;; - invalid-sha) - printf '[toolchain]\nmaestro = "%s"\ninstall_url = "%s"\nsha256 = "not-a-sha256"\n' \ - "$configured_version" "$expected_url" \ - >"$CASE_REPO/tools/dev/maestro.toml" - ;; - *) fail "unknown manifest mode: $manifest_mode" ;; - esac - printf 'previous installation\n' >"$CASE_HOME/.maestro/previous-marker" - git -C "$CASE_REPO" init -q - - if ( - cd "$CASE_REPO" - env \ - PATH="$case_path" \ - HOME="$CASE_HOME" \ - GITHUB_PATH="$CASE_GITHUB_PATH" \ - MAESTRO_TEST_ARCHIVE="$archive" \ - MAESTRO_TEST_CURL_ARGS="$CASE_CURL_ARGS" \ - MAESTRO_TEST_CURL_EXIT="$curl_exit" \ - MAESTRO_TEST_SHA256="$reported_sha256" \ - MAESTRO_TEST_MV_COUNTER="$CASE_ROOT/mv-count" \ - MAESTRO_TEST_MV_FAIL_SECOND="$fail_second_mv" \ - MAESTRO_TEST_AMBIENT_VERSION="$ambient_version" \ - bash "$CASE_REPO/tools/dev/setup-maestro.sh" - ) >"$CASE_LOG" 2>&1; then - CASE_STATUS=0 - else - CASE_STATUS=$? - fi -} - -assert_previous_preserved() { - [ -f "$CASE_HOME/.maestro/previous-marker" ] || fail "$CASE_ROOT did not preserve the previous installation" -} - -assert_no_staging_dirs() { - local leftover - leftover="$(find "$CASE_HOME" -maxdepth 1 -name '.maestro.install.*' -print -quit)" - [ -z "$leftover" ] || fail "$CASE_ROOT left installation staging behind: $leftover" -} - -run_case success "$configured_version" "$valid_archive" "$expected_sha256" -[ "$CASE_STATUS" = "0" ] || fail "valid pinned archive failed: $(cat "$CASE_LOG")" -[ -x "$CASE_HOME/.maestro/bin/maestro" ] || fail "valid install did not promote the launcher" -[ -f "$CASE_HOME/.maestro/lib/maestro-cli-$expected_version.jar" ] || fail "valid install did not promote the pinned jar" -[ ! -e "$CASE_HOME/.maestro/previous-marker" ] || fail "valid install retained stale installation contents" -[ "$("$CASE_HOME/.maestro/bin/maestro" --version)" = "$expected_version" ] || fail "promoted launcher has the wrong version" -[ "$(cat "$CASE_GITHUB_PATH")" = "$CASE_HOME/.maestro/bin" ] || fail "valid install wrote the wrong GitHub PATH entry" -assert_no_staging_dirs -for argument in \ - --fail \ - --location \ - --retry-all-errors \ - --retry-max-time \ - --connect-timeout \ - --max-time \ - --max-filesize \ - --proto \ - --proto-redir \ - --tlsv1.2 \ - --remove-on-error \ - '=https' \ - "$expected_url"; do - assert_curl_arg "$CASE_CURL_ARGS" "$argument" -done - -run_case prefixed-version "cli-$expected_version" "$valid_archive" "$expected_sha256" -[ "$CASE_STATUS" = "0" ] || fail "cli-prefixed configured version was not preserved" - -run_case same-version-ambient "$expected_version" "$valid_archive" "$expected_sha256" 0 0 pinned shasum "cli-$expected_version" -[ "$CASE_STATUS" = "0" ] || fail "a same-version ambient Maestro prevented the pinned installation" -[ -s "$CASE_CURL_ARGS" ] || fail "a same-version ambient Maestro bypassed the pinned archive download" -[ -x "$CASE_HOME/.maestro/bin/maestro" ] || fail "same-version ambient repair did not promote the pinned launcher" -assert_no_staging_dirs - -run_case sha256sum-fallback "$expected_version" "$valid_archive" "$expected_sha256" 0 0 pinned sha256sum -[ "$CASE_STATUS" = "0" ] || fail "sha256sum-only environment failed: $(cat "$CASE_LOG")" - -run_case missing-hash-command "$expected_version" "$valid_archive" "$expected_sha256" 0 0 pinned none -[ "$CASE_STATUS" = "127" ] || fail "missing hash utilities did not fail with status 127" -assert_contains "$CASE_LOG" "requires shasum or sha256sum" -[ ! -e "$CASE_CURL_ARGS" ] || fail "missing hash utilities reached the network" -assert_previous_preserved -assert_no_staging_dirs - -run_case checksum-mismatch "$expected_version" "$valid_archive" "0000000000000000000000000000000000000000000000000000000000000000" -[ "$CASE_STATUS" != "0" ] || fail "checksum mismatch unexpectedly succeeded" -assert_contains "$CASE_LOG" "archive checksum mismatch" -assert_previous_preserved -assert_no_staging_dirs - -run_case corrupt-archive "$expected_version" "$corrupt_archive" "$expected_sha256" -[ "$CASE_STATUS" != "0" ] || fail "corrupt archive unexpectedly succeeded" -assert_contains "$CASE_LOG" "invalid Maestro archive" -assert_previous_preserved -assert_no_staging_dirs - -run_case missing-layout "$expected_version" "$missing_jar_archive" "$expected_sha256" -[ "$CASE_STATUS" != "0" ] || fail "archive with a missing CLI jar unexpectedly succeeded" -assert_contains "$CASE_LOG" "missing expected archive entry" -assert_previous_preserved -assert_no_staging_dirs - -run_case wrong-version "$expected_version" "$wrong_version_archive" "$expected_sha256" -[ "$CASE_STATUS" != "0" ] || fail "archive with the wrong launcher version unexpectedly succeeded" -assert_contains "$CASE_LOG" "launcher version mismatch" -assert_previous_preserved -assert_no_staging_dirs - -run_case traversal "$expected_version" "$traversal_archive" "$expected_sha256" -[ "$CASE_STATUS" != "0" ] || fail "archive with path traversal unexpectedly succeeded" -assert_contains "$CASE_LOG" "unsafe archive member" -assert_previous_preserved -assert_no_staging_dirs - -run_case unpinned-version 999.999.998 "$valid_archive" "$expected_sha256" 0 0 unpinned -[ "$CASE_STATUS" != "0" ] || fail "unpinned manifest unexpectedly succeeded" -assert_contains "$CASE_LOG" "must contain exactly one quoted maestro version, install_url, and sha256 pin" -[ ! -e "$CASE_CURL_ARGS" ] || fail "unpinned configured version reached the network" -assert_previous_preserved -assert_no_staging_dirs - -run_case wrong-release-url "$expected_version" "$valid_archive" "$expected_sha256" 0 0 wrong-url -[ "$CASE_STATUS" != "0" ] || fail "manifest with a non-release URL unexpectedly succeeded" -assert_contains "$CASE_LOG" "install_url must be the exact release asset for cli-$expected_version" -[ ! -e "$CASE_CURL_ARGS" ] || fail "invalid release URL reached the network" -assert_previous_preserved -assert_no_staging_dirs - -run_case invalid-sha "$expected_version" "$valid_archive" "$expected_sha256" 0 0 invalid-sha -[ "$CASE_STATUS" != "0" ] || fail "manifest with an invalid checksum unexpectedly succeeded" -assert_contains "$CASE_LOG" "sha256 must be exactly 64 hexadecimal characters" -[ ! -e "$CASE_CURL_ARGS" ] || fail "invalid checksum metadata reached the network" -assert_previous_preserved -assert_no_staging_dirs - -run_case promotion-rollback "$expected_version" "$valid_archive" "$expected_sha256" 1 -[ "$CASE_STATUS" != "0" ] || fail "failed atomic promotion unexpectedly succeeded" -assert_contains "$CASE_LOG" "previous installation was restored" -assert_previous_preserved -assert_no_staging_dirs - -run_case transport-failure "$expected_version" "$valid_archive" "$expected_sha256" 0 22 -[ "$CASE_STATUS" != "0" ] || fail "transport failure unexpectedly succeeded" -assert_previous_preserved -assert_no_staging_dirs - -echo "setup-maestro installer tests passed" diff --git a/tools/release/public-consumer-smoke.test.mts b/tools/release/public-consumer-smoke.test.mts index 2017f69da..189c1555e 100644 --- a/tools/release/public-consumer-smoke.test.mts +++ b/tools/release/public-consumer-smoke.test.mts @@ -100,7 +100,7 @@ if (fixtureMode === 'prepare-npm') { JSON.stringify({ lock: frozen, plan: publicConsumerPlan(frozen, ['sdk'], graph(products)), - deadlineMilliseconds: Date.now() + (scenario === 'timeout' ? 2000 : 30000), + deadlineMilliseconds: scenario === 'expired' ? 0 : Date.now() + 60000, }), ); process.exit(0); diff --git a/tools/release/public-consumer-smoke.test.sh b/tools/release/public-consumer-smoke.test.sh index d48298edf..3789dc69f 100644 --- a/tools/release/public-consumer-smoke.test.sh +++ b/tools/release/public-consumer-smoke.test.sh @@ -12,6 +12,19 @@ while IFS= read -r -d '' entry; do clean+=("$entry"); done < "$scratch/cargo/env (cd "$scratch/cargo"; "${clean[@]}" cargo generate-lockfile) [[ -f "$scratch/cargo/Cargo.lock" ]] mkdir "$scratch/bin" +export PUBLIC_PROBE_TIMEOUT="$(command -v gtimeout || command -v timeout)" +cat > "$scratch/bin/gtimeout" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +# Stage/startup has its own generous budget. Shorten only the deliberate +# hanging consumer, while still exercising the real process-group timeout. +if [[ " $* " == *" npm install "* && -n "${HANG_PUBLIC_PROBE:-}" ]]; then + sleep 2 + set -- "$1" 2s "${@:3}" +fi +exec "$PUBLIC_PROBE_TIMEOUT" "$@" +SH +chmod +x "$scratch/bin/gtimeout" cat > "$scratch/bin/npm" <<'SH' #!/usr/bin/env bash set -euo pipefail @@ -28,16 +41,18 @@ SH chmod +x "$scratch/bin/npm" export PATH="$scratch/bin:$PATH" SENSITIVE_TOKEN=must-not-survive CARGO_REGISTRY_TOKEN=must-not-survive export PUBLIC_PROBE_COUNTER="$scratch/attempts" PUBLIC_PROBE_FIXTURE="$source_root/tools/release/public-consumer-smoke.test.mts" -for mode in success fail timeout; do +for mode in success fail timeout expired; do mkdir "$scratch/$mode" bun tools/release/public-consumer-smoke.test.mts prepare-npm "$scratch/$mode" "$mode" unset FAIL_PUBLIC_PROBE HANG_PUBLIC_PROBE PUBLIC_PROBE_CHILD expected=0 + if [[ "$mode" == expired ]]; then expected=1; fi if [[ "$mode" == fail ]]; then export FAIL_PUBLIC_PROBE=1; expected=7; fi if [[ "$mode" == timeout ]]; then export HANG_PUBLIC_PROBE=1 PUBLIC_PROBE_CHILD="$scratch/child-pid"; expected=124; fi status=0 bash tools/release/public-consumer-smoke.sh --surface "$scratch/$mode" npm > "$scratch/$mode/output" 2>&1 || status=$? if [[ "$status" != "$expected" ]]; then cat "$scratch/$mode/output" >&2; exit 1; fi + if [[ "$mode" == expired ]]; then grep -q "shared public-consumer deadline reached" "$scratch/$mode/output"; fi bun tools/release/public-consumer-smoke.test.mts assert-npm "$scratch/$mode" "$mode" done [[ "$(wc -l < "$scratch/attempts")" -eq 3 ]] diff --git a/tools/release/release_plan.mts b/tools/release/release_plan.mts index ca8ef94ae..fd84abb4e 100644 --- a/tools/release/release_plan.mts +++ b/tools/release/release_plan.mts @@ -57,6 +57,9 @@ function printGithubOutput(plan) { console.log( `has_extension_artifacts=${String(extensionArtifactProducts.length > 0).toLowerCase()}`, ); + console.log( + `requires_native_extension_lifecycle_evidence=${extensionArtifactProductsForReleaseProducts(products, { family: 'native', prefix: TOOL }).length > 0}`, + ); console.log(`products_json=${JSON.stringify(products)}`); console.log(`extension_products_json=${JSON.stringify(extensionProducts)}`); console.log( From 3abdb52e36d82c134365adc729ce6498e370dd20 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Tue, 29 Sep 2026 19:47:08 +0000 Subject: [PATCH 2/9] chore: exclude internal CI audit from pull request --- .../CI_RELEASE_PROCESS_AUDIT_2026-09-29.md | 1153 ----------------- 1 file changed, 1153 deletions(-) delete mode 100644 src/docs/internal/CI_RELEASE_PROCESS_AUDIT_2026-09-29.md diff --git a/src/docs/internal/CI_RELEASE_PROCESS_AUDIT_2026-09-29.md b/src/docs/internal/CI_RELEASE_PROCESS_AUDIT_2026-09-29.md deleted file mode 100644 index 2f65d9a33..000000000 --- a/src/docs/internal/CI_RELEASE_PROCESS_AUDIT_2026-09-29.md +++ /dev/null @@ -1,1153 +0,0 @@ -# CI inventory and correction backlog — 2026-09-29 - -Audit baseline: `origin/main` at `32ce7b29510b74333e799601b69a71fd28122e80` -(#225). Evidence, inventories and timings below describe that baseline. The -implementation status is recorded first. Checked items mean implemented and -locally checked; they do not assert hosted qualification. Local timing comparisons -are identified separately from hosted end-to-end savings. - -## First correction batch — local implementation - -Branch: `f0rr0/fix-ci-cache-reuse`. This batch changes cache/setup plumbing; -product selection, compiler flags, runtime tests and release admission remain -unchanged. Hosted warm-run timings are still required before claiming savings. - -| Finding | Implemented | Remaining verification/work | -| --- | --- | --- | -| CI-06 | Postmaster maps its four Cargo output directories from checked-in workspaces, without including their parent | Measure hosted restore/save and warm compilation. Use the executor's patched dependency context for runtime metadata; preserve upstream dependency-only cache policy | -| CI-07 | Android ABIs now use the existing target-scoped, bounded native extension ccache restore/save path | Measure warm hit rate and produced-artifact equivalence; no desktop cache expansion | -| CI-08 | One caller-resolved save policy reaches Rust, LLVM, Gradle and all four reusable producer/package workflows | Verify manual opt-in/opt-out on GitHub; source/action checks pass locally | -| CI-13 | Both normal and replay iOS installed-app jobs disable PostgreSQL build-tool installation | Android provisioning separation is implemented in the second batch; simulator/Xcode verification remains enabled | -| CI-16 | Deleted the unused summary action | Mobile execution-definition consolidation is implemented in the second batch | -| CI-19 | Normal WASIX and Postmaster use one cached builder setup with the existing recipe label, context and cache scope, before Postmaster compilation | Measure hosted Buildx reuse; shared acquisition deadlines now bound source/bootstrap retries | - -Local validation includes the workflow/security/planner gate, real pinned -Postmaster source preparation and locked Cargo metadata for its cache owners, -metadata collection before generated dependencies exist, shared builder-label -reuse, and the pinned Rust-cache cleanup on fixture output using the configured -paths. The affected artifact-packaging, native-extension packaging and WASIX TS -unit tasks, plus CI-tool formatting/lint, passed locally. The failed Wasmer-root -metadata probe identified why cache metadata must use the executor's patched -dependency configuration; that probe does not indicate a runtime-build regression. - -## Second correction batch — local implementation - -The changes retain product coverage and release admission. Existing Moon tasks, -source fetchers, artifact validators and platform tools own the work; there is no -new CI framework. Same-run artifact transfers preserve their existing validation. - -| Findings | Implemented | Remaining proof or limitation | -| --- | --- | --- | -| CI-02 | Timeout fixtures separate preparation from the deliberate child timeout and cover delayed startup plus an expired deadline | Full release-tool suite passes; no production retry or timeout relaxation | -| CI-03/12/13 | Rust and Deno defaults come from their manifests; Android uses its SDK manifest; Expo's installed React Native version catalog must match the explicitly provisioned Expo NDK | Native NDK remains unchanged. Clean hosted Gradle build must confirm no implicit download. Replay installs no native compiler/CMake; source checks request only their capabilities | -| CI-04/05 | Evidence generation uses all five required modes; the existing uncached WASIX regression task owns its producer dependencies and fresh observations | Truncated materialization evidence is rejected. Planner retains the existing requirement that portable WASIX production receives lifecycle qualification, without repeating its four producer dependencies | -| CI-09/23 | Per-host WASIX AOT and Node-API work share a reusable workflow; Linux consumers wait only for Linux; browser/TS packaging starts after source gates | Final all-host aggregates remain. manylinux keeps its container boundary; no incompatible Cargo cache reuse is claimed | -| CI-11 | WASIX lifecycle is grouped under E2E, alongside native lifecycle | Candidate-bound native release evidence is completed in the third batch | -| CI-14 | Matrix labels use short capability names; full selected tasks appear in the job summary | Product/aggregate display names are completed in the fifth batch; protocol IDs stay stable | -| CI-16/26 | CI and replay use one installed-app action. Maestro and its installer/flow are removed; continuous current-launch logs feed the existing structured receipt validator | Failure, crash, capture death, app death and stale PASS rejection are tested. Actual iOS simulator and Android emulator runs remain required | -| CI-17 | Browser-host Git and crate sources use the shared bounded, exact-pin fetcher and safe archive extraction | Live exact sources, transport fault tests and the full browser/TS package build pass; acquisition now precedes runtime-dependent consumers | -| CI-20 | Producer input groups exclude unrelated Markdown and unit fixtures; SDK README packaging does not select runtime compilation | Actual Moon affected-selection regressions pass. Postmaster's shipped README gets a cheap producer in the third batch | -| CI-21 | WASIX compilation cache has a compatible fallback. Reuse requires the current Moon input hash and verified compiler-output checksums; source preparation, staging and profile validation still run | Cold/warm, corrupted output and changed-input fault tests pass. Full hosted compiler output and warm timing still need validation | -| CI-22 | Android static compilation overlaps one Linux support producer. Both ABI packagers consume that same-run support output instead of compiling Linux again | SQL-only package fixture passes and missing inputs fail closed. Actual Android native artifacts still need hosted qualification | -| CI-24 | The existing transfer runner submits dependency-ready batches to Moon, allowing independent tasks to overlap | Real pinned Moon fixture proves sibling overlap, join ordering and no transferred-producer replay; existing task-failure checks pass | -| CI-25 | A frozen, validated iOS carrier can replace only the unchanged React Native package metadata dependency in ordinary affected runs | Changed native inputs, missing/corrupt cache and explicit product/full qualification retain producers. Same-SHA empty diffs are handled without invoking Moon on empty stdin | - -New intermediate handoffs retain 30 days so failed jobs can be rerun throughout -GitHub's [supported rerun window](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/re-run-workflows-and-jobs). -Release-input/proof retention is unchanged. Browser-host Cargo state has its own -source/toolchain-keyed cache, and its built host files travel with SDK consumer -inputs so consumers do not silently compile the browser host again. - -Local verification covers the exact pinned workflow/security gate, actual Moon -planning and transfer execution, Android installer fault cases, native extension -packaging, WASIX build orchestration, React Native and WASIX TypeScript owner -checks, extension metadata/evidence checks, source-fetch fault cases, and the full -release-tool suite. The browser host compiled and the TypeScript npm package -passed its package validator (`oliphaunt-wasix-ts:package`, 7m55s locally). The -mobile receipt tests and actual Moon transfer fixture also pass under a locally -built GNU Bash 3.2.57; this checks shell compatibility, not macOS platform APIs. -Test logs are in `/tmp/oliphaunt-ci-cache-fixes` for this local -session. A complete Linux workflow run cannot establish Apple/Windows/device -behavior: those platform runs and cache hit/timing measurements remain outstanding. -No new GitHub run or exact-SHA `Qualified` result is claimed for this working tree. - -| Verification command | Local result | -| --- | --- | -| `bash tools/ci/check-workflows.sh` with pinned Moon 2.5.4 | Passed actionlint/security checks, 66 planner tests, six artifact-transfer tests, cache/setup tests and real Moon scheduling fixtures | -| `bash tools/release/release-check.sh` | Passed metadata and release implementation checks, including delayed-start timeout fixtures | -| `moon run oliphaunt-react-native:test` and relevant format/lint/typecheck tasks | Passed; actual Apple transport checks remain macOS-only | -| `moon run oliphaunt-wasix-ts:test oliphaunt-wasix-ts:typecheck oliphaunt-wasix-ts:format-check` | Passed, including 352 SDK tests | -| `moon run oliphaunt-wasix-ts:package` | Compiled the pinned browser host, built the SDK and validated the npm archive | -| `bash src/third-party/tools/source-fetch-core.test.sh` and `bash src/third-party/tools/fetch-sources.sh production-all --validate-only` | Passed exact-pin, retry/failover, archive validation and checkout-preservation checks | - -## Third correction batch — rebuilds, release proof and coverage - -| Findings | Implemented | Verification and limit | -| --- | --- | --- | -| CI-01 | All five previously unwired SDK runtime tasks now have hosted execution owners: four in native consumers and WASIX resources in the WASIX regression job. They consume existing same-run artifacts, require positive test execution, and remain uncached | Rust SQL/runtime, mobile broker, Swift, Kotlin and both WASIX seed resource tests pass locally against real payloads. These Linux host tests do not replace device/platform qualification | -| CI-10 | Coalesced four identical Cargo feature/target configurations and removed a duplicate executor library compilation already covered by the complete product executor suite | Invocation/selection comparison preserves all previous selections while reducing 25 Cargo invocations to 20. Distinct compiler and compiler-free feature profiles remain separate; hosted compile savings are unmeasured | -| CI-11 | Native aggregate/shard proof is bound into the release candidate and revalidated before native extension-carrier publication, including runtime-owned contrib. Aggregate proof has the same 90-day retention as WASIX, includes all shard receipts, and supports reruns | Candidate write/verify and tamper fixtures pass, including wrong source/run, missing shards, incomplete published extension coverage and modified evidence. SDK-only releases retain their existing consumer qualification | -| CI-20 | Postmaster's shipped README has a cheap declared producer. Release assembly consumes its output, while prose-only affected CI avoids the runtime build and regression closure | Actual Moon affected/full-release planning tests pass. Runtime/compiler source changes still select their original producers | - -The native suites exposed an unused `pg_config` preflight requirement that did -not match shipped runtime archives; the shared preflight now requires only the -actual runtime inputs. A runnable staging fixture checks missing inputs, -inherited-path isolation, optional tools/broker, cleanup and exit propagation. -Swift preparation no longer regenerates bindings already owned by its Moon -dependency; the standalone Swift wrapper retains explicit generation. Kotlin's -runtime task similarly reuses the declared binding producer. - -Local checks include the full release-tool suite, candidate/receipt tests, -workflow/security and actual Moon planner/transfer checks, and the five real -SDK runtime tasks. The Postmaster source/fault suite and native evidence owner -tests also pass. The complete patched Wasmer compiler/runtime suite still -requires hosted qualification; the Cargo selection comparison proves retained -selections, not their runtime outcomes. Native payloads were rebuilt and packaged from this working -tree. WASIX resource tests used the unchanged runtime/resources downloaded from -successful baseline CI run `36567197390` at `32ce7b2`; that local exercise is not -exact-SHA qualification of these changes. Logs are under -`/tmp/oliphaunt-ci-remaining`. Full hosted platform runs and cold/warm timing -remain required before claiming a measured CI speedup. - -## Fourth correction batch — acquisition deadlines - -Source fetching and repository-owned bootstrap downloads now use one shared -shell deadline. Retries, mirrors, lock waits and dependent requests spend the -same budget. APT update/install defaults to 15 minutes, complete source scopes -to 30 minutes, and individual source pins to 15 minutes. The -[maintainer pattern and budget table](../maintainers/testing.md#acquisition-deadlines) -define the scope, override, cleanup and package-manager boundaries. - -Verification passed: `source-inputs:test`, `dev-tools:test`, -`dev-tools:test-mobile-setup`, `ci-tools:test`, `ci-workflows:llvm-install-unit`, -WASIX orchestration/installer tests, Postmaster builder identity tests and the -complete pinned workflow gate. Deadline fixtures cover final-attempt expiry, -shared retry/mirror budgets, lock ownership, child termination, cancellation, -APT update-to-install admission and preservation of valid Android packages. -Helper and PostgreSQL transport tests also pass under GNU Bash 3.2.57 on Linux. -Live WASIX sources fetched and verified exact pins; the final Docker recipe -completed APT in 56.2 seconds, installed the pinned compiler assets and passed -compiler smoke/version checks. Its warm build reused every layer, and the -recipe label passed Postmaster validation. Actual Moon affected queries select -all helper consumers. Logs: `/tmp/oliphaunt-acquisition-*`. Hosted platform and -wall-clock qualification remain outstanding; no release qualification is claimed. - -## Fifth correction batch — platform review and remaining efficiency - -The acquisition review corrected GNU timer discovery when Windows System32's -`timeout.exe` shadows Git Bash's Coreutils executable. It prefers `gtimeout`, -then a verified GNU `timeout`, then `/usr/bin/timeout`. Android setup checks this -prerequisite before inspecting or repairing SDK caches, and its macOS action -installs Coreutils when needed. Missing timer support must not masquerade as -corrupt command-line tools. The browser-host patch loop now works on Bash 3.2 -and propagates failure to read its patch list. - -| Findings | Implemented | Verification and limit | -| --- | --- | --- | -| CI-14 | Visible jobs distinguish Builds, Packages, Tests and E2E; mobile ABI labels identify seed production, mixed JavaScript/ICU work is explicit, and runtime labels use WASIX | Stable job IDs, aggregate gates and release/replay job-name references are preserved. The public `wasm_target` dispatch input remains compatible | -| CI-15 | Eleven planning, proof-aggregate and artifact-consuming finalizer jobs skip Moon task-output restore/save | Real graph checks prove no normally cached local task subtree remains in these jobs. The transfer adapter runs artifact-dependent work with `MOON_CACHE=off`. Verified tool archives still use their existing cache | -| CI-18 | Independent planner observations run through four native `xargs` workers, each with its own output file | All 93 parsed JSON outputs matched the serial run. Local elapsed time fell from 92.60 to 59.32 seconds (36%); this is the observation phase, not total CI wall time. A fault check rejects failed workers and covers paths containing spaces | - -The pinned workflow gate passed, including 66 planner tests, seven artifact -transfer/policy tests, scheduling fixtures and workflow/security checks. -Timer discovery, retry/deadline and process-cleanup fixtures pass on Bash 3.2.57 -on Linux. The Android installer and bootstrap installer fixtures also pass -with child shells using Bash 3.2. These are shell compatibility checks, not -Windows, macOS, iOS simulator or Android emulator qualification. - -The source-acquisition owner task passed again against the reviewed helper -(3m19s). A real Docker build completed its pinned APT transaction in 61.4 seconds -and compiler acquisition in 15.4 seconds, then verified wasixcc 0.4.3, Clang -21.1.2 and Binaryen 130. These runs followed a local machine restart that -interrupted the earlier build attempts; interrupted attempts are not counted -as passing checks. The full browser-host build also passed with Bash 3.2.57 -selected for the script and child shells, including real source acquisition, -patch application, Rust compilation, WASM optimization and Rollup (7m40s). - -A fresh read-only cache inventory found 284 Moon entries totaling 291.6 MB, -including 128 under 1 KB. Rust caches totaled 8,197.2 MB and LLVM 1,065.1 MB. -No remote caches were deleted. Retention/pruning of those larger caches still -needs warm restore/save and hit-rate measurements. Baseline successful source -jobs took roughly 23–201 seconds, with setup often dominant; warm timings alone -do not establish safe cold-run deadlines. Uncached release checks still consume -Git/release history. An isolated offline Cargo probe confirmed that packaging a -dependency first does not let a later standalone package resolve its unpublished -version; selecting both crates in the same invocation succeeds. Retain that -multi-crate staging, rather than deleting the repeated dependency selections. - -### Still open - -- CI-10: measure remaining distinct Postmaster feature/profile costs before - changing their guarantees or overlapping portable and host compilation. -- CI-15: budget/prune the large compiler caches using actual warm restore/save - size and hit rates; jobs without reusable Moon outputs now skip that cache. -- CI-18: measure cold source-group setup costs before changing group deadlines; - prove release-check inputs before caching them. Cargo's unpublished dependency - staging remains necessary; consolidate package ownership only if measured cost - justifies changing that graph. -- CI-19: measure hosted cache reuse and acquisition tails against the implemented - budgets; source/bootstrap transactions now share deadlines across retries and mirrors. -- Hosted verification: compare equivalent cold/warm runs and unchanged artifact - contracts, including both Android ABIs, iOS simulator and all WASIX hosts. - -## Assessment - -The second pass found larger structural waste than the first: duplicate -toolchain and host-runtime builds, affected selection that promotes documentation -or unit-test changes into full compiler pipelines, and whole-platform barriers -between otherwise independent producers and consumers. The backlog now contains -**26 findings**, including eight new structural findings, CI-19 through CI-26. - -For wall time, work on both long chains: Postmaster and WASIX runtime/AOT/SDK. -Fixing only the longest job moves the bottleneck rather than removing it. -For ordinary PR feedback, narrow compilation inputs before tuning small tests. -For runner cost, remove duplicate host builds and ineffective caches. These -three objectives overlap, but their savings must not be added together. - -The highest-return order is: - -1. Share the already-cached WASIX Docker builder with Postmaster; fix its Rust - cache mapping and the WASIX compilation-cache restore policy. -2. Stop documentation and isolated unit-test changes from scheduling unrelated - compilation; separate package changes from compiler input changes. -3. Remove cross-host WASIX barriers and start independent browser-host work - early; overlap Postmaster host compilation only after measuring cache fixes. -4. Reuse Android's duplicate Linux support build and avoid serializing it with - independent Android compilation; persist the useful extension compiler cache. -5. Remove Maestro's status-label-only role from SDK smoke, preserving exact-launch - receipts and failure detection. Address Android-to-iOS package coupling. - -At the audit baseline, correctness fixes CI-01 through CI-05 were also needed. -CI-01 through CI-05 are now implemented; hosted platform qualification remains. -Several recent failures detected real product/release defects; indiscriminate -test deletion would lose useful proof without addressing the largest delays. - -There is no justification here for a new CI framework, another product -registry, a generalized evidence service, a new retry service, or wholesale -workflow generation. Moon, product manifests, the existing candidate record, -and GitHub Actions already provide the required pieces. - -## Scope and evidence - -Inspected all seven workflows, their local actions and execution adapters, -the expanded Moon project/task graph, product/SDK task definitions, release -admission and artifact transfers, and representative underlying scripts/tests. -The inventory contains **56 Moon projects, 373 tasks across 45 task-owning -projects, 27 release manifest entries, and 15 local composite actions**. -The seven workflow files define 76 job entries before matrix expansion and -reusable-workflow calls. These are different counts, not competing inventories. - -An exhaustive current-tree source matrix selects 122 check targets in 21 groups, -3 policy targets, and 54 test targets in 17 groups. Actual affected/product runs -select fewer. There are 33 `ci-` task mappings, including native lifecycle. - -Downloaded the latest 100 workflow records and inspected the failed job -inventories for every failed CI/Release run in the September 25–29 subset. -That subset contained 53 runs: CI had 14 successes, 8 failures, 14 cancellations, -12 skipped runs and one running run; Release had two successes and two failures. -**This is not a flake rate.** These were different commits and scopes; -cancellations include ordinary supersession/merge cancellation, and skipped -runs include the deliberate PR-close tombstones. - -Timing figures use GitHub job `started_at`/`completed_at`, summed without OS -billing multipliers. They are runner-minutes, not billed minutes or CPU time. -Latest-attempt job lists can mix retained successes with rerun jobs. The clean -successful first-attempt run below is the cost baseline; the rerun's elapsed -hours are not treated as continuous computation. - -| Run | Purpose / result | Executed jobs | Runner-minutes | Wall time | -| --- | --- | ---: | ---: | ---: | -| [36492755110](https://github.com/f0rr0/oliphaunt/actions/runs/36492755110) | Merged release candidate `7b192697`; success | 105 | 800.8 | 122.0 min | -| [36480008083](https://github.com/f0rr0/oliphaunt/actions/runs/36480008083) | Corresponding release PR; success | 104 | 764.7 | About 95 min | -| [36521743783](https://github.com/f0rr0/oliphaunt/actions/runs/36521743783) | Focused Apple release-workflow fix; success | 9 | 4.3 | About 4 min | -| [36523497369](https://github.com/f0rr0/oliphaunt/actions/runs/36523497369) | Full manual `34e49318`; Android recovered, qualification receipt failed | Mixed attempts | Not a comparable single-attempt sample | Excluded | -| [36567197390](https://github.com/f0rr0/oliphaunt/actions/runs/36567197390) | Current main `32ce7b29`, manual full qualification | Running during collection | Excluded from complete-run totals | Excluded | - -The baseline's work distribution was: - -| Responsibility | Jobs, including aggregates | Runner-minutes | -| --- | ---: | ---: | -| Planning | 2 | 1.0 | -| Checks | 15 | 23.3 | -| Tests | 14 | 27.3 | -| Builds, packaging and build-owned consumer checks | 64 | 713.2 | -| E2E | 8 | 35.7 | -| Required + Qualified | 2 | 0.3 | - -Native extension producers accounted for **225.8 minutes** across seven targets; -Postmaster portable/target/finalization jobs for **147.2 minutes**. Together -they account for roughly 47% of baseline runner time. Optimizing small JSON -receipt validators will not materially change that bill. - -## Second-pass wall-time analysis - -Times below are minutes since the start of successful run `36492755110`, not -durations that can be summed across overlapping jobs. - -| Chain / checkpoint | Started | Finished | What controls the next stage | -| --- | ---: | ---: | --- | -| Cheap checks/tests ready | — | 6.6 | Heavy producers can start | -| Postmaster portable + qualification | 6.6 | 78.7 | Every Postmaster host waits for the whole job | -| Postmaster macOS + qualification | 78.8 | 120.8 | Longest host; final aggregation follows | -| WASIX portable producer | 10.7 | 54.1 | Runtime, tools, extensions and seeds share one producer job | -| WASIX AOT hosts | 54.2 | 78.1 | Linux x64 finished at 67.5; consumers wait for Windows | -| WASIX Node-API hosts | 78.1 | 91.7 | Linux x64 finished at 84.6; TS waits for Windows | -| WASIX Linux regression | 78.7 | 96.2 | Linux-only proof starts after the AOT matrix barrier | -| WASIX TS package + consumers | 91.8 | 102.4 | Includes independent 6m34s browser-host compilation | -| iOS app / installed E2E | 49.1 / 62.9 | 62.7 / 73.9 | Extension carriers, app build, simulator/driver startup | -| Android app / installed E2E | 52.5 / 62.5 | 62.5 / 65.1 | Slowest Android extension producer then app build | -| Final qualification | — | 122.0 | All selected branches must succeed | - -**The ceiling matters:** even deleting Postmaster's entire chain would leave -the WASIX TS branch finishing at minute 102.4, plus final gates. That is only -about 19 minutes of possible total improvement in this run. Postmaster's -147.2 runner-minutes are not 147.2 minutes of wall-time savings. - -| Priority | Structural correction | Measured exposure | Scope of benefit / limitation | -| --- | --- | --- | --- | -| First | Postmaster builder reuse, CI-19 | APT layer 39m52s; normal WASIX cached image setup 24s | Removes a demonstrated long-tail setup risk. Other observed APT layers were only 59–64s; do not promise 40 minutes on every run | -| First | Select work by actual inputs, CI-20 | README edits select full Postmaster or WASIX pipelines | Avoids whole expensive branches on focused PRs; no reduction for a legitimate full-product change | -| First | Effective compiler caches, CI-06/07/08/21 | Warm downstream WASIX outputs still preceded by 12m23s core compilation | Shortens both long branches; validate cache compatibility and cold behavior | -| Next | Host-specific WASIX dependencies, CI-09/23 | 10.6-minute Linux AOT wait, then another 7.2-minute TS wait | Reduces the second-longest chain; waits overlap other work and are not directly additive | -| Next | Android host support reuse/overlap, CI-22 | Linux support rebuilds took 24m49s and 20m06s across the two ABI jobs | Large compute saving; waiting for Linux before cross-compiling would preserve most wall time | -| Next | Remove status-label UI driver, CI-26 | iOS app passed at 23:38:12; Maestro finished at 23:42:50 | Roughly 4m38s avoidable tail in this installed-app sample; not the full-run critical path | -| After those | Transfer adapter scheduling, CI-24 | Windows AOT's serial step took 20m38s | Some independent tasks could overlap; CPU and compiler locks limit gains | -| Focused Android work | Decouple unchanged iOS carrier metadata, CI-25 | Real Android Kotlin edit selects iOS runtime production | Avoids unrelated Apple work; was not Android's limiting input in the full baseline | - -The portable producer itself holds core output for roughly another 26 minutes -while extension/tool work finishes. That is a scheduling observation, not a -26-minute savings estimate: full consumers need those extensions, and splitting -the producer carelessly duplicates its Docker/compiler workspace. Fix input -selection, caches and host barriers before adding more portable build jobs. - -Postmaster is a published GitHub-assets product: -[v0.1.0](https://github.com/f0rr0/oliphaunt/releases/tag/liboliphaunt-wasix-postmaster-v0.1.0) -contains Linux x64/arm64 and macOS arm64 carriers. It is not an SDK-registry -package, but that does not make it unreleased. Removing its required coverage -would be a separate product-support decision, not a CI optimization assumed by -this audit. - -## Workflow and ownership inventory - -| Workflow | Trigger / role | Owner and important boundary | -| --- | --- | --- | -| [ci.yml](../../../.github/workflows/ci.yml) — 3,522 lines, 60 job entries | PR, merge group, main push, manual qualification | Moon chooses task scope; Actions provisions hosts and transfers artifacts; `Required` aggregates selected work; `Qualified` records eligible exact-main proof | -| [release.yml](../../../.github/workflows/release.yml) — 1,855 lines, 8 jobs | Manual prepare or publish | Release tools select products, request missing qualification, freeze one candidate, publish it, verify public delivery, refresh docs | -| [extension-artifacts-native.yml](../../../.github/workflows/extension-artifacts-native.yml) | Reusable target producer | `extension-artifacts-native:build-target`; four caller partitions, seven full-matrix targets | -| [liboliphaunt-native-desktop.yml](../../../.github/workflows/liboliphaunt-native-desktop.yml) | Reusable desktop producer | Native runtime/tools/resources tasks; Linux and other-host partitions | -| [broker-runtime.yml](../../../.github/workflows/broker-runtime.yml) | Reusable broker producer | `oliphaunt-broker:build-release-assets`; Linux and other-host partitions | -| [mobile-extension-packages.yml](../../../.github/workflows/mobile-extension-packages.yml) | Reusable packaging | `extension-packages:package-mobile`; separate Android/iOS callers | -| [mobile-e2e.yml](../../../.github/workflows/mobile-e2e.yml) | Manual/reusable diagnostic replay | Resolves existing exact-SHA app artifacts, then installs/tests them; not an automatic second CI run | - -The local action layer owns tool installation: Moon, Node, Bun, Deno, Rust and -Rust tools, Swift, Apple, Android, MSVC, Wasmer LLVM, Maestro and npm publishing. -`setup-node-bun` composes existing installers. `collect-ci-summary` has no callers. - -### Product/lane inventory - -The following accounts for every current CI task-to-job mapping. Stable IDs -are shown because human display names are not consistently descriptive. - -| Job ID or related IDs | Product task owners / actual work | Target or consumer boundary | -| --- | --- | --- | -| `liboliphaunt-native-desktop` | `liboliphaunt-native` build/package/artifact tests; `postgres-tools-native` package/tests; native standard/ICU seeds | Linux x64/arm64, macOS arm64, Windows x64 | -| `liboliphaunt-native-android` | Native runtime build/package | Android arm64-v8a and x86_64 | -| `liboliphaunt-native-ios` | Native runtime XCFramework build/package | iOS device/simulator slices | -| `liboliphaunt-native-android-abi`, `liboliphaunt-native-ios-abi` | ABI finalization **and database-resource seed production** | Mobile compatibility domains; iOS seed work can require macOS | -| `liboliphaunt-native-release-assets` | Native asset aggregation | All selected native targets | -| `extension-artifacts-native` | Exact extension compilation/package | Four desktop targets, two Android ABIs, iOS XCFramework | -| `extension-artifacts-wasix` | Exact portable extension archive packaging | Portable WASIX | -| `extension-packages` | Public Cargo/npm/Maven/Apple extension carriers | Selected extensions and target families | -| `mobile-extension-packages` | Mobile extension carriers | Android/iOS partitioned | -| `broker-runtime`, `broker-release-assets` | Broker binaries and aggregate assets | Four desktop targets | -| `node-direct`, `node-direct-release-assets` | Native Node addon, built-artifact qualification, aggregate assets | Four desktop targets | -| `liboliphaunt-wasix-runtime` | Core portable runtime, extension/tool compiler outputs, portable tools, WASIX standard/ICU seeds | Linux-hosted portable production; multiple product owners share the job | -| `liboliphaunt-wasix-aot` | Core/tool/extension AOT; Rust SDK and pgwire host execution | Four native AOT hosts | -| `liboliphaunt-wasix-release-assets` | WASIX asset aggregation | Portable plus selected AOT hosts | -| `wasix-napi`, `wasix-napi-release-assets` | WASIX Node-API addon and aggregation | Four desktop targets | -| `wasix-postmaster` | Portable inputs, host carriers and release finalization | Separate portable job; Linux x64/arm64 and macOS arm64 host jobs; runtime-patch/regression/recovery tasks also explicitly invoked in YAML | -| `rust-sdk-package` | Native SDK, build helper, native bindings, broker crate and query crate; packed consumer compilation | Linux; package closure before execution | -| `wasix-rust-package` | WASIX Rust SDK and pgwire packages/consumer compilation | Linux; runtime tests are elsewhere | -| `js-sdk-package` | Native TS, query TS, WASIX tools TS **and ICU data** packages | Linux; heterogeneous products under a JS SDK label | -| `wasix-ts-sdk-package` | Browser-host build, TS package, browser/native consumers; PostgreSQL tools browser/native consumers | Linux; artifact-dependent tests and independent packaging combined | -| `swift-bindings`, `swift-sdk-package` | XCFramework production on macOS; source/carrier assembly on Linux | Swift SDK | -| `kotlin-sdk-package`, `kotlin-maven-staging` | Maven SDK packaging, staging | Kotlin/JVM/Android SDK | -| `react-native-sdk-package` | npm package and clean package consumer | React Native SDK | -| `native-consumers` | Native TS Node/Bun/Deno, Rust installed SDK, broker consumer; release-only published-dependency TS variant | Canonical Linux x64 artifacts | -| `native-extension-lifecycle` | Native direct/broker/server, restart and backup/restore | Linux x64; three full-catalog shards plus aggregate receipt | -| `mobile-build-android`, `mobile-build-ios` | Expo installed-app production | Android x86_64 emulator app; iOS simulator app | - -Additional hosted execution: `wasix-release-regression`, native lifecycle -aggregation and Android/iOS installed-app E2E. WASIX regression is an explicit -workflow call to the evidence collector, not a `ci-`-mapped Moon root. - -Product selection is not a second directory-based planner: keep release product -identity in product manifests, execution/data dependencies in Moon, and runner -and transport topology in Actions. `tools/ci` and `.github/scripts` currently -split adapter ownership; neither should acquire independent SDK policy. - -### SDK proof comparison - -| SDK/surface | Source proof | Packed/installed proof | Missing or misleading hosted ownership | -| --- | --- | --- | --- | -| Native Rust | Rust unit/doc tests, formatting, Clippy | Packed crate compile plus real installed direct/broker consumer | `test-integration` not hosted; environment-dependent tests can return without execution in the ordinary test lane | -| WASIX Rust | Rust unit/doc/public API tests | Packed crate compile; AOT tests on four hosts; Linux exhaustive extension regression | `test-integration` resource tests not hosted; `test-regression` task bypassed by collector | -| Native TS | TypeScript/unit/format/lint | Packed SDK on Node/Bun/Deno, native direct/broker execution | Linux execution is intentional, not proof on every desktop host | -| WASIX TS | TypeScript/unit/format/lint | Packed browser and native consumers, tools consumers | Independent package build delayed behind runtime/addon matrices | -| Swift | Linux portable tests; Apple platform tests; iOS broker compile check | XCFramework/source package; React Native app exercises the native backend | Public Swift `NativeRuntimeTests` suite disabled without env; `test-native` not hosted | -| Kotlin | JVM and Android unit tests, plugin check, lint/format | Maven package; React Native app exercises the native backend | `NativeBindingsTest` assumes prepared PGDATA and skips otherwise; owner task not hosted | -| React Native | TS/unit, C++ tests, codegen | Packed consumer, actual Android/iOS app builds and installed E2E | Retain distinct device/platform checks; these caught actual product failures | -| Shared query / native bindings / mobile bindings | Shared protocol tests and Rust/TS source gates | Included in dependent SDK package closures | Mobile bindings' native broker roundtrip task not hosted | - -Five currently unreachable integration tasks are confirmed against the expanded -task graph and the actual check/test matrix writer, including explicit -Postmaster workflow roots: - -- `oliphaunt-rust:test-integration` -- `oliphaunt-swift:test-native` -- `oliphaunt-kotlin:test-native-bindings` -- `oliphaunt-mobile-bindings:test-native` -- `oliphaunt-wasix-rust:test-integration` - -This does **not** mean the runtimes or mobile broker are untested: installed -consumers, native extension lifecycle, AOT, and mobile app E2E already execute. -It means those particular SDK guarantees are not provided by their existing -dedicated tests. Compare coverage before wiring overlapping suites wholesale. -The broker's separate local `test-integration` is not another such gap: its -`postgres_client` test is exercised by the hosted `test-consumer` wrapper. - -Docs generation and public-site checks have a separate delivery path, including -the release docs-refresh job. `docs:test` and `docs:test-package` are not selected -by the quality-tag adapter. Vercel's live deployment configuration was not -audited; do not claim GitHub's `Required` proves that site's build or availability. - -## Correction inventory - -P1: correctness/reliability or a demonstrated large recurring cost. P2: measured -efficiency/maintenance improvement. P3: small cleanup. Owners below are code -ownership areas, not invented individual assignees. - -### CI-01 — P1 — Make runtime test ownership truthful - -- [x] Wire the unique guarantees from the five tasks above into their SDK's - artifact-consuming lane; remove or mark genuinely redundant local aliases. -- [x] Change selection tests to assert final executable roots/dependency closure, - not merely that a task occurs in the affected-task inventory. - -**Evidence:** `write-affected-moon-target-matrices.mts` selects quality/unit, -coverage and quality/static/format/smoke tags. The five tasks have neither -qualifying tags nor hosted `ci-` roots/dependents. The planner test at -`tools/ci/ci-plan-node-products.test.mts:448` asserts native Rust/Swift affected -selection, which passes without proving hosted execution. Swift's native suite -is env-disabled, Kotlin uses `assumeTrue`, and WASIX resource tests are ignored -unless explicitly requested. - -**Owner:** SDK Moon tasks and `tools/ci`. **Done when:** full qualification and -relevant SDK-only changes select actual execution; a missing runtime fails the -runtime lane; the selected test count is positive; existing same-run artifacts -are consumed without compiling a second runtime. Preserve cheap unit lanes. - -### CI-02 — P1 — Remove the deadline fixture's scheduling race - -- [x] Give fixture preparation a separate budget from the deliberate child - timeout; start/coordinate the timeout test after the child can actually run. - Keep a separate deterministic assertion for an already-expired deadline. - -**Evidence:** `tools/release/public-consumer-smoke.test.mts:103` grants the timeout -scenario 2 seconds including process startup/staging. The shell floors the -deadline to seconds and returns 1 if it expires before starting the child, -whereas the fixture expects 124 and a child PID. The full local release suite -has failed here repeatedly while isolated execution passed. This audit's -unchanged control passed; adding a two-second delay only before reading the -timeout scenario's context reproduced exit 1 and `shared public-consumer -deadline reached`. - -**Owner:** release tools. **Done when:** delayed startup cannot turn this into a -false failure, while hanging descendants are still terminated and genuine -consumer failure is never retried. Do not delete the timeout/credential tests or -solve this by retrying the entire release suite. - -### CI-03 — P1 — Provision the Android app's actual NDK before its build - -- [x] Resolve Expo/React Native's intended Gradle `ndkVersion` and provision - that exact version through the existing bounded SDK installer before Gradle. -- [x] Use one shared pin if platform compatibility allows it; otherwise make the - native-runtime and Expo-app versions two explicit, owned requirements. - -**Evidence:** [job 109274401218](https://github.com/f0rr0/oliphaunt/actions/runs/36523497369/job/109274401218) -successfully installed setup's `27.0.12077973`, then Gradle independently fetched -`27.1.12297006` and failed with `Archive is not a ZIP archive`. The same app -succeeded on rerun. `ANDROID_NDK_HOME` does not set the generated app's Gradle -version. The existing installer already has bounded retries; this second -download bypasses it. - -**Owner:** Android setup and Expo runner. **Done when:** a clean runner has every -declared NDK before compilation and Gradle does no surprise NDK installation. -Do not silently change the released native ABI/toolchain merely to match Expo. -Android documents explicit selection via -[`android.ndkVersion`](https://developer.android.com/studio/projects/install-ndk). - -### CI-04 — P1 — Use one WASIX lifecycle mode contract - -- [x] Make the evidence table and qualification validator consume the same - required mode definition, including materialization and physical backup/restore. - -**Evidence:** `src/extensions/tools/extension-evidence.mts` declares five modes, -but `evidenceMatrix()` enumerates four and omits materialization. A real report -with materialization removed passes the table's current-evidence check and -fails candidate validation. The collector normally records all five, and the -candidate validator catches the omission: this is validator drift, not proof -that an incomplete release was published. - -**Owner:** extension evidence contract, consumed by release qualification. -**Done when:** the same truncated report is rejected by both entry points; normal -reports pass; deliberate compatibility with frozen old report schemas remains -explicit. Reuse the existing contract module rather than add another registry. - -### CI-05 — P1 — Put WASIX qualification dependencies in the task graph - -- [x] Make the existing `oliphaunt-wasix-rust:test-regression` task the execution - owner for fresh lifecycle observations and let the workflow collect its receipt. -- [x] Remove the hand-maintained four-producer insertion in - `requiredTasksForAffected()` after graph-based selection replaces it. - -**Evidence:** the collector runs `runtime-smoke.sh regression` directly, bypassing -the existing Moon task. `tools/ci/ci_plan.mts:298` separately inserts runtime AOT, -extension portable/AOT, and tools AOT dependencies. #222 repaired an actual -missed-input/selection failure here. The next new input should not require -another synchronized YAML/planner/script list. - -**Owner:** WASIX Rust task + CI adapter. **Done when:** changing a required producer -changes the graph once, planner/transfer tests follow it, and qualification gets -fresh positive observations rather than a cached success without a receipt. - -### CI-06 — P1 — Stop discarding Postmaster's compiler cache - -- [x] Configure the existing Rust cache action for the actual Postmaster Cargo - workspaces/target directories; expose its existing mapping input through - `setup-rust` if needed. Include the browser-host custom Cargo directory in the - same audit. Keep saved compiler state bounded and keyed to toolchain/source - inputs; continue verifying produced artifacts. - -**Evidence:** setup currently supplies `. -> target`. Postmaster compiles under -`target/oliphaunt-wasix-postmaster/runtime/...`, but that intermediate directory -is not a Cargo target root. The pinned cache action interprets it as a profile, -keeps only `build`, `.fingerprint` and `deps`, and removes its `runtime` child. -A fixture executing the pinned cleanup code confirmed that the nested compiled -artifact is deleted. The portable job restored an approximately 89 MB cache, -then spent **62m49s** in `runtime-build`; the cache saved afterward was still -approximately 89 MB. macOS runtime compilation took **27m42s**. - -**Owner:** Rust setup and Postmaster/browser-host producers. **Done when:** a -second clean hosted run restores the intended compiler dependencies and shows -a material reduction in compilation time, with correct rebuilds after pin, -patch, compiler or target changes. Do not claim the whole 62 minutes is saved: -that task also performs other work. The native cache action already supports -[workspace/target mappings](https://github.com/Swatinem/rust-cache); -the inspected cleanup implementation is -[pinned here](https://github.com/Swatinem/rust-cache/blob/e18b497796c12c097a38f9edb9d0641fb99eee32/src/cleanup.ts). - -### CI-07 — P1 — Persist compiler state for expensive native extension lanes - -- [x] Extend the existing bounded extension ccache restore/save pattern first - to Android; measure other targets before expanding it. - -**Evidence:** the reusable extension workflow persists ccache only for -`ios-xcframework`, despite configuring it for other targets. The successful -Android arm64 extension job took **44.8 minutes**; ccache reported **73 hits / -6,168 cacheable calls (1.18%)**, 6,095 misses, and only about 0.1 GB used of its -2 GB limit. Android x86_64 took 37.7 minutes. This is a stronger optimization -candidate than deleting short regression tests. - -**Owner:** native extension producers. **Done when:** unchanged-source warm runs -reuse a bounded cache, publish identical valid outputs and record hit/miss -statistics; compiler/source changes remain safe. Budget storage alongside CI-15. -Do not transfer final product qualification from a previous source commit. - -### CI-08 — P2 — Honor manual cache-save policy in reusable workflows - -- [x] Pass the parent workflow's resolved cache-save choice to reusable - producers instead of recomputing it as push-only inside them. - -**Evidence:** CI exposes `save_heavy_caches` for manual main qualification and -computes `HEAVY_CACHE_SAVE_IF` accordingly. `extension-artifacts-native.yml` -redefines it as `event == push && ref == main`; the manual opt-in cannot save -its iOS extension cache. The `setup-rust` wrapper likewise does not expose every -lower-level cache option, so each caller's effective policy needs checking. - -**Owner:** CI reusable-workflow interfaces. **Done when:** main push, manual -opt-in, manual opt-out and PR cases have tested, consistent save behavior. - -### CI-09 — P2 — Start independent WASIX packaging earlier - -- [x] Run browser-host compilation and TS package production after source - checks, then feed the package to artifact-dependent consumers. -- [x] Narrow Linux-only consumers' dependencies to Linux producers where the - gain justifies the existing reusable-workflow partition pattern. - -**Evidence:** the baseline's Linux AOT completed at about minute 67.5, but WASIX -regression waited for Windows AOT until minute 78.1: roughly 10.6 minutes of -unnecessary waiting for that consumer. Linux Node-API finished at minute 84.6; -the TS job waited for Windows until minute 91.8, then spent **6m34s compiling -the browser host** before package/browser tests. The host build has no runtime -artifact dependency. Native Linux consumers also depend on desktop aggregates -despite existing Linux partitions. - -**Owner:** WASIX SDK and CI topology. **Done when:** the independent package -producer begins after cheap gates, each consumer waits only for bytes it uses, -and aggregate release gates still require every selected platform. These gains -improve feedback; they do not add directly to total wall-time savings while -Postmaster remains the critical path. Do not create a job per tiny task. - -### CI-10 — P2 — Reduce Postmaster compile variants before cutting behavior tests - -- [x] Consolidate equivalent Cargo feature/profile invocations in - `src/wasix/postmaster/wasmer/tests.sh`; keep genuinely different feature tests. -- [ ] Measure remaining distinct feature/profile compilation costs on cold and - warm hosted runs. -- [ ] After fixing cache layout, evaluate starting independent native host - compilation before portable guest qualification completes, only if the - remaining critical-path gain warrants the added artifact transfer. - -**Evidence:** portable `runtime-patch-tests` took **23m31s**, alongside -`runtime-build` at 62m49s. The script invokes many filtered Cargo runs with -several repeated feature sets. The critical path was approximately 6.6 minutes -to start Postmaster, 72.1 portable-job minutes, 42.0 macOS-target minutes, then -aggregation. macOS's actual initdb stress and backend-wave stress were only -1m20s and 50s; immediate recovery took 34s. Deleting those tests first targets -the wrong cost and loses reliability coverage. - -**Owner:** Postmaster. **Done when:** timings distinguish compilation from test -execution, equivalent invocations are grouped without losing named tests, and -warm/cold measurements justify any topology change. No promised percentage -saving until measured. - -Second-pass correction: the 62m49s `runtime-build` duration was **not mostly -Rust compilation**. Its Docker APT layer alone took 39m52s; see CI-19. Fixing -the Rust cache alone would not remove that delay. Portable tasks also overlap, -so their individual durations must not be summed into a job duration. - -### CI-11 — P2 — Make release evidence consistent without weakening admission - -- [x] Decide and document which product guarantees need a candidate-bound - receipt, then bind native lifecycle proof through the existing candidate - record if native extensions require the same release-level guarantee. -- [x] Put WASIX regression under the E2E aggregate, alongside native lifecycle, - while preserving the independent release-product requirement check. - -**Evidence:** native lifecycle already validates exact source/tree, selected -extension/shard coverage, artifact hashes, direct/broker/server and lifecycle -PASS records. The real report had 39 extensions, three shards and 46 consumed -artifact hashes. It is enforced by CI but is not bound/replayed by publication -like WASIX evidence. Native evidence retention is 30 days, versus 90 for WASIX -and the candidate. WASIX is displayed as E2E but actually required by `Builds`. - -The generic gate proves the jobs the plan selected; it does not independently -prove that the plan selected every product-required job. A synthetic incomplete -plan passed generic candidate coverage and was rejected by the separate WASIX -release requirement. Preserve that independent check. Runtime-specific receipt -contents can differ; shared source/run/attempt/digest binding should not. - -**Owner:** release qualification + extension owners. **Done when:** omission, -wrong source/run, future attempts and missing modes fail consistently, retained -successful jobs from earlier attempts still work, and required evidence lasts -for the supported approval window. JSON revalidation is cheap and is not a -second lifecycle execution. - -### CI-12 — P2 — Remove duplicated authoritative toolchain pins - -- [x] Read Rust's default from `rust-toolchain.toml` and Android defaults from - `tools/dev/android-sdk.toml` instead of repeating them in action inputs. -- [x] Review Node/Bun/Deno/npm/LLVM pin consumers for the same pattern; distinguish - intentional fixture values from authoritative live configuration. - -**Evidence:** Rust `1.93.1` appears in both the root toolchain and -`setup-rust-tools` default. Android's NDK/CMake/API defaults repeat the TOML. -`ci.yml` defines `NPM_VERSION` without a consumer in that workflow; release -publishing has its own live definition. Deno's caller and verified installer -also share a version contract. Manual parallel edits are avoidable. - -**Owner:** tool installers. **Done when:** bumping the owner pin updates the -effective local and hosted setup; fixture pins remain explicit test inputs; -verification still rejects wrong versions. Do not substitute mutable latest. - -### CI-13 — P2 — Install only the capabilities a lane consumes - -- [x] Separate Android compilation setup from Java/Gradle checks and emulator - execution using the existing setup action's narrow inputs/capabilities. -- [x] Disable Apple build-dependency installation in installed-app replay when - it only needs Xcode/simulator tools. - -**Evidence:** `setup-android` always provisions NDK and CMake. It is also used for -Kotlin formatting and installed-APK E2E, which do not compile native code. -Apple replay calls the general Apple action, whose build-dependency setup is -already optional. These are unnecessary download/failure opportunities even -when hosted images make the observed setup fast. - -**Owner:** platform setup actions and capability tags. **Done when:** formatting -and APK replay succeed without native compiler installation, compilation still -receives its exact toolchain, and no second general provisioning layer appears. - -### CI-14 — P2 — Make display names describe work and keep protocol IDs stable - -- [x] Use short capability/owner group labels with full Moon targets in the job - summary; do not concatenate every expanded project/task title. -- [x] Rename visible heterogeneous jobs and aggregates consistently: runtime - producers, packages, consumers, lifecycle, qualification. Clarify mobile ABI - jobs' seed-production responsibility and the mixed JS/ICU package job. -- [x] Use WASIX in human-facing runtime names; retain `.wasm` and upstream - WebAssembly target terminology where technically correct. Migrate the public - `wasm_target` dispatch input deliberately if renamed (retained for compatibility). - -**Evidence:** a successful check job name was **384 characters**. Names include -`Builds / broker-runtime`, human-readable names, reusable caller prefixes, and -E2E under Builds. `Checks / Policy` combines release metadata, workflow behavior -and broker license auditing. These make failures harder to locate, without -adding useful proof. - -**Owner:** CI adapter/display labels. **Done when:** names are short and stable, -the exact selected targets remain visible, and consumers of `Required`, -`Qualified`, `Builds` and other protocol identities continue to work. - -### CI-15 — P2 — Budget caches and artifact retention by purpose - -- [x] Skip Moon task-output cache transfers for jobs with no reusable local - tasks; retain verified tool archive caching and validate the task graph. -- [ ] Fix directory effectiveness first, then bound/retire low-value cache - entries using observed size and warm-run savings. -- [ ] Retain release inputs/proof for the supported window; give disposable - intermediate/debug artifacts an explicit shorter policy where replay permits. - -**Evidence:** the cache snapshot had **258 entries / 10.68 GB**: Rust caches -8.29 GB, Linux Wasmer LLVM 1.07 GB, 207 Moon entries only 0.23 GB. Numerous Moon -entries are nearly empty; the largest storage opportunity is not their count. -The baseline stored **114 artifacts / 2.07 GB**, including repeated envelopes -of native/iOS inputs and packaged outputs, many with default 90-day retention. -Some copies are necessary fan-out/release inputs; don't delete them by name. - -**Owner:** CI artifact/cache consumers. **Done when:** each retained artifact has -a consumer and retention reason; warm-run benefits exceed restore/save costs; -expired evidence fails explicitly. Actual eviction/billing limits were not -queried, so 10.68 GB alone does not establish paid usage or thrashing. GitHub -documents [cache limits, immutability and eviction](https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching). - -### CI-16 — P2 — Consolidate mobile execution definitions and drop dead setup - -- [x] Share the installed-app execution/setup definition between main CI and - manual mobile replay while retaining same-run versus explicit-run artifact - resolution at their respective boundaries. -- [x] Delete the unused `.github/actions/collect-ci-summary/action.yml` unless - a real caller is introduced as part of CI-14. - -**Evidence:** Android/iOS steps, environment values, emulator configuration, -transport handling and report upload are maintained both in `ci.yml` and -`mobile-e2e.yml`. The latter has no workflow caller today but remains a useful -manual diagnostic entry point. It is not currently an automatic duplicate run. -`collect-ci-summary` has no references and merely prints command suggestions. - -**Owner:** mobile CI adapter. **Done when:** normal CI and replay run the same -installed-app assertions, replay does not rebuild apps, and platform setup fixes -have one owner. Prefer one existing reusable workflow or a small composite -action; do not add both and a generator. - -### CI-17 — P2 — Move fragile source acquisition out of late consumer work - -- [x] Reuse the existing exact-pin source acquisition behavior for the WASIX - browser host's upstream Git/crate downloads, with bounded retries and complete - validation before package/consumer work. - -**Evidence:** `src/wasix/browser-host/build-sdk.sh` owns its own Git fetch and two -one-shot `curl` downloads; they execute after the long runtime/addon dependency -chain. Digests and commits are checked, which is good. This audit did not -observe a browser-host network failure; this is a concrete unprotected path, -not a claimed measured flake rate. - -**Owner:** browser-host producer, using `src/third-party` acquisition helpers. -**Done when:** transient download failures recover within a bound, invalid bytes -fail without replacing valid output, and exhausted failures occur in the -producer lane. No blanket test retries or unverifiable fallback source. - -### CI-18 — P3 — Right-size source groups and deadlines after the major fixes - -- [x] Run independent planner observations concurrently with a bounded native - worker pool; retain output equivalence and fail the gate on a worker error. -- [ ] Keep capability grouping, but measure setup versus execution before - changing group size; use bounded category-appropriate job deadlines. -- [ ] Review always-uncached deterministic release checks and duplicate package - prerequisite invocations only after their complete inputs are declared. - -**Evidence:** generic check/test jobs all allow 90 minutes although the baseline's -slowest source groups finished in roughly 6 minutes. Grouping is alphabetical, -up to eight static/four unit targets. `release-tools:test` and `metadata` are -uncached; most product unit checks are cached. Broker crate packaging invokes -Cargo packaging of query/bindings after declaring their package dependencies. -These are real cleanup candidates but much smaller than compiler costs. - -**Owner:** source-task owners and CI capability adapter. **Done when:** a stuck -short source check stops promptly, cold setup has enough budget, and cached -checks invalidate on every input they actually consume. Don't add a historical -duration scheduler, force serial formatting before every unit test, or remove -valid cold-package checks to save seconds. - -### CI-19 — P1 — Reuse the existing WASIX builder in Postmaster - -- [x] Use the existing Buildx/GHA cache setup for both WASIX consumers, preserving - Postmaster's recipe label and immutable recipe identity. -- [x] Prepare and validate the toolchain before expensive runtime compilation; - bound the overall acquisition transaction as well as individual retries. - `tools/dev/acquisition.sh` provides the common pattern and fault checks; - [acquisition deadlines](../maintainers/testing.md#acquisition-deadlines) - documents owner budgets and boundaries. A real cold Docker build completed - the pinned TLS-verified APT transaction in 65.6 seconds and verified all - compiler assets and their versions. - -**Evidence:** `fresh_ensure_docker_image` in -`src/wasix/postmaster/lib/common.sh:1694` invokes a separate raw Docker build of -the same `src/wasix/runtime/assets/build/docker` recipe. Postmaster's workflow -does not restore the `wasix-builder` cache already used by the normal WASIX -producer. The sysroot step invokes this after six serial Cargo binary builds -in `src/wasix/postmaster/wasmer/bin/build-runtime.sh`. - -In [the baseline portable job](https://github.com/f0rr0/oliphaunt/actions/runs/36492755110/job/109167195616), -the APT layer took **2,392.3 seconds**, including 133 MB downloaded in 23m16s. -The regular WASIX job's cached builder setup took **24 seconds**. The same -Postmaster APT layer took **64.1 seconds** on the release PR and **59.2 seconds** -on the current-main run. This is a demonstrated bad tail, not a typical -40-minute saving. The old install helper combined outer retries with APT retries -and per-request timeouts without a transaction-wide deadline. It now bounds -update, install and retry waits together to 15 minutes by default. - -**Owner:** shared WASIX build-tool setup + Postmaster. **Done when:** both jobs -restore a compatible builder, a recipe change invalidates it, and Postmaster's -recipe-label validation still rejects stale images. Merely retagging the current -normal image is insufficient: it lacks Postmaster's required label. Use the -existing setup/cache policy; no registry, published builder product or new -promotion service. Docker's [GHA cache backend](https://docs.docker.com/build/cache/backends/gha/) -already supports the scoped reuse needed here. - -### CI-20 — P1 — Stop turning documentation and unit-test edits into compiler work - -- [x] Narrow producer inputs and distinguish source compilation, package content, - unit fixtures and runtime qualification in the existing Moon task definitions. -- [x] Add small affected-selection regressions for these concrete boundaries. - -**Evidence:** the pinned Moon query plus the actual `jobs-for-affected` planner, -run with one changed file at a time, selected: - -| Changed existing file | Heavy jobs selected | -| --- | --- | -| `src/wasix/postmaster/README.md` | Full Postmaster portable and host pipeline | -| `src/wasix/postmaster/lib/durable-publication.test.mts` | Full Postmaster pipeline in addition to unit work | -| `src/wasix/sdks/ts/README.md` | WASIX portable, AOT, Node-API, extensions and TS consumers | -| `src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh` | WASIX and Postmaster, plus native extension producers/packages | - -Postmaster's carrier/portable/package inputs include broad project globs; the -runtime Docker input directory includes its tests. The planner then expands -required producer closures. Selecting the Postmaster job starts its explicitly -listed heavy workflow roots regardless of which narrow source edit caused it. - -**Owner:** product Moon inputs first, CI planner only where the job envelope -cannot represent the resulting scope. **Done when:** a README still repackages -any package containing it, and a unit-test edit still runs that test, without -invalidating unrelated compilers. Compiler recipes, patches, source pins, -catalogs and real runtime tests must retain their full dependencies. Do not -solve this with repository-wide docs ignores or by skipping package integrity. -Selection alone does not prove all tasks rebuilt; Postmaster's uncached heavy -tasks and CI-21 show why these selections are nevertheless expensive today. - -### CI-21 — P1 — Let WASIX reuse compatible compiler state across light commits - -- [x] Add a compatible restore fallback to the existing compilation cache, - scoped to all required compiler/profile inputs and trusted cache provenance. -- [x] Avoid running the uncached compiler prerequisite when a complete, - validated matching output closure can be restored. - -**Evidence:** `ci.yml`'s `Restore WASIX compilation cache` uses the head SHA as -its key and only the PR base / push-before / current SHA as its restore key. -There is no compatible prefix fallback. Manual dispatch searches the current -SHA twice. An intervening cheap workflow-only commit that saved no compilation -cache therefore breaks reuse of the previous heavy build. - -The baseline saved a roughly 296 MB compilation cache for `7b192697`; current -main's manual run searched only `32ce7b29`, never that prior compatible key. -This establishes a lookup defect, not proof that the older entry could never -have been evicted. In the current-main WASIX log, `compiler-output` still took -**12m23s**, although downstream portable runtime, seeds, tools and extension -outputs restored in milliseconds. Its task hash matched the baseline. -`src/wasix/runtime/moon.yml:129` deliberately disables compiler-output caching -because generated container Makefiles reference `/work`; dependencies still -execute before cached downstream tasks. - -**Owner:** WASIX compilation task/cache contract. **Done when:** an unchanged -compiler input after a docs/workflow commit or manual dispatch gets useful -reuse; changing sources/toolchain/flags cannot reuse incompatible output. -Keep exact-current-source qualification. Do not simply enable Moon caching on -the entire absolute-path compiler tree or accept old release receipts. - -### CI-22 — P1 — Remove duplicate Linux builds from Android extension production - -- [x] Reuse one matching Linux extension-support output across both Android - ABIs, preferably from the existing Linux extension producer. -- [x] Let independent Android compilation proceed before the support input is - needed for packaging; measure bounded overlap against runner CPU/memory. - -**Evidence:** `package_android_target` in -`src/extensions/artifacts/native/tools/package-release-assets.sh:712` first calls -`build_mobile_host_extension_runtime`, then `build_mobile_static_artifacts`. -Both Android jobs build a Linux x64 PostgreSQL runtime with selected extensions; -the separate Linux extension job builds the same class of support output too. - -| Baseline producer | Linux runtime phase | Android archive phase | -| --- | ---: | ---: | -| Android arm64 | 24m49s | 17m18s | -| Android x86_64 | 20m06s | 14m03s | -| Dedicated Linux x64 | 25m24s | — | - -The two Android host phases alone consume **44m55s**. Existing iOS packaging -already overlaps host/device/simulator lanes. Android's host dependency is -real today: the artifact packager copies SQL/control/data **and host dynamic -modules**, alongside the Android static archives. A static-archive-only -replacement would violate the existing package layout; whether every host -module is needed by downstream mobile consumers remains a separate question. - -**Owner:** native extension build/package boundary. **Done when:** both ABIs -consume matching catalog/version/feature support files, target archive and -platform checks pass, and measurements show reduced compute without a longer -Android critical path. Waiting for the whole Linux job before starting Android -compilation mostly trades duplicate compute for serial waiting. First use -compatible existing output/cache; do not add a generic build service or assume -the base runtime and extension-enabled runtime are interchangeable. - -### CI-23 — P1 — Organize WASIX host production around each consuming host - -- [x] Partition Linux consumer dependencies from other AOT/Node-API hosts using - the pattern already used for native producers. -- [x] Prefer keeping same-host AOT and Node-API production/qualification together - where that removes transfers and repeated setup without changing ABI needs. -- [x] Keep all-platform aggregation at the final release/qualification boundary. - -**Evidence:** every Node-API host starts after the whole AOT matrix. Linux-only -regression also waits for Windows. The TS consumer then waits for every Node-API -host. Baseline Linux AOT was ready at minute **67.5**, but the matrix finished -at **78.1**; Linux Node-API finished at **84.6**, but TS started at **91.8**. -These are workflow barriers, not requirements to test Linux bytes. - -**Owner:** WASIX Actions topology + existing task ownership. **Done when:** -Linux proof can finish while Windows/macOS builds continue, all selected hosts -remain mandatory at the final gate, and independent browser-host packaging -starts early (CI-09). Do not claim Rust build reuse across incompatible target -or glibc environments: Linux Node-API uses a compatibility-container boundary. -Avoid creating a separate job for every small task. This finding expands CI-09; -its savings must not be counted twice. - -### CI-24 — P2 — Preserve safe local parallelism after artifact transfer - -- [x] First reduce unnecessary transfer boundaries by colocating dependent - same-host work. Then measure remaining serial local tasks before changing the - existing execution adapter. -- [x] If material, execute only dependency-ready batches in the existing - resolver, keeping transferred producers skipped and their consumers uncached. - -**Evidence:** `.github/scripts/run-planned-moon-job.sh:55` loops through planned -targets with one `MOON_CACHE=off moon run --upstream none` invocation per target -when dependencies were transferred. This preserves ordering but serializes -independent local tasks. The Windows AOT step took **20m38s**, including -extension AOT 7m46s, core AOT 2m25s, pgwire AOT test 3m50s, integration 1m49s, -tools AOT 26s and Rust SDK AOT test 4m11s. - -**Validation rejects a tempting shortcut:** in an isolated pinned-Moon fixture, -`moon run --upstream none a b c` started `c` before its declared `a`/`b` -prerequisites finished. Simply batching every root is incorrect. Moon's -[execution plan](https://moonrepo.dev/docs/guides/exec-plan) also does not provide -an already-completed-producer import primitive; target exclusion is documented -as forthcoming. Do not base this fix on an assumed scheduler feature. - -**Owner:** existing artifact-transfer adapter. **Done when:** the small ordering -fixture passes, transferred producers never rerun, invalid transferred bytes -still fail, and measured wall time improves under actual CPU/Cargo-lock limits. -No new scheduler framework. The total serial duration is not the possible -saving; core saturation can make extra concurrency slower. - -### CI-25 — P2 — Avoid rebuilding iOS to package an unchanged carrier for Android - -- [x] Reuse verified frozen iOS carrier metadata when native iOS inputs and - version have not changed; keep its ownership with the native carrier output. - -**Evidence:** changing the real file -`src/native/sdks/kotlin/oliphaunt/src/androidMain/kotlin/dev/oliphaunt/OliphauntBrokerService.kt` -selects both Android and iOS runtime/ABI jobs. The dependency path is Android -app → React Native npm package → `finalize-runtime-ios-abi`. -`src/native/sdks/react-native/moon.yml:114` and -`tools/stage-release-artifacts.mts` in that project require iOS archive bytes to -generate the package's carrier manifest. This is a genuine integrity dependency, -not merely a redundant `needs` entry. The shared -`src/native/sdks/swift/tools/ios-carrier-manifest.mts:1120` already supports a -`baseCarrierManifest` input for a frozen base carrier. - -**Owner:** native carrier metadata + React Native packaging. **Done when:** an -Android-only change uses the same complete canonical npm package contract -without recompiling unchanged Apple code, while a changed native carrier -regenerates and validates its manifest. Do not omit iOS files from a special -CI-only npm package or weaken archive hashes. No whole-baseline saving claimed: -Android extensions, not iOS runtime metadata, were that run's limiting input. - -### CI-26 — P2 — Remove UI-driver overhead from self-running SDK smoke - -- [x] Make the installed app's exact-launch structured result authoritative on - both platforms, without requiring Maestro to read example status labels. -- [x] Reuse the existing continuous log capture and report validation, with - failure precedence, process/capture failure detection and an overall deadline. - -**Evidence:** `src/examples/native/react-native-expo/maestro/installed-smoke.yaml` -only waits for the passed label and asserts result labels. It performs no SDK -interaction or lifecycle transition; the app runs those tests itself. In -[baseline iOS E2E](https://github.com/f0rr0/oliphaunt/actions/runs/36492755110/job/109183796475), -the app reported all **39 extensions passing at 23:38:12.942**, about eight -seconds after launch. Maestro finished at **23:42:50.918**, another **4m38s** -later. Simulator preparation and uninstall/install accounted for additional -minutes; they are not all removable test execution. - -The `run_maestro_installed_smoke` loop in -`src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh:51` has no -overall deadline while waiting for the driver; the flow timeout does not bound -driver startup. The existing non-Maestro branch is **not a safe drop-in**: it -polls the previous 30 seconds of logs, hardcodes the process name and checks -PASS before failure. Exact-launch continuous capture currently lives inside -the Maestro branch and should become the shared SDK-smoke mechanism. - -**Owner:** installed-app runner, shared with manual replay (CI-16). -**Done when:** real app install/launch, SQL/extensions, ICU/resource and receipt -validation remain; stale PASS, explicit FAIL, crash, dead capture and missing -receipt all fail within a bound. Keep separate lifecycle tests. Retain Maestro -only for a real UI interaction guarantee if one is required. A bare switch to -the old log mode or accepting UI success alone would weaken correctness. - -## Failure history: what to keep and what is already corrected - -| Evidence | Interpretation | Current disposition | -| --- | --- | --- | -| [36523497369 Android](https://github.com/f0rr0/oliphaunt/actions/runs/36523497369/job/109274401218): corrupt NDK archive; same-source rerun succeeded | Confirmed infrastructure/acquisition flake | CI-03 exact-package repair and shared acquisition deadlines implemented; hosted verification pending | -| Public-consumer full local suite failed; unchanged isolated control passed; delayed-start probe failed | Confirmed timing-sensitive test harness | CI-02 corrected; delayed-start and expired-deadline fixtures pass | -| [36523497369 Qualified](https://github.com/f0rr0/oliphaunt/actions/runs/36523497369/job/109397213590): attempt-1 producer receipt rejected on attempt 3 | Qualification bookkeeping defect, not failed SDK behavior | Fixed in #225; preserve same-run/source and reject future attempts | -| [36518236951](https://github.com/f0rr0/oliphaunt/actions/runs/36518236951/job/109245492836): Apple carrier selection lost `PRODUCTS_JSON` and query failure was hidden | Release assembly defect | Fixed in #224 with workflow-step behavioral regression and real artifact replay | -| [36451399513](https://github.com/f0rr0/oliphaunt/actions/runs/36451399513) and [36451108400](https://github.com/f0rr0/oliphaunt/actions/runs/36451108400): missing Moon projects / skipped required WASIX regression | Planner/ownership drift | Immediate defects fixed in #222; structural correction remains CI-05 | -| [36449348921](https://github.com/f0rr0/oliphaunt/actions/runs/36449348921): declared npm carrier version rewrite rejected in `bun.lock` | Release normalization contract omitted valid derived data | Fixed in #220; keep semantic non-version-change rejection | -| [36240045462](https://github.com/f0rr0/oliphaunt/actions/runs/36240045462/job/108405753480): browser `instant` panic during WASIX initdb | Useful real browser execution failure | Later complete candidate passed; do not remove browser execution as flaky | -| [36348549841](https://github.com/f0rr0/oliphaunt/actions/runs/36348549841): Windows `fsync` unresolved symbol; iOS missing `backupDataForJsi:completion:` selector | Real platform compilation defects | Later complete candidate passed; retain platform link/app builds | -| [36345320273](https://github.com/f0rr0/oliphaunt/actions/runs/36345320273): Windows import-library contract unit failures | Useful cheap contract gate | Caught before expensive producers; not evidence of a flaky test | -| [36310991740](https://github.com/f0rr0/oliphaunt/actions/runs/36310991740): WASIX `clang --version` SIGPIPE/source-fingerprint rejection; Android backend exited before ReadyForQuery | Build/runtime failures; log is not enough to classify these as harmless infrastructure | Preserve regression coverage; investigate the exact signature if it recurs; do not add automatic success retries | -| [36227119157](https://github.com/f0rr0/oliphaunt/actions/runs/36227119157/job/108374367397): iOS app preparation/build failed | Earlier app packaging failure | Later candidate passed; the terminal job summary alone does not prove a remaining defect | - -The retry semantics now used by #225 fit GitHub's model: a rerun retains the -original source SHA/ref and can rerun just failed jobs. See -[GitHub's rerun documentation](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/re-run-workflows-and-jobs). -Earlier successful job evidence must remain usable without accepting another -run, source commit, or future attempt. - -## Verification ROI and ordering decisions - -| Verification | Decision | Reason | -| --- | --- | --- | -| Workflow syntax/security and real planner behavior | Keep early | Cheap; actual planning failures justify it. Test final execution, not only intermediate lists | -| Source-only formatting/unit/lint before expensive producers | Keep | Already mostly correct; don't replace with speculative all-platform setup | -| Packed package reopening / clean consumer | Keep | Source tests cannot detect omitted files, bad exports or installed resource discovery | -| Native extension lifecycle plus WASIX lifecycle | Keep | Different runtimes and loading/materialization behavior; not interchangeable | -| Representative AOT execution on all supported hosts | Keep | Portable/Linux success does not prove host machine-code deserialization/execution | -| Exhaustive extension catalog on every AOT host | Do not add | Current Linux exhaustive + per-host representative split is right-sized | -| iOS and Android app builds/E2E | Keep | Platform constraints differ; real app builds caught defects that portable tests did not | -| Maestro reading self-running example status labels | Remove from SDK qualification after CI-26 | No unique SDK interaction; exact-launch app receipts already carry the result. Preserve authoritative failure handling | -| Rust consumer built with stubs, then real embedded carriers | Keep distinct guarantees | Compile/package closure versus resource-free installed execution; the second build was about 24 seconds in the baseline | -| Source tests replayed at publication | Do not reintroduce | Current frozen-candidate publication avoids the old blanket replay | -| Digest/identity/registry checks after artifact transfer or approval | Keep | Validate new bytes or mutable external state; not repeated source qualification | -| PR release qualification + exact merge qualification | Retain current guarantee | Combined sample cost was 1,565.5 runner-minutes, but different source identities are intentional. Reusing compiler outputs is the first optimization, not accepting a PR receipt as main | -| Main push plus manual full dispatch for the same SHA | Avoid operationally when an adequate run already exists | Concurrency serializes non-PR same-SHA runs; it does not deduplicate them. Release already has a missing-qualification resolver | -| Source spelling/prose assertions | No blanket deletion proposed | Reviewed matches were mostly parsed contracts, generated outputs, logs or behavioral results, not proof that arbitrary source strings should stay unchanged | -| Unused summary action | Delete | No caller or unique proof | - -The earlier September audit's blanket problems should not be carried forward -as if still present: main pushes now use affected scope, the focused workflow -PR above ran in about four minutes, release metadata and mutation tests have -separate owners, frozen publication avoids rebuilding the candidate, and the -manual mobile replay is not automatically fired after normal CI. - -## Recommended execution order - -1. **Remove long setup/recompilation tails:** CI-19, CI-06, CI-21 and CI-08. - These use existing builder/cache facilities and attack both long branches. -2. **Stop selecting work that did not change:** CI-20. Prove documentation, - unit-test, SDK-only and compiler-input boundaries with the actual planner. -3. **Shorten the second critical path:** CI-09/23 together. Measure again before - adding Postmaster overlap (CI-10) or changing the transfer adapter (CI-24). -4. **Shorten mobile feedback:** CI-07/22, then CI-26 and CI-25. Distinguish saved - runner time from end-to-end Android/iOS completion time. -5. **In parallel with those priorities, fix correctness:** CI-01 through CI-05 - and CI-11. Coverage and admission defects do not become optional because - they are not timing improvements. -6. **Then routine maintenance:** CI-12 through CI-18. Naming, small validators - and unused setup are worthwhile, but do not lead a wall-time initiative. - -Measure each structural change on a complete full run and a focused PR, with -cold versus compatible-warm caches identified. Reuse the existing job and phase -timestamps; no benchmark service is needed. Compare the final `Required`/ -`Qualified` finish, the new longest chain, setup/compiler/test time and total -runner-minutes. A successful isolated job speedup is insufficient if a newly -introduced prerequisite delays the consumer. The present sample cannot support -a defensible whole-pipeline percentage or p95 promise. - -Defer catalog-test sharding until it becomes a limiting stage. WASIX regression -spent 8m53s in the library's full extension tests and 2m17s in server extension -tests; useful behavior dominates those sections. The existing native shard -pattern is available if required later, but additional WASIX jobs would repeat -Rust setup/compilation and need complete aggregate evidence. Do not remove -extension materialization/restart/restore coverage or shard every small suite. - -Complexity cuts, ranked by recurring maintenance value: **shrink** the manual -WASIX dependency insertion into its existing Moon owner; **shrink** duplicate -evidence mode rules into the existing contract; **shrink** duplicated mobile -execution definitions; **native** use the existing Rust-cache directory mapping; -**delete** the unused summary action. No credible net line/dependency saving is -claimed before implementation; several valuable fixes are configuration or -coverage corrections rather than deletions. - -## Audit validation and limits - -Executed during this audit: - -- Real pinned Moon `query projects`, `query tasks` and `task-graph --json`, the - current CI config mapper, and the actual check/test matrix writer. -- Dependency reachability probe confirming the five SDK tasks are absent from - hosted roots, with explicit workflow-owned Postmaster tasks accounted for. -- Unchanged public-consumer fixture: pass; same fixture with delayed timeout - setup: expected reproduction of the current false failure. -- Pinned Rust-cache cleanup fixture: confirmed deletion of the custom nested - compiler artifact under the current root-workspace mapping. -- Read-only GitHub run/job/step/artifact/cache inspection; selected causal logs - and complete successful baseline timings. -- Second-pass phase attribution across Postmaster portable/host, normal WASIX, - Windows AOT/Node-API, both Android extension ABIs, Linux extension production - and iOS installed-app execution. Compared the slow Postmaster APT layer with - two faster runs instead of treating the slow sample as a fixed saving. -- One-file affected queries for both READMEs, the Postmaster unit test, the APT - installer test and an existing Android Kotlin implementation file; passed - directly affected tasks to the real planner with the expanded project/task - graph. No tracked files were modified to simulate these changes. -- Isolated pinned-Moon artifact-transfer fixture: a/b depend on an imported - producer; c depends on a/b. Confirmed `--upstream none a b c` skips dependency - ordering and fails c, ruling out the naive batch-all replacement. -- Prior same-tree evidence probes: native receipt verified; missing native PASS - markers rejected; incomplete WASIX materialization accepted by table but - rejected by candidate; incomplete product-required plan rejected at release. - -The temporary audit inputs and probes are under -`/tmp/oliphaunt-ci-audit-2026-09-29`; prior receipt probes are under -`/tmp/oliphaunt-evidence-audit`. They are local scratch evidence, not release -artifacts or a new maintained testing framework. GitHub links and exact source -locations above provide the durable provenance. - -The audit itself changed no product code, CI workflow, repository setting, cache -entry or release, and launched no expensive build matrix. The subsequent local -CI/setup correction batch is recorded at the top of this document. -Whole-system inventory does not imply exhaustive line-by-line review of every -test body. No statistical flake rate, runner cost in currency, live registry -publication success, or Vercel configuration correctness is inferred from this -sample. From c25e6b11e1b8b9732ceffafb0022a345daa27338 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Tue, 29 Sep 2026 20:22:53 +0000 Subject: [PATCH 3/9] fix(ci): isolate fixture workspaces and name job components --- .github/scripts/moon-task-capabilities.mts | 36 +++++++++++-------- .../scripts/moon-task-capabilities.test.mts | 23 ++++++++++-- .../resolve-planned-moon-execution.test.sh | 2 ++ 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/.github/scripts/moon-task-capabilities.mts b/.github/scripts/moon-task-capabilities.mts index 993710831..e488bda18 100644 --- a/.github/scripts/moon-task-capabilities.mts +++ b/.github/scripts/moon-task-capabilities.mts @@ -28,6 +28,14 @@ const DISPLAY_WORDS = Object.freeze({ }); const DISPLAY_PARTS = Object.freeze({ 'extension-artifacts-native': 'Native Extension Artifacts', + 'liboliphaunt-native': 'Native Runtime', + 'liboliphaunt-wasix': 'WASIX Runtime', + 'oliphaunt-rust': 'Rust SDK', + 'oliphaunt-kotlin': 'Kotlin SDK', + 'oliphaunt-swift': 'Swift SDK', + 'oliphaunt-react-native': 'React Native SDK', + 'oliphaunt-wasix-rust': 'WASIX Rust SDK', + 'oliphaunt-wasix-ts': 'WASIX TypeScript SDK', }); export const MAX_TARGETS_PER_JOB = 4; @@ -133,22 +141,13 @@ function compareTargets(left, right) { return left.target < right.target ? -1 : left.target > right.target ? 1 : 0; } -function groupRow(targets, index) { +function groupRow(targets) { const first = targets[0]; + const projects = [...new Set(targets.map(({ target }) => target.split(':')[0]))]; return { - label: `${ - [ - first.requires_apple && 'Apple', - first.requires_android_sdk && 'Android', - first.requires_wasmer_llvm && 'WASIX', - first.requires_swift && 'Swift', - first.requires_rust && 'Rust', - first.requires_maintainer_tools && 'Tooling', - first.requires_workspace && 'JavaScript', - ] - .filter(Boolean) - .join(' / ') || 'Source' - } ${index + 1}`, + label: projects + .map((project) => taskLabel(project).replace(/^(?:Oliphaunt|liboliphaunt) /u, '')) + .join(' + '), target_count: targets.length, requires_rust: first.requires_rust, requires_maintainer_tools: first.requires_maintainer_tools, @@ -195,5 +194,12 @@ export function groupTargets(targets, { maxTargets = MAX_TARGETS_PER_JOB } = {}) groups.push(targetsWithSameSetup.slice(index, index + maxTargets)); } } - return groups.map(groupRow); + const rows = groups.map(groupRow); + return rows.map((row, index) => { + if (rows.filter(({ label }) => label === row.label).length === 1) return row; + // A component may have separate setup profiles or span multiple batches. + // Name the distinguishing tasks instead of assigning an opaque shard number. + const tasks = [...new Set(groups[index].map(({ target }) => taskLabel(target.split(':')[1])))]; + return { ...row, label: `${row.label} (${tasks.join(' + ')})` }; + }); } diff --git a/.github/scripts/moon-task-capabilities.test.mts b/.github/scripts/moon-task-capabilities.test.mts index 1892c935e..f7f4f6778 100644 --- a/.github/scripts/moon-task-capabilities.test.mts +++ b/.github/scripts/moon-task-capabilities.test.mts @@ -1,5 +1,5 @@ -import { strict as assert } from 'node:assert'; import { describe, test } from 'bun:test'; +import { strict as assert } from 'node:assert'; import { groupTargets, @@ -71,7 +71,8 @@ describe('Moon task capabilities', () => { groups.map(({ target_count }) => target_count), [1, 1, 1, 4, 4, 1, 2, 1], ); - assert.equal(groups[3].label, 'Source 4'); + assert.equal(groups[3].label, 'Plain (0 + 1 + 2 + 3)'); + assert.equal(groups[6].label, 'Rust'); assert.equal(groups.filter(({ requires_rust }) => requires_rust).length, 1); assert.equal(groups.filter(({ requires_android_sdk }) => requires_android_sdk).length, 1); assert.equal(groups.filter(({ requires_apple }) => requires_apple).length, 1); @@ -84,6 +85,24 @@ describe('Moon task capabilities', () => { assert.deepEqual(selected.sort(), [...taskMap.keys()].sort()); }); + test('names grouped components and distinguishes their task batches without setup labels', () => { + const taskMap = tasks( + { target: 'liboliphaunt-native:test' }, + { target: 'liboliphaunt-wasix:test' }, + { target: 'oliphaunt-kotlin:format-check', tags: ['requires-android-sdk'] }, + { target: 'oliphaunt-kotlin:lint', tags: ['requires-android-sdk', 'requires-rust'] }, + ); + const targets = [...taskMap.values()].map((task) => matrixTarget(task, 'deep', taskMap)); + const labels = (rows) => groupTargets(rows).map(({ label }) => label); + assert.deepEqual(labels(targets), [ + 'Native Runtime + WASIX Runtime', + 'Kotlin SDK (Format Check)', + 'Kotlin SDK (Lint)', + ]); + assert.deepEqual(labels([...targets].reverse()), labels(targets)); + assert.equal(new Set(labels(targets)).size, labels(targets).length); + }); + test('rejects duplicate targets and invalid shard limits', () => { const row = { target: 'repo:check', diff --git a/.github/scripts/resolve-planned-moon-execution.test.sh b/.github/scripts/resolve-planned-moon-execution.test.sh index 0936fc2c0..6e9b98ed1 100644 --- a/.github/scripts/resolve-planned-moon-execution.test.sh +++ b/.github/scripts/resolve-planned-moon-execution.test.sh @@ -87,6 +87,8 @@ touch "$1.done" SH ( cd "$scratch/parallel" + # Moon tasks inherit their parent's root; this is a separate fixture workspace. + export MOON_WORKSPACE_ROOT="$PWD" git init -q git -c user.name=fixture -c user.email=fixture@example.invalid commit -q --allow-empty -m fixture export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"parallel":["fixture:joined"]}' From 6eddc9ab680803e1b2512836d95612d2f4a40c09 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Tue, 29 Sep 2026 20:42:46 +0000 Subject: [PATCH 4/9] fix(ci): initialize scheduling fixture on explicit base branch --- .github/scripts/resolve-planned-moon-execution.test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/scripts/resolve-planned-moon-execution.test.sh b/.github/scripts/resolve-planned-moon-execution.test.sh index 6e9b98ed1..9fc57597b 100644 --- a/.github/scripts/resolve-planned-moon-execution.test.sh +++ b/.github/scripts/resolve-planned-moon-execution.test.sh @@ -89,10 +89,12 @@ SH cd "$scratch/parallel" # Moon tasks inherit their parent's root; this is a separate fixture workspace. export MOON_WORKSPACE_ROOT="$PWD" - git init -q + git init -q --initial-branch=main git -c user.name=fixture -c user.email=fixture@example.invalid commit -q --allow-empty -m fixture export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"parallel":["fixture:joined"]}' export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["fixture:downloaded"]' - MOON_CONCURRENCY=2 bash .github/scripts/run-planned-moon-job.sh parallel + # Exercise CI's base-ref lookup locally as well as on GitHub. + CI=true GITHUB_ACTIONS=true GITHUB_REF=refs/pull/1/merge GITHUB_BASE_REF=main \ + MOON_CONCURRENCY=2 bash .github/scripts/run-planned-moon-job.sh parallel test -f joined.done ) From 23c565d492992d29ef262918bb3b2c7207957e52 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Tue, 29 Sep 2026 21:07:40 +0000 Subject: [PATCH 5/9] fix(ci): scope build environment inputs to their producers --- .github/workflows/ci.yml | 3 ++- tools/ci/affected.mts | 6 +++++- tools/ci/ci-plan-node-products.test.mts | 16 ++++++++++++++++ tools/ci/workflow-moon-transfers.test.mts | 13 +++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6475cbad5..565e7fd79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,6 @@ env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 ACTIONLINT_VERSION: 1.7.12 - ASSET_PROFILE: release WASMER_LLVM_VERSION: "22.1" WASMER_LLVM_LINUX_X64_URL: https://github.com/wasmerio/llvm-custom-builds/releases/download/22.x/llvm-linux-amd64.tar.xz WASMER_LLVM_LINUX_X64_SHA256: 5fb1c687c5e895d517a23e7aabea9ec3557e3a3e33f8a8d3a8d21395157b3906 @@ -2083,6 +2082,8 @@ jobs: if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-runtime') }} runs-on: ubuntu-24.04 timeout-minutes: 360 + env: + ASSET_PROFILE: release steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd diff --git a/tools/ci/affected.mts b/tools/ci/affected.mts index 10870e5e2..cfe704614 100644 --- a/tools/ci/affected.mts +++ b/tools/ci/affected.mts @@ -21,7 +21,11 @@ export function triggeringTaskNames(value = {}) { return Object.entries(value) .filter(([, detail]) => { if (detail === null || Array.isArray(detail) || typeof detail !== 'object') return false; - return detail.other === true || (Array.isArray(detail.files) && detail.files.length > 0); + return ( + detail.other === true || + (Array.isArray(detail.files) && detail.files.length > 0) || + (Array.isArray(detail.env) && detail.env.length > 0) + ); }) .map(([task]) => task) .sort(); diff --git a/tools/ci/ci-plan-node-products.test.mts b/tools/ci/ci-plan-node-products.test.mts index f81d93be0..c52a2ae4b 100644 --- a/tools/ci/ci-plan-node-products.test.mts +++ b/tools/ci/ci-plan-node-products.test.mts @@ -122,6 +122,22 @@ function effects(paths) { }; } +test('environment input changes retain their producer and runtime qualification', () => { + const roots = new Set( + triggeringTaskNames({ + 'liboliphaunt-wasix:compiler-output': { env: ['ASSET_PROFILE'], other: false }, + 'liboliphaunt-wasix:runtime-portable': { + upstream: ['liboliphaunt-wasix:compiler-output'], + other: false, + }, + }), + ); + assert.deepEqual([...roots], ['liboliphaunt-wasix:compiler-output']); + const required = requiredTasksForAffected(roots); + assert(required.has('liboliphaunt-wasix:runtime-portable')); + assert(planJobsForAffected(roots).has('wasix-release-regression')); +}); + test('shared Windows DLL policy changes select native and WASIX packaging consumers', () => { const result = effects(paths.windowsVcRuntimePolicy); for (const target of [ diff --git a/tools/ci/workflow-moon-transfers.test.mts b/tools/ci/workflow-moon-transfers.test.mts index b80c7a170..a8c79a96f 100644 --- a/tools/ci/workflow-moon-transfers.test.mts +++ b/tools/ci/workflow-moon-transfers.test.mts @@ -27,6 +27,19 @@ const tasks = new Map( ]), ); +if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) + test('workflow-wide environment does not force product builds during planning', () => { + for (const task of tasks.values()) { + for (const input of task.inputs ?? []) { + if (typeof input !== 'string' || !input.startsWith('$')) continue; + assert( + !Object.hasOwn(workflow.env ?? {}, input.slice(1)), + `${input} affects ${task.target}; scope it to the consuming build job`, + ); + } + } + }); + if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) test('jobs without a Moon cache have no cacheable local task subtree', () => { for (const [id, job] of Object.entries(workflow.jobs)) { From 6e417aa11983f159a0b3ce1e29775c1ec33c039c Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 30 Sep 2026 00:37:04 +0000 Subject: [PATCH 6/9] fix(wasix): avoid blocking browser reads and restore CI base refs --- .github/workflows/ci.yml | 2 +- src/wasix/browser-host/build-provenance.mts | 2 +- src/wasix/browser-host/build-sdk.sh | 11 +- ...smer-js-run-configured-wasix-process.patch | 3 +- ...virtual-fs-poll-contended-pipe-reads.patch | 109 ++++++++++++++++++ src/wasix/browser-host/source.toml | 7 ++ tools/ci/workflow-moon-transfers.test.mts | 6 + 7 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 src/wasix/browser-host/patches/0036-virtual-fs-poll-contended-pipe-reads.patch diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 565e7fd79..91f798126 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2718,7 +2718,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: - fetch-depth: 1 + fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false diff --git a/src/wasix/browser-host/build-provenance.mts b/src/wasix/browser-host/build-provenance.mts index d3621baa0..5725a92e4 100644 --- a/src/wasix/browser-host/build-provenance.mts +++ b/src/wasix/browser-host/build-provenance.mts @@ -8,7 +8,7 @@ const repositoryRoot = resolve(hostDirectory, '../../..'); const sourceManifestPath = 'src/wasix/browser-host/source.toml'; const buildScriptPath = 'src/wasix/browser-host/build-sdk.sh'; const provenanceScriptPath = 'src/wasix/browser-host/build-provenance.mts'; -const safePatchName = /^\d{4}-wasmer-(?:(?:js|wasix)-)?[a-z0-9-]+\.patch$/u; +const safePatchName = /^\d{4}-(?:wasmer-(?:(?:js|wasix)-)?|virtual-fs-)[a-z0-9-]+\.patch$/u; export async function loadHostBuildContract() { const source = await readFile(resolve(repositoryRoot, sourceManifestPath), 'utf8'); diff --git a/src/wasix/browser-host/build-sdk.sh b/src/wasix/browser-host/build-sdk.sh index 36c9b9620..459d58c16 100755 --- a/src/wasix/browser-host/build-sdk.sh +++ b/src/wasix/browser-host/build-sdk.sh @@ -30,8 +30,9 @@ toml_value() { wasmer_js_version="$(toml_value wasmer-js version)" wasmer_wasix_version="$(toml_value wasmer-wasix version)" wasmer_version="$(toml_value wasmer version)" +virtual_fs_version="$(toml_value virtual-fs version)" -for value in "$wasmer_js_version" "$wasmer_wasix_version" "$wasmer_version"; do +for value in "$wasmer_js_version" "$wasmer_wasix_version" "$wasmer_version" "$virtual_fs_version"; do if [[ -z "$value" ]]; then echo "wasix-ts host build: malformed $source_manifest" >&2 exit 1 @@ -81,6 +82,7 @@ trap cleanup EXIT wasmer_js_dir="$build_root/wasmer-js" wasmer_wasix_dir="$build_root/wasmer-wasix-$wasmer_wasix_version" wasmer_dir="$build_root/wasmer-$wasmer_version" +virtual_fs_dir="$build_root/virtual-fs-$virtual_fs_version" # Use the same bounded transports, exact pins and safe extraction as the other # producers. Only temporary source trees are patched; verified archives persist. @@ -88,7 +90,7 @@ source "$repo_root/src/third-party/tools/fetch-sources.sh" bun - "$source_manifest" "$build_root" <<'JS' import {readFileSync, writeFileSync} from 'node:fs'; const pins = Bun.TOML.parse(readFileSync(process.argv[2], 'utf8')); -for (const name of ['wasmer-js', 'wasmer-wasix', 'wasmer']) { +for (const name of ['wasmer-js', 'wasmer-wasix', 'wasmer', 'virtual-fs']) { const pin = pins[name]; const source = name === 'wasmer-js' ? {name, kind:'git', url:pin.url, branch:'oliphaunt-pinned', commit:pin.commit} @@ -98,7 +100,7 @@ for (const name of ['wasmer-js', 'wasmer-wasix', 'wasmer']) { } JS oliphaunt_acquisition_start 'browser host sources' 1800 -for source_name in wasmer-js wasmer-wasix wasmer; do +for source_name in wasmer-js wasmer-wasix wasmer virtual-fs; do fetch_source "$build_root/$source_name.json" "$build_root" "$target_parent/archives" fetch done actual_wasmer_js_version="$(bun "$repo_root/tools/dev/node-info.mts" package-version "$wasmer_js_dir/package.json")" @@ -120,6 +122,9 @@ while IFS= read -r patch_name; do ????-wasmer-*.patch) patch_dir="$wasmer_dir" ;; + ????-virtual-fs-*.patch) + patch_dir="$virtual_fs_dir" + ;; *) echo "wasix-ts host build: patch target is not declared by its canonical name: $patch_name" >&2 exit 1 diff --git a/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch b/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch index cdae8598d..0fb55c36b 100644 --- a/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch +++ b/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch @@ -11,7 +11,7 @@ index b21b6eb..4a26370 100644 [package.metadata.wasm-pack.profile.release.wasm-bindgen] debug-js-glue = false -@@ -135,9 +135,11 @@ demangle-name-section = false +@@ -135,9 +135,12 @@ demangle-name-section = false dwarf-debug-info = false [package.metadata.wasm-pack.profile.release] @@ -21,6 +21,7 @@ index b21b6eb..4a26370 100644 [patch.crates-io] +wasmer = { path = "../wasmer-6.1.0" } +wasmer-wasix = { path = "../wasmer-wasix-0.601.0" } ++virtual-fs = { path = "../virtual-fs-0.601.0" } #webc = {git = "https://github.com/wasmerio/pirita", branch = "in-memory-manifest"} #virtual-net = { git = "https://github.com/wasmerio/wasmer", branch = "master" } #virtual-fs = { git = "https://github.com/wasmerio/wasmer", branch = "master" } diff --git a/src/wasix/browser-host/patches/0036-virtual-fs-poll-contended-pipe-reads.patch b/src/wasix/browser-host/patches/0036-virtual-fs-poll-contended-pipe-reads.patch new file mode 100644 index 000000000..7b277fd51 --- /dev/null +++ b/src/wasix/browser-host/patches/0036-virtual-fs-poll-contended-pipe-reads.patch @@ -0,0 +1,109 @@ +diff --git a/src/pipe.rs b/src/pipe.rs +--- a/src/pipe.rs ++++ b/src/pipe.rs +@@ -46,6 +46,23 @@ + } + + impl PipeRx { ++ fn poll_receiver<'a>( ++ rx: &'a Mutex, ++ cx: &mut Context<'_>, ++ ) -> Poll> { ++ match rx.try_lock() { ++ Ok(guard) => Poll::Ready(guard), ++ Err(std::sync::TryLockError::WouldBlock) => { ++ // A writer can briefly hold this lock while notifying its interest ++ // handler. Polling must not block the browser's main thread. The ++ // synchronous lock has no waker registration, so poll again. ++ cx.waker().wake_by_ref(); ++ Poll::Pending ++ } ++ Err(error) => panic!("{error}"), ++ } ++ } ++ + // Tries to read from the internal buffer if data is available. + fn try_read_from_buffer( + rx: &mut MutexGuard<'_, PipeReceiver>, +@@ -108,7 +125,7 @@ + ))); + }; + +- let mut rx = rx.lock().unwrap(); ++ let mut rx = std::task::ready!(Self::poll_receiver(rx, cx)); + loop { + { + if let Some(inner_buf) = rx.buffer.as_mut() { +@@ -460,7 +477,7 @@ + ))); + }; + +- let mut rx = rx.lock().unwrap(); ++ let mut rx = std::task::ready!(Self::poll_receiver(rx, cx)); + loop { + if Self::try_read_from_buffer(&mut rx, buf.remaining(), |read_buf| { + buf.put_slice(read_buf); +@@ -601,3 +618,63 @@ + /// Shared version of BidiPipe for situations where you need + /// to emulate the old behaviour of `Pipe` (both send and recv on one channel). + pub type WasiBidirectionalSharedPipePair = ArcFile; ++ ++#[cfg(test)] ++mod contention_tests { ++ use super::*; ++ use std::sync::atomic::{AtomicUsize, Ordering}; ++ use std::task::{Wake, Waker}; ++ ++ #[derive(Default)] ++ struct WakeCount(AtomicUsize); ++ ++ impl Wake for WakeCount { ++ fn wake(self: Arc) { ++ self.0.fetch_add(1, Ordering::SeqCst); ++ } ++ } ++ ++ #[test] ++ fn contended_pipe_reads_yield_and_preserve_data() { ++ let (mut tx, mut rx) = Pipe::new().split(); ++ std::io::Write::write_all(&mut tx, b"abc").unwrap(); ++ let shared = rx.rx.as_ref().unwrap().clone(); ++ let wake_count = Arc::new(WakeCount::default()); ++ let waker = Waker::from(wake_count.clone()); ++ let mut cx = Context::from_waker(&waker); ++ let mut bytes = [0; 2]; ++ let mut buf = tokio::io::ReadBuf::new(&mut bytes); ++ ++ // Deterministic contention: neither poll method may wait for this guard. ++ let guard = shared.lock().unwrap(); ++ assert!(Pin::new(&mut rx).poll_read_ready(&mut cx).is_pending()); ++ assert!(Pin::new(&mut rx).poll_read(&mut cx, &mut buf).is_pending()); ++ assert_eq!(wake_count.0.load(Ordering::SeqCst), 2); ++ assert!(buf.filled().is_empty()); ++ drop(guard); ++ ++ assert!(matches!( ++ Pin::new(&mut rx).poll_read_ready(&mut cx), ++ Poll::Ready(Ok(3)) ++ )); ++ assert!(matches!( ++ Pin::new(&mut rx).poll_read(&mut cx, &mut buf), ++ Poll::Ready(Ok(())) ++ )); ++ assert_eq!(buf.filled(), b"ab"); ++ buf.clear(); ++ assert!(matches!( ++ Pin::new(&mut rx).poll_read(&mut cx, &mut buf), ++ Poll::Ready(Ok(())) ++ )); ++ assert_eq!(buf.filled(), b"c"); ++ buf.clear(); ++ assert!(Pin::new(&mut rx).poll_read(&mut cx, &mut buf).is_pending()); ++ tx.close(); ++ assert!(matches!( ++ Pin::new(&mut rx).poll_read(&mut cx, &mut buf), ++ Poll::Ready(Ok(())) ++ )); ++ assert!(buf.filled().is_empty()); ++ } ++} diff --git a/src/wasix/browser-host/source.toml b/src/wasix/browser-host/source.toml index f2786467f..4c3fecbc7 100644 --- a/src/wasix/browser-host/source.toml +++ b/src/wasix/browser-host/source.toml @@ -21,6 +21,12 @@ version = "6.1.0" url = "https://static.crates.io/crates/wasmer/wasmer-6.1.0.crate" sha256 = "2d85671948f8886a1cc946141c0b688a5617603c103699a5fceeebeb4e75b0b6" +[virtual-fs] +name = "virtual-fs" +version = "0.601.0" +url = "https://static.crates.io/crates/virtual-fs/virtual-fs-0.601.0.crate" +sha256 = "8defc513ce70576bdbb4305ee67c0cba647af18c0995de8f1fa1271b724c9859" + [patches] series = [ "0001-wasmer-js-run-configured-wasix-process.patch", @@ -50,4 +56,5 @@ series = [ # before its dependent first-poll fast path. "0035-wasmer-wasix-preserve-shared-seek-offset.patch", "0033-wasmer-wasix-poll-ready-asyncify-work.patch", + "0036-virtual-fs-poll-contended-pipe-reads.patch", ] diff --git a/tools/ci/workflow-moon-transfers.test.mts b/tools/ci/workflow-moon-transfers.test.mts index a8c79a96f..098ca18ca 100644 --- a/tools/ci/workflow-moon-transfers.test.mts +++ b/tools/ci/workflow-moon-transfers.test.mts @@ -222,6 +222,12 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) const rawTransfers = step.env?.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON; if (rawTransfers === undefined) continue; + const checkout = steps.find((candidate) => candidate.uses?.startsWith('actions/checkout@')); + assert.equal( + checkout?.with?.['fetch-depth'], + 0, + `${workflowJob} needs the base ref for Moon execution`, + ); const plannedJob = run.match(/run-planned-moon-job[.]sh ([a-z0-9-]+)/u)?.[1] ?? (run.includes('collect-wasix-evidence.sh') ? 'wasix-release-regression' : undefined); From 49ed5e47348d68a35fd3f104ca0c0672aefa024e Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 30 Sep 2026 07:08:01 +0000 Subject: [PATCH 7/9] fix(ci): run planned tasks without base history --- .github/actions/setup-moon/action.yml | 2 +- .../resolve-planned-moon-execution.test.sh | 11 +++++++++-- .github/scripts/run-moon-targets.sh | 6 ++++-- .github/scripts/run-moon-targets.test.sh | 16 ++++++++-------- .github/workflows/ci.yml | 9 ++------- .github/workflows/release.yml | 2 +- src/extensions/tools/collect-wasix-evidence.sh | 4 ++-- tools/ci/workflow-moon-transfers.test.mts | 6 ------ 8 files changed, 27 insertions(+), 29 deletions(-) diff --git a/.github/actions/setup-moon/action.yml b/.github/actions/setup-moon/action.yml index d8f3aafc2..d1c14b269 100644 --- a/.github/actions/setup-moon/action.yml +++ b/.github/actions/setup-moon/action.yml @@ -74,7 +74,7 @@ runs: node --version bun --version # This shell probe needs no source comparison or fetched base branch. - .github/scripts/run-moon-targets.sh --base HEAD --head HEAD ci-workflows:verify-bash + .github/scripts/run-moon-targets.sh ci-workflows:verify-bash - name: Save verified tool archives if: ${{ steps.restore_verified_moon_toolchain.outputs.cache-hit != 'true' }} diff --git a/.github/scripts/resolve-planned-moon-execution.test.sh b/.github/scripts/resolve-planned-moon-execution.test.sh index 9fc57597b..be6ed1f88 100644 --- a/.github/scripts/resolve-planned-moon-execution.test.sh +++ b/.github/scripts/resolve-planned-moon-execution.test.sh @@ -57,6 +57,8 @@ cp .github/scripts/{run-planned-moon-job.sh,run-moon-targets.sh,resolve-planned- cat > "$scratch/parallel/.moon/workspace.yml" <<'YAML' projects: fixture: . +vcs: + defaultBranch: main YAML cat > "$scratch/parallel/moon.yml" <<'YAML' id: fixture @@ -87,10 +89,15 @@ touch "$1.done" SH ( cd "$scratch/parallel" + git init -q --initial-branch=fixture + git add . + git -c user.name=fixture -c user.email=fixture@example.invalid commit -q -m fixture + git clone -q --depth 1 "file://$PWD" "$scratch/shallow" + cd "$scratch/shallow" + # Planned execution needs only the candidate, with no main ref or history. # Moon tasks inherit their parent's root; this is a separate fixture workspace. export MOON_WORKSPACE_ROOT="$PWD" - git init -q --initial-branch=main - git -c user.name=fixture -c user.email=fixture@example.invalid commit -q --allow-empty -m fixture + test "$(git rev-parse --is-shallow-repository)" = true export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"parallel":["fixture:joined"]}' export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["fixture:downloaded"]' # Exercise CI's base-ref lookup locally as well as on GitHub. diff --git a/.github/scripts/run-moon-targets.sh b/.github/scripts/run-moon-targets.sh index b6a691e50..ee04ad4c2 100755 --- a/.github/scripts/run-moon-targets.sh +++ b/.github/scripts/run-moon-targets.sh @@ -5,6 +5,8 @@ unset MOON_BASE unset MOON_HEAD moon_bin="${MOON_BIN:-moon}" +# Planning already selected these targets. Execute the candidate without +# requiring a second changed-file comparison or a fetched base branch. if [ "${1:-}" = --matrix ]; then groups="$(bun .github/scripts/select-moon-target-groups.mts)" @@ -14,8 +16,8 @@ if [ "${1:-}" = --matrix ]; then printf '\nSelected Moon tasks:\n\n' >> "$GITHUB_STEP_SUMMARY" printf -- '- `%s`\n' "${target_args[@]}" >> "$GITHUB_STEP_SUMMARY" fi - "$moon_bin" run --upstream "$upstream" "${target_args[@]}" + "$moon_bin" run --base HEAD --head HEAD --upstream "$upstream" "${target_args[@]}" done <<<"$groups" else - exec "$moon_bin" run "$@" + exec "$moon_bin" run --base HEAD --head HEAD "$@" fi diff --git a/.github/scripts/run-moon-targets.test.sh b/.github/scripts/run-moon-targets.test.sh index 179ede0b1..b4dc3c7e3 100644 --- a/.github/scripts/run-moon-targets.test.sh +++ b/.github/scripts/run-moon-targets.test.sh @@ -15,19 +15,19 @@ MOON chmod +x "$MOON_BIN" MOON_BASE=missing-base MOON_HEAD=missing-head \ bash .github/scripts/run-moon-targets.sh ci-workflows:verify-bash -printf 'run ci-workflows:verify-bash\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD ci-workflows:verify-bash\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" export MOON_TARGET_MATRIX_JSON='{"include":[{"target":"sdk:b"},{"target":"sdk:a"},{"target":"sdk:a"},{"target":"native:c","upstream":"none"}]}' bash .github/scripts/run-moon-targets.sh --matrix -printf 'run --upstream deep sdk:a sdk:b\nrun --upstream none native:c\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD --upstream deep sdk:a sdk:b\nrun --base HEAD --head HEAD --upstream none native:c\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" if MOON_TEST_EXIT=7 bash .github/scripts/run-moon-targets.sh --matrix; then echo 'Moon failure was ignored' >&2 exit 1 fi -printf 'run --upstream deep sdk:a sdk:b\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD --upstream deep sdk:a sdk:b\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" if MOON_TARGET_MATRIX_JSON='{"include":[{"target":"sdk:a"},{"target":"-bad target"}]}' \ @@ -45,7 +45,7 @@ export MOON_TEST_GRAPH="$fixture/graph.json" cat >"$MOON_BIN" <<'MOON' #!/usr/bin/env bash if [ "$1" = task-graph ]; then cat "$MOON_TEST_GRAPH"; exit; fi -if [[ "$*" == 'run --upstream none '* && "${MOON_CACHE:-}" != off ]]; then +if [[ "$*" == 'run --base HEAD --head HEAD --upstream none '* && "${MOON_CACHE:-}" != off ]]; then echo 'transferred consumers must execute without incomplete dependency cache keys' >&2 exit 9 fi @@ -55,26 +55,26 @@ MOON export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"fixture":["sdk:a-consumer","sdk:z-package"]}' export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["native:ios"]' bash .github/scripts/run-planned-moon-job.sh fixture -printf 'run sdk:build\nrun --upstream none sdk:z-package\nrun --upstream none sdk:a-consumer\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD sdk:build\nrun --base HEAD --head HEAD --upstream none sdk:z-package\nrun --base HEAD --head HEAD --upstream none sdk:a-consumer\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" if MOON_FAIL_TARGET=sdk:z-package bash .github/scripts/run-planned-moon-job.sh fixture; then echo 'failed package reached its consumer' >&2 exit 1 fi -printf 'run sdk:build\nrun --upstream none sdk:z-package\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD sdk:build\nrun --base HEAD --head HEAD --upstream none sdk:z-package\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" # A narrowed consumer still needs its intermediate package, but never its transferred producer. : >"$MOON_CALLS" export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"fixture":["sdk:a-consumer"]}' bash .github/scripts/run-planned-moon-job.sh fixture -printf 'run sdk:build\nrun --upstream none sdk:z-package\nrun --upstream none sdk:a-consumer\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD sdk:build\nrun --base HEAD --head HEAD --upstream none sdk:z-package\nrun --base HEAD --head HEAD --upstream none sdk:a-consumer\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" if MOON_FAIL_TARGET=sdk:z-package bash .github/scripts/run-planned-moon-job.sh fixture; then echo 'failed intermediate package reached its consumer' >&2 exit 1 fi -printf 'run sdk:build\nrun --upstream none sdk:z-package\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD sdk:build\nrun --base HEAD --head HEAD --upstream none sdk:z-package\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91f798126..1843c01db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2704,7 +2704,7 @@ jobs: retention-days: 90 wasix-release-regression: - name: E2E / WASIX Release Regression + name: E2E / WASIX Extension Lifecycle needs: - affected - extension-artifacts-wasix @@ -2718,7 +2718,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: - fetch-depth: 0 + fetch-depth: 1 ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false @@ -2819,11 +2819,6 @@ jobs: CI_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT: ${{ github.workspace }}/target/extension-artifacts run: | - actual_sha="$(git rev-parse HEAD)" - if [[ "$actual_sha" != "$CI_HEAD_SHA" ]]; then - echo "checked-out candidate $actual_sha does not match requested SHA $CI_HEAD_SHA" >&2 - exit 1 - fi run_id="$(date -u +%Y-%m-%dT%H%M%SZ)-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${CI_HEAD_SHA}" run_path="src/extensions/evidence/runs/$run_id.json" echo "run_path=$run_path" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86b38f32e..5996a5663 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -360,7 +360,7 @@ jobs: "$RELEASE_HEAD_SHA" \ target/release-candidate/wasix-evidence \ --run-id "$CI_RUN_ID" \ - --job "E2E / WASIX Release Regression" \ + --job "E2E / WASIX Extension Lifecycle" \ --artifact wasix-release-regression-evidence - name: Download required exact-SHA native evidence if: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).has_release_changes == 'true' && inputs.approval_run_id == '' && fromJSON(needs.plan-candidate.outputs.release_plan).requires_native_extension_lifecycle_evidence == 'true' }} diff --git a/src/extensions/tools/collect-wasix-evidence.sh b/src/extensions/tools/collect-wasix-evidence.sh index a24e3e002..7acb1d473 100755 --- a/src/extensions/tools/collect-wasix-evidence.sh +++ b/src/extensions/tools/collect-wasix-evidence.sh @@ -36,7 +36,7 @@ trap 'rm -rf "$OLIPHAUNT_EXTENSION_EVIDENCE_DIR"' EXIT .github/scripts/run-planned-moon-job.sh wasix-release-regression bash src/extensions/tools/check-extension-model.sh \ --record-wasix-evidence-run "$run_id" \ - --observed-at "$observed_at" -bash src/extensions/tools/check-extension-model.sh --check --require-current-evidence + --observed-at "$observed_at" \ + --require-current-evidence echo "recorded immutable WASIX extension evidence run: $run_id" diff --git a/tools/ci/workflow-moon-transfers.test.mts b/tools/ci/workflow-moon-transfers.test.mts index 098ca18ca..a8c79a96f 100644 --- a/tools/ci/workflow-moon-transfers.test.mts +++ b/tools/ci/workflow-moon-transfers.test.mts @@ -222,12 +222,6 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) const rawTransfers = step.env?.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON; if (rawTransfers === undefined) continue; - const checkout = steps.find((candidate) => candidate.uses?.startsWith('actions/checkout@')); - assert.equal( - checkout?.with?.['fetch-depth'], - 0, - `${workflowJob} needs the base ref for Moon execution`, - ); const plannedJob = run.match(/run-planned-moon-job[.]sh ([a-z0-9-]+)/u)?.[1] ?? (run.includes('collect-wasix-evidence.sh') ? 'wasix-release-regression' : undefined); From be6339871d137be5aa5a35dd4a071eede6be2acb Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 30 Sep 2026 23:00:52 +0000 Subject: [PATCH 8/9] fix(ci): keep runner regression compatible with Bash 3.2 --- .github/scripts/run-moon-targets.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/run-moon-targets.test.sh b/.github/scripts/run-moon-targets.test.sh index b4dc3c7e3..adc656c3a 100644 --- a/.github/scripts/run-moon-targets.test.sh +++ b/.github/scripts/run-moon-targets.test.sh @@ -5,7 +5,7 @@ trap 'rm -rf "$fixture"' EXIT export MOON_BIN="$fixture/moon" MOON_CALLS="$fixture/calls" cat >"$MOON_BIN" <<'MOON' #!/usr/bin/env bash -if [[ -v MOON_BASE || -v MOON_HEAD ]]; then +if [[ -n "${MOON_BASE+x}${MOON_HEAD+x}" ]]; then echo 'explicit task execution inherited affectedness revisions' >&2 exit 8 fi From f9e32827139ba2527c09760830a72e0f667e02e2 Mon Sep 17 00:00:00 2001 From: Sid Jain Date: Wed, 30 Sep 2026 23:01:50 +0000 Subject: [PATCH 9/9] fix(wasix): retain wakeups before browser workers park --- src/wasix/browser-host/build-provenance.mts | 2 +- src/wasix/browser-host/build-sdk.sh | 11 ++- ...smer-js-run-configured-wasix-process.patch | 3 +- ...0037-virtual-mio-retain-inline-wakes.patch | 88 +++++++++++++++++++ src/wasix/browser-host/source.toml | 7 ++ 5 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 src/wasix/browser-host/patches/0037-virtual-mio-retain-inline-wakes.patch diff --git a/src/wasix/browser-host/build-provenance.mts b/src/wasix/browser-host/build-provenance.mts index 5725a92e4..ee54b2b9e 100644 --- a/src/wasix/browser-host/build-provenance.mts +++ b/src/wasix/browser-host/build-provenance.mts @@ -8,7 +8,7 @@ const repositoryRoot = resolve(hostDirectory, '../../..'); const sourceManifestPath = 'src/wasix/browser-host/source.toml'; const buildScriptPath = 'src/wasix/browser-host/build-sdk.sh'; const provenanceScriptPath = 'src/wasix/browser-host/build-provenance.mts'; -const safePatchName = /^\d{4}-(?:wasmer-(?:(?:js|wasix)-)?|virtual-fs-)[a-z0-9-]+\.patch$/u; +const safePatchName = /^\d{4}-(?:wasmer-(?:(?:js|wasix)-)?|virtual-(?:fs|mio)-)[a-z0-9-]+\.patch$/u; export async function loadHostBuildContract() { const source = await readFile(resolve(repositoryRoot, sourceManifestPath), 'utf8'); diff --git a/src/wasix/browser-host/build-sdk.sh b/src/wasix/browser-host/build-sdk.sh index 459d58c16..f83c13bce 100755 --- a/src/wasix/browser-host/build-sdk.sh +++ b/src/wasix/browser-host/build-sdk.sh @@ -31,8 +31,9 @@ wasmer_js_version="$(toml_value wasmer-js version)" wasmer_wasix_version="$(toml_value wasmer-wasix version)" wasmer_version="$(toml_value wasmer version)" virtual_fs_version="$(toml_value virtual-fs version)" +virtual_mio_version="$(toml_value virtual-mio version)" -for value in "$wasmer_js_version" "$wasmer_wasix_version" "$wasmer_version" "$virtual_fs_version"; do +for value in "$wasmer_js_version" "$wasmer_wasix_version" "$wasmer_version" "$virtual_fs_version" "$virtual_mio_version"; do if [[ -z "$value" ]]; then echo "wasix-ts host build: malformed $source_manifest" >&2 exit 1 @@ -83,6 +84,7 @@ wasmer_js_dir="$build_root/wasmer-js" wasmer_wasix_dir="$build_root/wasmer-wasix-$wasmer_wasix_version" wasmer_dir="$build_root/wasmer-$wasmer_version" virtual_fs_dir="$build_root/virtual-fs-$virtual_fs_version" +virtual_mio_dir="$build_root/virtual-mio-$virtual_mio_version" # Use the same bounded transports, exact pins and safe extraction as the other # producers. Only temporary source trees are patched; verified archives persist. @@ -90,7 +92,7 @@ source "$repo_root/src/third-party/tools/fetch-sources.sh" bun - "$source_manifest" "$build_root" <<'JS' import {readFileSync, writeFileSync} from 'node:fs'; const pins = Bun.TOML.parse(readFileSync(process.argv[2], 'utf8')); -for (const name of ['wasmer-js', 'wasmer-wasix', 'wasmer', 'virtual-fs']) { +for (const name of ['wasmer-js', 'wasmer-wasix', 'wasmer', 'virtual-fs', 'virtual-mio']) { const pin = pins[name]; const source = name === 'wasmer-js' ? {name, kind:'git', url:pin.url, branch:'oliphaunt-pinned', commit:pin.commit} @@ -100,7 +102,7 @@ for (const name of ['wasmer-js', 'wasmer-wasix', 'wasmer', 'virtual-fs']) { } JS oliphaunt_acquisition_start 'browser host sources' 1800 -for source_name in wasmer-js wasmer-wasix wasmer virtual-fs; do +for source_name in wasmer-js wasmer-wasix wasmer virtual-fs virtual-mio; do fetch_source "$build_root/$source_name.json" "$build_root" "$target_parent/archives" fetch done actual_wasmer_js_version="$(bun "$repo_root/tools/dev/node-info.mts" package-version "$wasmer_js_dir/package.json")" @@ -125,6 +127,9 @@ while IFS= read -r patch_name; do ????-virtual-fs-*.patch) patch_dir="$virtual_fs_dir" ;; + ????-virtual-mio-*.patch) + patch_dir="$virtual_mio_dir" + ;; *) echo "wasix-ts host build: patch target is not declared by its canonical name: $patch_name" >&2 exit 1 diff --git a/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch b/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch index 0fb55c36b..1416e7e80 100644 --- a/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch +++ b/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch @@ -11,7 +11,7 @@ index b21b6eb..4a26370 100644 [package.metadata.wasm-pack.profile.release.wasm-bindgen] debug-js-glue = false -@@ -135,9 +135,12 @@ demangle-name-section = false +@@ -135,9 +135,13 @@ demangle-name-section = false dwarf-debug-info = false [package.metadata.wasm-pack.profile.release] @@ -22,6 +22,7 @@ index b21b6eb..4a26370 100644 +wasmer = { path = "../wasmer-6.1.0" } +wasmer-wasix = { path = "../wasmer-wasix-0.601.0" } +virtual-fs = { path = "../virtual-fs-0.601.0" } ++virtual-mio = { path = "../virtual-mio-0.601.0" } #webc = {git = "https://github.com/wasmerio/pirita", branch = "in-memory-manifest"} #virtual-net = { git = "https://github.com/wasmerio/wasmer", branch = "master" } #virtual-fs = { git = "https://github.com/wasmerio/wasmer", branch = "master" } diff --git a/src/wasix/browser-host/patches/0037-virtual-mio-retain-inline-wakes.patch b/src/wasix/browser-host/patches/0037-virtual-mio-retain-inline-wakes.patch new file mode 100644 index 000000000..f9133889b --- /dev/null +++ b/src/wasix/browser-host/patches/0037-virtual-mio-retain-inline-wakes.patch @@ -0,0 +1,88 @@ +diff --git a/src/waker.rs b/src/waker.rs +--- a/src/waker.rs ++++ b/src/waker.rs +@@ -1,31 +1,29 @@ + #![allow(dead_code, unused)] + use std::{ +- sync::{Arc, Condvar, Mutex}, ++ sync::{atomic::{AtomicBool, Ordering}, Arc}, ++ thread::{self, Thread}, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, + }; + + use futures::Future; + + pub struct InlineWaker { +- lock: Mutex<()>, +- condvar: Condvar, ++ thread: Thread, ++ notified: AtomicBool, + } + impl InlineWaker { + pub fn new() -> Arc { + Arc::new(Self { +- lock: Mutex::new(()), +- condvar: Condvar::new(), ++ thread: thread::current(), ++ notified: AtomicBool::new(false), + }) + } + + fn wake_now(&self) { +- // Note: This guard should be there to prevent race conditions however in the +- // browser it causes a lock up - some strange browser issue. What I suspect +- // is that the Condvar::wait call is not releasing the mutex lock +- #[cfg(not(feature = "js"))] +- let _guard = self.lock.lock().unwrap(); +- +- self.condvar.notify_all(); ++ // Retain a wake delivered during poll, before the worker parks. ++ // No synchronous mutex may block the browser thread delivering a wake. ++ self.notified.store(true, Ordering::Release); ++ self.thread.unpark(); + } + + pub fn as_waker(self: &Arc) -> Waker { +@@ -43,10 +41,11 @@ + // We loop waiting for the waker to be woken, then we poll again + let mut task = Box::pin(task); + loop { +- let lock = inline_waker.lock.lock().unwrap(); + match task.as_mut().poll(&mut cx) { + Poll::Pending => { +- inline_waker.condvar.wait(lock).ok(); ++ while !inline_waker.notified.swap(false, Ordering::Acquire) { ++ thread::park(); ++ } + } + Poll::Ready(ret) => { + return ret; +@@ -75,3 +74,28 @@ + |s| drop(Arc::from_raw(s as *const InlineWaker)), // decrease refcount + ) + }; ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use std::{future::poll_fn, sync::mpsc, time::Duration}; ++ ++ #[test] ++ fn wake_during_poll_is_retained_until_the_next_poll() { ++ let (tx, rx) = mpsc::channel(); ++ thread::spawn(move || { ++ let mut polled = false; ++ InlineWaker::block_on(poll_fn(|cx| { ++ if polled { ++ Poll::Ready(()) ++ } else { ++ polled = true; ++ cx.waker().wake_by_ref(); ++ Poll::Pending ++ } ++ })); ++ tx.send(()).unwrap(); ++ }); ++ rx.recv_timeout(Duration::from_secs(2)).unwrap(); ++ } ++} diff --git a/src/wasix/browser-host/source.toml b/src/wasix/browser-host/source.toml index 4c3fecbc7..abb31cc86 100644 --- a/src/wasix/browser-host/source.toml +++ b/src/wasix/browser-host/source.toml @@ -27,6 +27,12 @@ version = "0.601.0" url = "https://static.crates.io/crates/virtual-fs/virtual-fs-0.601.0.crate" sha256 = "8defc513ce70576bdbb4305ee67c0cba647af18c0995de8f1fa1271b724c9859" +[virtual-mio] +name = "virtual-mio" +version = "0.601.0" +url = "https://static.crates.io/crates/virtual-mio/virtual-mio-0.601.0.crate" +sha256 = "a14806a3f25b70315764760c376d9f68937f5467013891bc7497a33d24105097" + [patches] series = [ "0001-wasmer-js-run-configured-wasix-process.patch", @@ -57,4 +63,5 @@ series = [ "0035-wasmer-wasix-preserve-shared-seek-offset.patch", "0033-wasmer-wasix-poll-ready-asyncify-work.patch", "0036-virtual-fs-poll-contended-pipe-reads.patch", + "0037-virtual-mio-retain-inline-wakes.patch", ]