diff --git a/.github/actions/collect-ci-summary/action.yml b/.github/actions/collect-ci-summary/action.yml deleted file mode 100644 index 0fcacf360..000000000 --- a/.github/actions/collect-ci-summary/action.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Collect CI summary -description: Append a small Moon/release summary to the GitHub step summary. - -runs: - using: composite - steps: - - name: Write CI summary - shell: bash - run: | - { - echo "## Oliphaunt CI" - echo - echo "- Moon projects: \`moon query projects\`" - echo "- Moon tasks: \`moon query tasks\`" - echo "- Release plan: \`bash tools/release/release-plan.sh --from-product-tags --head-ref \`" - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/run-mobile-e2e/action.yml b/.github/actions/run-mobile-e2e/action.yml new file mode 100644 index 000000000..8798723bd --- /dev/null +++ b/.github/actions/run-mobile-e2e/action.yml @@ -0,0 +1,111 @@ +name: Run installed mobile app +description: Shared platform setup, exact-launch SDK smoke, and reports for CI and replay. +inputs: + platform: + required: true + description: android or ios + source-sha: + required: true + description: Exact source SHA of the downloaded app +runs: + using: composite + steps: + - name: Set up Node and Bun + uses: ./.github/actions/setup-node-bun + + - name: Reclaim Android emulator disk + if: ${{ inputs.platform == 'android' }} + shell: bash + run: bash .github/scripts/reclaim-android-mobile-build-disk.sh + + - name: Set up Android + if: ${{ inputs.platform == 'android' }} + uses: ./.github/actions/setup-android + with: + gradle-cache: "false" + native-tools: "false" + + - name: List Android app artifact + if: ${{ inputs.platform == 'android' }} + shell: bash + run: find target/mobile-build/react-native/android -maxdepth 2 -type f -print + + - name: Provision runner KVM access + if: ${{ inputs.platform == 'android' }} + shell: bash + run: | + test -c /dev/kvm + if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then + sudo chmod a+rw /dev/kvm + fi + + - name: Start Android emulator + if: ${{ inputs.platform == 'android' }} + env: + OLIPHAUNT_ANDROID_EMULATOR_API: "35" + OLIPHAUNT_ANDROID_EMULATOR_DISK_HEADROOM_MB: "2048" + OLIPHAUNT_ANDROID_EMULATOR_PARTITION_SIZE_MB: "6144" + shell: bash + run: tools/ci/start-android-emulator-ci.sh + + - name: Run Android installed-app E2E + if: ${{ inputs.platform == 'android' }} + env: + CI_HEAD_SHA: ${{ inputs.source-sha }} + OLIPHAUNT_EXPO_ANDROID_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/android + OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE: release + OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE: "0" + shell: bash + run: bash src/native/sdks/react-native/tools/mobile-e2e.sh android + + - name: Upload Android E2E reports + if: ${{ always() && inputs.platform == 'android' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: react-native-mobile-android-e2e-reports + path: | + target/mobile/react-native/android-e2e/reports + target/mobile/react-native/android-e2e/logs + ${{ runner.temp }}/oliphaunt-android-emulator.log + if-no-files-found: ignore + - name: Set up Apple + if: ${{ inputs.platform == 'ios' }} + uses: ./.github/actions/setup-apple + with: + install-build-dependencies: "false" + + - name: Verify and extract iOS app artifact + if: ${{ inputs.platform == 'ios' }} + shell: bash + run: | + rm -rf target/mobile-build/react-native/ios + bash src/native/sdks/react-native/tools/ios-app-transport.sh verify-extract \ + --transport-dir target/mobile-build/react-native/ios-transport \ + --output-dir target/mobile-build/react-native/ios + + - name: List iOS app artifact + if: ${{ inputs.platform == 'ios' }} + shell: bash + run: find target/mobile-build/react-native/ios -maxdepth 2 -print + + - name: Run iOS installed-app E2E + if: ${{ inputs.platform == 'ios' }} + env: + CI_HEAD_SHA: ${{ inputs.source-sha }} + OLIPHAUNT_EXPO_IOS_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/ios + OLIPHAUNT_EXPO_IOS_CONFIGURATION: Release + OLIPHAUNT_EXPO_IOS_SDK: iphonesimulator + OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE: "0" + shell: bash + run: bash src/native/sdks/react-native/tools/mobile-e2e.sh ios + + - name: Upload iOS E2E reports + if: ${{ always() && inputs.platform == 'ios' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: react-native-mobile-ios-e2e-reports + path: | + target/mobile/react-native/ios-e2e/reports + target/mobile/react-native/ios-e2e/logs + target/mobile/react-native/ios-e2e/*.log + if-no-files-found: ignore diff --git a/.github/actions/setup-android/action.yml b/.github/actions/setup-android/action.yml index 1eb37ab36..8d85d3133 100644 --- a/.github/actions/setup-android/action.yml +++ b/.github/actions/setup-android/action.yml @@ -2,18 +2,12 @@ name: Set up Android description: Set up Java and expose Android SDK paths for Gradle/Expo jobs. inputs: - ndk-version: - description: Android NDK side-by-side version required by native SDK builds. - required: false - default: "27.0.12077973" - cmake-version: - description: Android CMake version required by native SDK builds. - required: false - default: "3.22.1" - compile-sdk: - description: Android platform API level used by SDK checks. - required: false - default: "36" + native-tools: + description: Install NDK and CMake for native compilation. + default: "true" + expo: + description: Also provision the Expo app NDK before Gradle runs. + default: "false" native-ccache: description: Whether to install and configure ccache for native Android C/C++ builds. required: false @@ -109,12 +103,21 @@ runs: ccache --set-config=max_size=2G ccache --zero-stats + - name: Set up macOS acquisition timer + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + . tools/dev/acquisition.sh + if ! oliphaunt_acquisition_timeout >/dev/null 2>&1; then + HOMEBREW_NO_AUTO_UPDATE=1 brew install coreutils + fi + oliphaunt_acquisition_timeout >/dev/null + - name: Configure Android SDK shell: bash env: - NDK_VERSION: ${{ inputs.ndk-version }} - CMAKE_VERSION: ${{ inputs.cmake-version }} - COMPILE_SDK: ${{ inputs.compile-sdk }} + NATIVE_TOOLS: ${{ inputs.native-tools }} + EXPO: ${{ inputs.expo }} run: | # zizmor: ignore[github-env] Android SDK paths are runner-owned or HOME-scoped and validated before export. set -euo pipefail if [[ -z "${ANDROID_HOME:-}" ]]; then @@ -128,16 +131,12 @@ runs: export ANDROID_HOME="$HOME/android-sdk" fi fi - tools/dev/setup-android-sdk.sh \ - --sdk-root "$ANDROID_HOME" \ - --ndk-version "$NDK_VERSION" \ - --cmake-version "$CMAKE_VERSION" \ - --compile-sdk "$COMPILE_SDK" + args=(--sdk-root "$ANDROID_HOME" --native-tools "$NATIVE_TOOLS") + case "$EXPO" in true) args+=(--expo) ;; false) ;; *) exit 1 ;; esac + tools/dev/setup-android-sdk.sh "${args[@]}" echo "ANDROID_HOME=${ANDROID_HOME}" >> "$GITHUB_ENV" echo "ANDROID_SDK_ROOT=${ANDROID_HOME}" >> "$GITHUB_ENV" - echo "ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/${NDK_VERSION}" >> "$GITHUB_ENV" echo "${ANDROID_HOME}/cmdline-tools/latest/bin" >> "$GITHUB_PATH" echo "${ANDROID_HOME}/platform-tools" >> "$GITHUB_PATH" echo "ANDROID_HOME=${ANDROID_HOME}" - echo "ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/${NDK_VERSION}" diff --git a/.github/actions/setup-deno/action.yml b/.github/actions/setup-deno/action.yml index dd2097672..57a0fb52a 100644 --- a/.github/actions/setup-deno/action.yml +++ b/.github/actions/setup-deno/action.yml @@ -3,9 +3,9 @@ description: Install the pinned Deno toolchain for TypeScript runtime checks. inputs: deno-version: - description: Deno version. + description: Optional assertion against the repository Deno pin. required: false - default: "v2.8.1" + default: "" outputs: execution-envelope: diff --git a/.github/actions/setup-maestro/action.yml b/.github/actions/setup-maestro/action.yml deleted file mode 100644 index 332bd595c..000000000 --- a/.github/actions/setup-maestro/action.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Set up Maestro -description: Install the pinned open-source Maestro CLI for local emulator/simulator E2E. - -runs: - using: composite - steps: - - name: Set up Java - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a - with: - distribution: temurin - java-version: "17" - - - name: Install Maestro CLI - shell: bash - run: tools/dev/setup-maestro.sh diff --git a/.github/actions/setup-moon/action.yml b/.github/actions/setup-moon/action.yml index 923b8993d..d1c14b269 100644 --- a/.github/actions/setup-moon/action.yml +++ b/.github/actions/setup-moon/action.yml @@ -2,6 +2,10 @@ name: Set up Moon description: Install verified Node.js, Moon, and Bun binaries and optionally hydrate JavaScript workspace dependencies. inputs: + task-cache: + description: Restore/save Moon task outputs; disable for planning and uncached finalizers. + required: false + default: "true" install-workspace: description: Install Bun workspace dependencies for JavaScript-family tasks. required: false @@ -17,7 +21,7 @@ runs: path: | ${{ runner.temp }}/oliphaunt-moon-toolchain ${{ runner.temp }}/oliphaunt-pinned-tools - key: verified-moon-toolchain-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.moon/toolchains.yml', '.github/actions/setup-moon/action.yml', '.github/actions/setup-node-bun/action.yml', '.github/actions/setup-moon/install-pinned-toolchain.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/dev/moon-cli.toml', 'tools/dev/moon-plugins.toml', 'tools/dev/proto.toml', 'tools/dev/bun.toml', 'tools/dev/install-pinned-js-runtime.sh', 'tools/dev/extract-pinned-zip.sh', 'tools/packaging/portable-archive.mts', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh') }} + key: verified-moon-toolchain-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.moon/toolchains.yml', '.github/actions/setup-moon/action.yml', '.github/actions/setup-node-bun/action.yml', '.github/actions/setup-moon/install-pinned-toolchain.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/dev/moon-cli.toml', 'tools/dev/moon-plugins.toml', 'tools/dev/proto.toml', 'tools/dev/bun.toml', 'tools/dev/install-pinned-js-runtime.sh', 'tools/dev/extract-pinned-zip.sh', 'tools/packaging/portable-archive.mts', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} - name: Set up exact Node.js and Bun uses: ./.github/actions/setup-node-bun @@ -70,7 +74,7 @@ runs: node --version bun --version # This shell probe needs no source comparison or fetched base branch. - .github/scripts/run-moon-targets.sh --base HEAD --head HEAD ci-workflows:verify-bash + .github/scripts/run-moon-targets.sh ci-workflows:verify-bash - name: Save verified tool archives if: ${{ steps.restore_verified_moon_toolchain.outputs.cache-hit != 'true' }} @@ -80,7 +84,7 @@ runs: path: | ${{ runner.temp }}/oliphaunt-moon-toolchain ${{ runner.temp }}/oliphaunt-pinned-tools - key: verified-moon-toolchain-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.moon/toolchains.yml', '.github/actions/setup-moon/action.yml', '.github/actions/setup-node-bun/action.yml', '.github/actions/setup-moon/install-pinned-toolchain.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/dev/moon-cli.toml', 'tools/dev/moon-plugins.toml', 'tools/dev/proto.toml', 'tools/dev/bun.toml', 'tools/dev/install-pinned-js-runtime.sh', 'tools/dev/extract-pinned-zip.sh', 'tools/packaging/portable-archive.mts', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh') }} + key: verified-moon-toolchain-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.moon/toolchains.yml', '.github/actions/setup-moon/action.yml', '.github/actions/setup-node-bun/action.yml', '.github/actions/setup-moon/install-pinned-toolchain.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/dev/moon-cli.toml', 'tools/dev/moon-plugins.toml', 'tools/dev/proto.toml', 'tools/dev/bun.toml', 'tools/dev/install-pinned-js-runtime.sh', 'tools/dev/extract-pinned-zip.sh', 'tools/packaging/portable-archive.mts', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} - name: Install workspace dependencies if: ${{ inputs.install-workspace == 'true' }} @@ -88,6 +92,7 @@ runs: run: bun install --frozen-lockfile - name: Restore Moon task outputs + if: ${{ inputs.task-cache == 'true' }} uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: # casOutputsCache uses these directories; hashes/ is diagnostic metadata. diff --git a/.github/actions/setup-moon/install-pinned-node.sh b/.github/actions/setup-moon/install-pinned-node.sh index 7b9be6984..ac04fa343 100755 --- a/.github/actions/setup-moon/install-pinned-node.sh +++ b/.github/actions/setup-moon/install-pinned-node.sh @@ -33,6 +33,8 @@ done # shellcheck source=tools/dev/curl-platform-flags.sh . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "Node.js bootstrap" 300 manifest_value() { @@ -204,15 +206,14 @@ if [ "$archive_valid" != "1" ]; then curl_args=( --fail --location --silent --show-error --proto '=https' --proto-redir '=https' --tlsv1.2 - --retry 5 --retry-all-errors --retry-connrefused --retry-delay 2 --retry-max-time 300 - --connect-timeout 20 --max-time 300 --speed-limit 1024 --speed-time 30 + --connect-timeout 20 --speed-limit 1024 --speed-time 30 --remove-on-error --max-filesize "$archive_bytes" --output "$partial" ) if [ -n "$curl_tls_flag" ]; then curl_args+=("$curl_tls_flag") fi curl_args+=("$url") - if ! "$curl_command" "${curl_args[@]}"; then + if ! oliphaunt_acquisition_curl 300 6 2 "$curl_command" "${curl_args[@]}"; then rm -f "$partial" fail "could not download pinned Node.js archive $url" fi diff --git a/.github/actions/setup-moon/install-pinned-node.test.sh b/.github/actions/setup-moon/install-pinned-node.test.sh index cc226c37d..ce579367c 100755 --- a/.github/actions/setup-moon/install-pinned-node.test.sh +++ b/.github/actions/setup-moon/install-pinned-node.test.sh @@ -4,6 +4,10 @@ set -euo pipefail root="$(git rev-parse --show-toplevel)" installer="$root/.github/actions/setup-moon/install-pinned-node.sh" work="$(mktemp -d)" +mkdir -p "$work/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$work/no-delay/sleep" +chmod +x "$work/no-delay/sleep" +export PATH="$work/no-delay:$PATH" trap 'rm -rf "$work"' EXIT mkdir -p "$work/payload/node-v22.22.3-linux-x64/bin" "$work/bin" @@ -42,7 +46,7 @@ set -euo pipefail output="" last="" joined=" $* " -for required in "--fail" "--location" "--proto =https" "--proto-redir =https" "--tlsv1.2" "--retry-all-errors" "--retry-connrefused" "--remove-on-error" "--max-filesize"; do +for required in "--fail" "--location" "--proto =https" "--proto-redir =https" "--tlsv1.2" "--retry 0" "--remove-on-error" "--max-filesize"; do [[ "$joined" == *" $required "* ]] || { echo "missing hardened curl argument: $required" >&2 exit 91 diff --git a/.github/actions/setup-moon/install-pinned-toolchain.sh b/.github/actions/setup-moon/install-pinned-toolchain.sh index 47b1bc883..9a8fd2a10 100755 --- a/.github/actions/setup-moon/install-pinned-toolchain.sh +++ b/.github/actions/setup-moon/install-pinned-toolchain.sh @@ -42,6 +42,8 @@ done # shellcheck source=tools/dev/curl-platform-flags.sh . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "Moon toolchain and plugins" 900 command -v bun >/dev/null 2>&1 || fail "Bun is required; run setup-node-bun first" @@ -257,8 +259,7 @@ curl_tls_flag="$(oliphaunt_curl_platform_tls_flag)" curl_common=( --fail --location --silent --show-error --proto '=https' --proto-redir '=https' --tlsv1.2 - --retry 6 --retry-all-errors --retry-connrefused --retry-max-time 300 - --connect-timeout 20 --max-time 300 --speed-limit 1024 --speed-time 30 + --connect-timeout 20 --speed-limit 1024 --speed-time 30 --remove-on-error ) if [ -n "$curl_tls_flag" ]; then @@ -287,7 +288,7 @@ download_verified() { args+=(--header "Authorization: Bearer $bearer") fi args+=("$url") - if "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then + if oliphaunt_acquisition_curl 300 7 2 "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then : else rc=$? @@ -317,7 +318,7 @@ registry_token() { --output "$response" "https://ghcr.io/token?scope=repository:$repository:pull" ) - if ! "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then + if ! oliphaunt_acquisition_curl 300 7 2 "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then rm -f "$response" fail "could not obtain a bounded read-only GHCR token for $repository" fi @@ -356,7 +357,7 @@ download_oci_manifest() { --output "$partial" "https://ghcr.io/v2/$repository/manifests/sha256:$digest" ) - if ! "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then + if ! oliphaunt_acquisition_curl 300 7 2 "${OLIPHAUNT_MOON_CURL:-curl}" "${args[@]}"; then rm -f "$partial" "$headers" fail "could not fetch pinned OCI manifest $repository@sha256:$digest" fi diff --git a/.github/actions/setup-moon/install-pinned-toolchain.test.sh b/.github/actions/setup-moon/install-pinned-toolchain.test.sh index eb1f3656b..f814b2b67 100755 --- a/.github/actions/setup-moon/install-pinned-toolchain.test.sh +++ b/.github/actions/setup-moon/install-pinned-toolchain.test.sh @@ -5,6 +5,10 @@ root="$(git rev-parse --show-toplevel)" installer="$root/.github/actions/setup-moon/install-pinned-toolchain.sh" extractor="$root/.github/actions/setup-moon/toolchain-archive.mts" tmp="$(mktemp -d)" +mkdir -p "$tmp/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$tmp/no-delay/sleep" +chmod +x "$tmp/no-delay/sleep" +export PATH="$tmp/no-delay:$PATH" trap 'rm -rf "$tmp"' EXIT fail() { @@ -27,6 +31,7 @@ mkdir -p \ "$fixture/content" \ "$fixture/blobs" cp "$root/tools/dev/curl-platform-flags.sh" "$fixture/tools/dev/curl-platform-flags.sh" +cp "$root/tools/dev/acquisition.sh" "$fixture/tools/dev/acquisition.sh" moon_version="9.8.7" proto_version="7.6.5" @@ -201,7 +206,7 @@ final="$(bash "$installer")" [ "$(find "$final/plugins" -mindepth 1 -maxdepth 1 | wc -l | tr -d '[:space:]')" = "4" ] || fail "wrong plugin count" [ "$(wc -l <"$FAKE_CURL_LOG" | tr -d '[:space:]')" = "13" ] || fail "unexpected first-install request count" while IFS= read -r call; do - for flag in --ssl-revoke-best-effort --tlsv1.2 --retry-all-errors --retry-connrefused --max-filesize --max-time --speed-limit; do + for flag in --ssl-revoke-best-effort --tlsv1.2 --retry --max-filesize --max-time --speed-limit; do [[ "$call" == *"$flag"* ]] || fail "curl request omitted $flag" done done <"$FAKE_CURL_LOG" diff --git a/.github/actions/setup-node-runtime/action.yml b/.github/actions/setup-node-runtime/action.yml index 52c36fc9d..004b1c9a5 100644 --- a/.github/actions/setup-node-runtime/action.yml +++ b/.github/actions/setup-node-runtime/action.yml @@ -14,7 +14,7 @@ runs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ runner.temp }}/oliphaunt-node-runtime - key: verified-node-runtime-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.github/actions/setup-node-runtime/action.yml', '.github/actions/setup-moon/install-pinned-node.sh', 'tools/dev/node-runtime.toml', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh') }} + key: verified-node-runtime-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.github/actions/setup-node-runtime/action.yml', '.github/actions/setup-moon/install-pinned-node.sh', 'tools/dev/node-runtime.toml', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} - name: Install verified Node.js runtime id: install @@ -41,4 +41,4 @@ runs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ runner.temp }}/oliphaunt-node-runtime - key: verified-node-runtime-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.github/actions/setup-node-runtime/action.yml', '.github/actions/setup-moon/install-pinned-node.sh', 'tools/dev/node-runtime.toml', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh') }} + key: verified-node-runtime-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.prototools', '.github/actions/setup-node-runtime/action.yml', '.github/actions/setup-moon/install-pinned-node.sh', 'tools/dev/node-runtime.toml', 'tools/dev/extract-pinned-binary.sh', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} diff --git a/.github/actions/setup-npm-publisher/action.yml b/.github/actions/setup-npm-publisher/action.yml index a2168f53b..015a7036c 100644 --- a/.github/actions/setup-npm-publisher/action.yml +++ b/.github/actions/setup-npm-publisher/action.yml @@ -25,7 +25,7 @@ runs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ runner.temp }}/oliphaunt-npm-publisher - key: verified-npm-publisher-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-npm-publisher/action.yml', '.github/actions/setup-npm-publisher/install.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/release/npm-publisher.toml', 'tools/packaging/portable-archive.mts', 'tools/dev/curl-platform-flags.sh') }} + key: verified-npm-publisher-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-npm-publisher/action.yml', '.github/actions/setup-npm-publisher/install.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/release/npm-publisher.toml', 'tools/packaging/portable-archive.mts', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} - name: Install exact npm publisher id: install @@ -98,4 +98,4 @@ runs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ runner.temp }}/oliphaunt-npm-publisher - key: verified-npm-publisher-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-npm-publisher/action.yml', '.github/actions/setup-npm-publisher/install.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/release/npm-publisher.toml', 'tools/packaging/portable-archive.mts', 'tools/dev/curl-platform-flags.sh') }} + key: verified-npm-publisher-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-npm-publisher/action.yml', '.github/actions/setup-npm-publisher/install.sh', '.github/actions/setup-moon/toolchain-archive.mts', 'tools/release/npm-publisher.toml', 'tools/packaging/portable-archive.mts', 'tools/dev/curl-platform-flags.sh', 'tools/dev/acquisition.sh') }} diff --git a/.github/actions/setup-npm-publisher/install.sh b/.github/actions/setup-npm-publisher/install.sh index 52432f18c..afefd51b3 100755 --- a/.github/actions/setup-npm-publisher/install.sh +++ b/.github/actions/setup-npm-publisher/install.sh @@ -29,6 +29,8 @@ for path in "$manifest" "$extractor" "$curl_platform_flags"; do done # shellcheck source=tools/dev/curl-platform-flags.sh . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "npm publisher bootstrap" 300 command -v node >/dev/null 2>&1 || fail "the verified Node.js runtime must be on PATH" @@ -196,12 +198,12 @@ if ! { [ -f "$archive" ] && [ ! -L "$archive" ] && partial="$(mktemp "$archive_root/.download.XXXXXX")" tls_flag="$(oliphaunt_curl_platform_tls_flag)" curl_args=(--fail --location --silent --show-error --proto '=https' --proto-redir '=https' - --tlsv1.2 --retry 5 --retry-all-errors --retry-connrefused --retry-delay 2 - --retry-max-time 300 --connect-timeout 20 --max-time 300 --speed-limit 1024 + --tlsv1.2 + --connect-timeout 20 --speed-limit 1024 --speed-time 30 --remove-on-error --max-filesize "$archive_bytes" --output "$partial") [ -z "$tls_flag" ] || curl_args+=("$tls_flag") curl_args+=("$url") - if ! "$curl_command" "${curl_args[@]}"; then + if ! oliphaunt_acquisition_curl 300 6 2 "$curl_command" "${curl_args[@]}"; then rm -f "$partial"; fail "could not download pinned npm publisher archive" fi [ "$(wc -c <"$partial" | tr -d '[:space:]')" = "$archive_bytes" ] && @@ -212,7 +214,6 @@ if ! { [ -f "$archive" ] && [ ! -L "$archive" ] && chmod 0444 "$partial" mv "$partial" "$archive" fi - stage="$(mktemp -d "$install_parent/.verified.stage.XXXXXX")" backup="" old_moved=0 @@ -229,7 +230,6 @@ trap cleanup EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM - extract_args=(extract --archive "$archive" --format "$format" --prefix "$prefix" --entry-count "$entry_count" --expected-bytes "$archive_bytes" --expanded-bytes "$expanded_bytes" --destination "$stage/npm" @@ -248,7 +248,6 @@ chmod 0444 "$stage/bin/npm.cmd" "$stage/bin/npx.cmd" printf '%s\n' "$receipt_text" >"$stage/receipt" chmod 0444 "$stage/receipt" cache_valid "$stage" || fail "staged npm publisher failed integrity or version validation" - if [ -e "$final" ] || [ -L "$final" ]; then backup="$(mktemp -d "$install_parent/.verified.backup.XXXXXX")"; rmdir "$backup" mv "$final" "$backup"; old_moved=1 diff --git a/.github/actions/setup-npm-publisher/install.test.sh b/.github/actions/setup-npm-publisher/install.test.sh index 4f2b07ca6..416cec097 100755 --- a/.github/actions/setup-npm-publisher/install.test.sh +++ b/.github/actions/setup-npm-publisher/install.test.sh @@ -5,6 +5,10 @@ root="$(git rev-parse --show-toplevel)" installer="$root/.github/actions/setup-npm-publisher/install.sh" extractor="$root/.github/actions/setup-moon/toolchain-archive.mts" work="$(mktemp -d)" +mkdir -p "$work/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$work/no-delay/sleep" +chmod +x "$work/no-delay/sleep" +export PATH="$work/no-delay:$PATH" trap 'rm -rf "$work"' EXIT mkdir -p "$work/payload/package/bin" "$work/bin" "$work/blockers" bash "$root/tools/dev/bun.sh" build "$root/.github/actions/setup-npm-publisher/testdata/package-manager-fixture.mts" \ @@ -62,7 +66,7 @@ cat >"$work/bin/curl" <<'EOF' #!/usr/bin/env bash set -euo pipefail joined=" $* " -for required in "--fail" "--location" "--proto =https" "--proto-redir =https" "--tlsv1.2" "--retry-all-errors" "--retry-connrefused" "--remove-on-error" "--max-filesize" "--ssl-revoke-best-effort"; do +for required in "--fail" "--location" "--proto =https" "--proto-redir =https" "--tlsv1.2" "--retry 0" "--remove-on-error" "--max-filesize" "--ssl-revoke-best-effort"; do [[ "$joined" == *" $required "* ]] || exit 91 done output="" diff --git a/.github/actions/setup-rust-tools/action.yml b/.github/actions/setup-rust-tools/action.yml index 3938ad3b2..321bb3cc5 100644 --- a/.github/actions/setup-rust-tools/action.yml +++ b/.github/actions/setup-rust-tools/action.yml @@ -5,7 +5,7 @@ inputs: toolchain: description: Rust toolchain version. required: false - default: "1.93.1" + default: "" components: description: Comma-separated Rust components. required: false @@ -30,10 +30,24 @@ inputs: runs: using: composite steps: + - name: Resolve Rust toolchain + id: toolchain + shell: bash + env: + TOOLCHAIN_INPUT: ${{ inputs.toolchain }} + run: | + set -euo pipefail + version="$TOOLCHAIN_INPUT" + if [[ -z "$version" ]]; then + version="$(sed -n 's/^channel = "\([^"]*\)"/\1/p' rust-toolchain.toml)" + fi + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo 'expected a pinned Rust version' >&2; exit 1; } + echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: - toolchain: ${{ inputs.toolchain }} + toolchain: ${{ steps.toolchain.outputs.version }} components: ${{ inputs.components }} - name: Cache Cargo output diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index e175487e5..2c1463777 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -18,6 +18,10 @@ inputs: description: Whether to enable the Cargo cache. required: false default: "true" + cache-workspaces: + description: Workspace and target-directory mappings for the Cargo cache. + required: false + default: ". -> target" runs: using: composite @@ -27,5 +31,6 @@ runs: with: components: ${{ inputs.components }} cache: ${{ inputs.cache }} + cache-workspaces: ${{ inputs.cache-workspaces }} cache-save-if: ${{ inputs.cache-save-if }} tools: ${{ inputs.tools }} diff --git a/.github/actions/setup-swift/action.yml b/.github/actions/setup-swift/action.yml index e608e31e9..8b13e6dbf 100644 --- a/.github/actions/setup-swift/action.yml +++ b/.github/actions/setup-swift/action.yml @@ -10,7 +10,13 @@ runs: - name: Import Swift signing keys shell: bash # Pin Swift's upstream key bundle instead of relying on the website endpoint. - run: curl --proto '=https' --proto-redir '=https' --tlsv1.2 --fail --silent --show-error --location --retry 3 --connect-timeout 15 --max-time 120 https://raw.githubusercontent.com/swiftlang/swift-org-website/894b4ef6cfd942971e433497feb0458af641ec1e/keys/all-keys.asc | gpg --batch --import - + run: | + . tools/dev/acquisition.sh + oliphaunt_acquisition_start 'Swift signing keys' 120 + keys=$(mktemp) + trap 'rm -f "$keys"' EXIT + oliphaunt_acquisition_curl 120 4 2 curl --proto '=https' --proto-redir '=https' --tlsv1.2 --fail --silent --show-error --location --connect-timeout 15 --output "$keys" https://raw.githubusercontent.com/swiftlang/swift-org-website/894b4ef6cfd942971e433497feb0458af641ec1e/keys/all-keys.asc + gpg --batch --import "$keys" - name: Install Swift uses: swift-actions/setup-swift@d8e84bc3a450686a95474d7d6fa4a3301498debc # v3, Swiftly 1.1.0 with: diff --git a/.github/actions/setup-wasix-builder/action.yml b/.github/actions/setup-wasix-builder/action.yml new file mode 100644 index 000000000..afd6678e1 --- /dev/null +++ b/.github/actions/setup-wasix-builder/action.yml @@ -0,0 +1,34 @@ +name: Set up WASIX builder +description: Load the pinned WASIX Docker builder using the shared build cache. + +inputs: + cache-save-if: + description: Whether this run may save the builder cache. + required: true + +runs: + using: composite + steps: + - name: Identify builder recipe + id: recipe + shell: bash + run: | + source src/wasix/postmaster/lib/common.sh + recipe_sha256="$(fresh_wasix_builder_recipe_sha256)" + printf 'sha256=%s\n' "$recipe_sha256" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd + + - name: Load WASIX builder + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f + with: + context: . + file: src/wasix/runtime/assets/build/docker/Dockerfile + tags: | + oliphaunt-wasix-wasix-build:ci + oliphaunt-wasix-wasix-build:local + labels: dev.oliphaunt.wasix-builder.recipe-sha256=${{ steps.recipe.outputs.sha256 }} + load: true + cache-from: type=gha,scope=wasix-builder + cache-to: ${{ inputs.cache-save-if == 'true' && 'type=gha,mode=max,scope=wasix-builder,ignore-error=true' || '' }} diff --git a/.github/actions/setup-wasmer-llvm/install.sh b/.github/actions/setup-wasmer-llvm/install.sh index 86574ff93..a7656f5a8 100755 --- a/.github/actions/setup-wasmer-llvm/install.sh +++ b/.github/actions/setup-wasmer-llvm/install.sh @@ -19,6 +19,8 @@ if [ ! -f "$curl_platform_flags" ] || [ -L "$curl_platform_flags" ]; then fi # shellcheck source=tools/dev/curl-platform-flags.sh . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "Wasmer LLVM" 1800 if [[ ! "$LLVM_URL" =~ ^https://[^[:space:]]+$ ]]; then echo "Wasmer LLVM URL must be a single HTTPS URL" >&2 @@ -201,12 +203,7 @@ curl_args=( --location --fail --show-error - --retry 4 - --retry-all-errors - --retry-delay 10 - --retry-max-time 3600 --connect-timeout 30 - --max-time 1800 --max-filesize "$LLVM_BYTES" --proto '=https' --proto-redir '=https' @@ -217,7 +214,7 @@ if [ -n "$curl_platform_tls_flag" ]; then curl_args+=("$curl_platform_tls_flag") fi curl_args+=(--output "$archive" "$LLVM_URL") -curl "${curl_args[@]}" +oliphaunt_acquisition_curl 1800 5 10 curl "${curl_args[@]}" actual_bytes="$(wc -c < "$archive" | tr -d '[:space:]')" if [ "$actual_bytes" != "$LLVM_BYTES" ]; then diff --git a/.github/actions/setup-wasmer-llvm/install.test.sh b/.github/actions/setup-wasmer-llvm/install.test.sh index cab124eed..22b303065 100755 --- a/.github/actions/setup-wasmer-llvm/install.test.sh +++ b/.github/actions/setup-wasmer-llvm/install.test.sh @@ -20,6 +20,10 @@ sha256_file() { } work_root="$(mktemp -d)" +mkdir -p "$work_root/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$work_root/no-delay/sleep" +chmod +x "$work_root/no-delay/sleep" +export PATH="$work_root/no-delay:$PATH" cleanup() { rm -rf "$work_root" } @@ -223,12 +227,9 @@ grep -F "$success_final/bin" "$success_runner/github-path" >/dev/null || fail "G for flag in \ '--location' \ '--fail' \ - '--retry 4' \ - '--retry-all-errors' \ - '--retry-delay 10' \ - '--retry-max-time 3600' \ + '--retry 0' \ '--connect-timeout 30' \ - '--max-time 1800' \ + '--max-time' \ "--max-filesize $valid_bytes" \ '--proto =https' \ '--proto-redir =https' \ diff --git a/.github/moon.yml b/.github/moon.yml index 4208183b8..932c55d33 100644 --- a/.github/moon.yml +++ b/.github/moon.yml @@ -29,6 +29,7 @@ tasks: - "actions/setup-wasmer-llvm/**/*" - "/tools/packaging/portable-archive.mts" - "/tools/dev/curl-platform-flags.sh" + - "/tools/dev/acquisition.sh" - "/tools/packaging/testdata/tar-fixture.mts" options: cache: true @@ -47,12 +48,21 @@ tasks: - "/tools/ci/with-projects.sh" - "/tools/dev/bun.sh" - "/tools/dev/curl-platform-flags.sh" + - "/tools/dev/acquisition.sh" - "/tools/ci/workflow-security.mts" - "/tools/ci/workflow-security.test.mts" + - "/tools/ci/workflow-caches.test.mts" + - "/src/wasix/postmaster/lib/common.sh" + - "/src/wasix/postmaster/executor/Cargo.toml" + - "/src/wasix/postmaster/executor/Cargo.lock" + - "/src/wasix/postmaster/executor/src/**" + - "/src/wasix/postmaster/sources/*.toml" + - "/src/third-party/postgres/source.toml" + - "/src/wasix/runtime/assets/build/docker/**" - "/tools/ci/ci-plan-node-products.test.mts" - "/tools/ci/ci-plan-wasix-postmaster-release.test.mts" - "/tools/ci/ci-plan-test-*.mts" - - "/tools/ci/capture-ci-test-observations.sh" + - "/tools/ci/capture-ci-test-observations*.sh" - "/tools/ci/ci-release-scope.test.*" - "/tools/ci/workflow-moon-transfers.test.mts" - "/tools/ci/check-workflows.sh" @@ -61,6 +71,8 @@ tasks: - "/tools/ci/ci-plan.sh" - "/src/native/sdks/ts/tools/published-consumer.mts" - "/src/native/sdks/ts/package.json" + - "/src/native/sdks/swift/tools/ios-carrier-manifest.mts" + - "/src/native/runtime/VERSION" - "/tools/release/check_registry_publication.mts" - "/tools/release/public-consumer-smoke.mts" - "/tools/release/release-graph.mts" diff --git a/.github/scripts/moon-task-capabilities.mts b/.github/scripts/moon-task-capabilities.mts index ca69f49c5..e488bda18 100644 --- a/.github/scripts/moon-task-capabilities.mts +++ b/.github/scripts/moon-task-capabilities.mts @@ -28,6 +28,14 @@ const DISPLAY_WORDS = Object.freeze({ }); const DISPLAY_PARTS = Object.freeze({ 'extension-artifacts-native': 'Native Extension Artifacts', + 'liboliphaunt-native': 'Native Runtime', + 'liboliphaunt-wasix': 'WASIX Runtime', + 'oliphaunt-rust': 'Rust SDK', + 'oliphaunt-kotlin': 'Kotlin SDK', + 'oliphaunt-swift': 'Swift SDK', + 'oliphaunt-react-native': 'React Native SDK', + 'oliphaunt-wasix-rust': 'WASIX Rust SDK', + 'oliphaunt-wasix-ts': 'WASIX TypeScript SDK', }); export const MAX_TARGETS_PER_JOB = 4; @@ -135,8 +143,11 @@ function compareTargets(left, right) { function groupRow(targets) { const first = targets[0]; + const projects = [...new Set(targets.map(({ target }) => target.split(':')[0]))]; return { - label: targets.map(({ label }) => label).join(' + '), + label: projects + .map((project) => taskLabel(project).replace(/^(?:Oliphaunt|liboliphaunt) /u, '')) + .join(' + '), target_count: targets.length, requires_rust: first.requires_rust, requires_maintainer_tools: first.requires_maintainer_tools, @@ -183,5 +194,12 @@ export function groupTargets(targets, { maxTargets = MAX_TARGETS_PER_JOB } = {}) groups.push(targetsWithSameSetup.slice(index, index + maxTargets)); } } - return groups.map(groupRow); + const rows = groups.map(groupRow); + return rows.map((row, index) => { + if (rows.filter(({ label }) => label === row.label).length === 1) return row; + // A component may have separate setup profiles or span multiple batches. + // Name the distinguishing tasks instead of assigning an opaque shard number. + const tasks = [...new Set(groups[index].map(({ target }) => taskLabel(target.split(':')[1])))]; + return { ...row, label: `${row.label} (${tasks.join(' + ')})` }; + }); } diff --git a/.github/scripts/moon-task-capabilities.test.mts b/.github/scripts/moon-task-capabilities.test.mts index 44e5430ab..f7f4f6778 100644 --- a/.github/scripts/moon-task-capabilities.test.mts +++ b/.github/scripts/moon-task-capabilities.test.mts @@ -1,5 +1,5 @@ -import { strict as assert } from 'node:assert'; import { describe, test } from 'bun:test'; +import { strict as assert } from 'node:assert'; import { groupTargets, @@ -71,7 +71,8 @@ describe('Moon task capabilities', () => { groups.map(({ target_count }) => target_count), [1, 1, 1, 4, 4, 1, 2, 1], ); - assert.equal(groups[3].label, 'Plain / 0 + Plain / 1 + Plain / 2 + Plain / 3'); + assert.equal(groups[3].label, 'Plain (0 + 1 + 2 + 3)'); + assert.equal(groups[6].label, 'Rust'); assert.equal(groups.filter(({ requires_rust }) => requires_rust).length, 1); assert.equal(groups.filter(({ requires_android_sdk }) => requires_android_sdk).length, 1); assert.equal(groups.filter(({ requires_apple }) => requires_apple).length, 1); @@ -84,6 +85,24 @@ describe('Moon task capabilities', () => { assert.deepEqual(selected.sort(), [...taskMap.keys()].sort()); }); + test('names grouped components and distinguishes their task batches without setup labels', () => { + const taskMap = tasks( + { target: 'liboliphaunt-native:test' }, + { target: 'liboliphaunt-wasix:test' }, + { target: 'oliphaunt-kotlin:format-check', tags: ['requires-android-sdk'] }, + { target: 'oliphaunt-kotlin:lint', tags: ['requires-android-sdk', 'requires-rust'] }, + ); + const targets = [...taskMap.values()].map((task) => matrixTarget(task, 'deep', taskMap)); + const labels = (rows) => groupTargets(rows).map(({ label }) => label); + assert.deepEqual(labels(targets), [ + 'Native Runtime + WASIX Runtime', + 'Kotlin SDK (Format Check)', + 'Kotlin SDK (Lint)', + ]); + assert.deepEqual(labels([...targets].reverse()), labels(targets)); + assert.equal(new Set(labels(targets)).size, labels(targets).length); + }); + test('rejects duplicate targets and invalid shard limits', () => { const row = { target: 'repo:check', diff --git a/.github/scripts/release-candidate-lib.mts b/.github/scripts/release-candidate-lib.mts index d256992a0..172fb54c6 100644 --- a/.github/scripts/release-candidate-lib.mts +++ b/.github/scripts/release-candidate-lib.mts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; -import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; import path from 'node:path'; +import { verifyReceipts } from '../../src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts'; function compareText(left, right) { return left < right ? -1 : left > right ? 1 : 0; @@ -150,6 +151,17 @@ export function affectedPlanBinding(planPath, wasixReleaseRegressionRequired) { projects, extensionPackageProducts, wasixReleaseRegressionRequired, + ...(jobs.includes('native-extension-lifecycle') + ? { + nativeExtensionLifecycle: { + extensions: sortedUniqueStrings( + plan.native_extension_lifecycle_sql_names, + 'native lifecycle extensions', + ), + shardCount: plan.native_extension_lifecycle_shard_count, + }, + } + : {}), ...(qualification === undefined ? {} : { qualification }), }; } @@ -300,6 +312,41 @@ export function wasixEvidenceBinding( }; } +export function nativeEvidenceBinding( + evidenceRoot, + { repository, runId, runAttempt, sha, tree, selection }, +) { + const root = path.resolve(evidenceRoot); + const file = 'output/aggregate-receipt.json'; + const { bytes, value: evidence } = strictJson(path.join(root, file), 'native lifecycle evidence'); + same(evidence.github?.repository, repository, 'native evidence GitHub repository'); + same(evidence.github?.workflow, 'CI', 'native evidence GitHub workflow'); + same(evidence.github?.job, 'native-extension-lifecycle-aggregate', 'native evidence GitHub job'); + same(evidence.github?.runId, Number(runId), 'native evidence GitHub runId'); + positiveInteger(evidence.github?.runAttempt, 'native evidence GitHub runAttempt'); + assert( + evidence.github.runAttempt <= runAttempt, + 'native evidence attempt is newer than candidate', + ); + const verified = verifyReceipts({ + receipts: path.join(root, 'input'), + 'candidate-sha': sha, + 'candidate-tree': tree, + 'expected-extensions-csv': selection.extensions.join(','), + 'expected-shard-count': String(selection.shardCount), + repository, + 'run-id': String(runId), + 'run-attempt': String(evidence.github.runAttempt), + }); + assertBindingMatches(evidence, verified, 'native aggregate receipt'); + return { + artifact: 'native-extension-lifecycle-evidence', + file, + digest: sha256(bytes), + github: evidence.github, + }; +} + export function assertCandidateBindingShape(candidate) { assert( candidate?.schemaVersion === 2, @@ -437,9 +484,47 @@ export function assertCandidateBindingShape(candidate) { requirements.wasixReleaseRegression === candidate.affectedPlan.wasixReleaseRegressionRequired, 'release candidate WASIX evidence requirement is inconsistent with affected plan', ); - const expectedArtifacts = requirements.wasixReleaseRegression - ? ['wasix-release-regression-evidence'] - : []; + const nativeRequired = jobs.includes('native-extension-lifecycle'); + assert( + (requirements.nativeExtensionLifecycle ?? false) === nativeRequired, + 'release candidate native evidence requirement is inconsistent with selected jobs', + ); + if (nativeRequired) { + const selection = candidate.affectedPlan.nativeExtensionLifecycle; + assert( + selection && + sortedUniqueStrings(selection.extensions, 'native lifecycle extensions').length > 0, + 'release candidate native lifecycle selection is missing', + ); + assert( + Number.isSafeInteger(selection.shardCount) && + selection.shardCount > 0 && + selection.shardCount <= selection.extensions.length, + 'release candidate native lifecycle shard count is invalid', + ); + const evidence = candidate.evidence?.nativeExtensionLifecycle; + assert( + /^sha256:[0-9a-f]{64}$/.test(evidence?.digest), + 'release candidate native evidence digest is missing or invalid', + ); + same(evidence.github?.repository, candidate.repository, 'native evidence repository'); + same(evidence.github?.runId, Number(candidate.runId), 'native evidence runId'); + positiveInteger(evidence.github?.runAttempt, 'native evidence runAttempt'); + assert( + evidence.github.runAttempt <= candidate.runAttempt, + 'native evidence attempt is newer than candidate', + ); + } else { + assert( + candidate.evidence?.nativeExtensionLifecycle == null && + candidate.affectedPlan.nativeExtensionLifecycle == null, + 'release candidate carries native evidence that its plan did not require', + ); + } + const expectedArtifacts = [ + ...(nativeRequired ? ['native-extension-lifecycle-evidence'] : []), + ...(requirements.wasixReleaseRegression ? ['wasix-release-regression-evidence'] : []), + ]; assert( JSON.stringify(requirements.artifacts) === JSON.stringify(expectedArtifacts), 'release candidate evidence artifact requirements are inconsistent', diff --git a/.github/scripts/resolve-planned-moon-execution.mts b/.github/scripts/resolve-planned-moon-execution.mts index 263770171..27ef381b2 100644 --- a/.github/scripts/resolve-planned-moon-execution.mts +++ b/.github/scripts/resolve-planned-moon-execution.mts @@ -91,6 +91,23 @@ function parseTransferred() { return value; } +export function executionBatches(targets, tasks) { + const levels = new Map(); + const batches = []; + // resolveExecution already orders prerequisites before consumers. Only batch + // independent tasks; --upstream none does not enforce their dependency order. + for (const target of targets) { + const level = Math.max( + 0, + ...dependencyTargets(tasks.get(target), tasks).map((dep) => (levels.get(dep) ?? -1) + 1), + ); + levels.set(target, level); + batches[level] ??= []; + batches[level].push(target); + } + return batches; +} + function taskMap() { const graph = JSON.parse(readFileSync(process.env.OLIPHAUNT_MOON_TASK_GRAPH_FILE, 'utf8')); return new Map(Object.values(graph.data ?? {}).map((task) => [task.target, task])); @@ -126,7 +143,10 @@ if (import.meta.main) { for (const target of targets) collect(target); const execution = resolveExecution(targets, transferred, tasks); for (const target of execution.localDependencies) console.log(`local\t${target}`); - for (const target of execution.targets) console.log(`target\t${target}`); + const batches = execution.transferred.length + ? executionBatches(execution.targets, tasks) + : [execution.targets]; + for (const batch of batches) console.log(`target\t${batch.join(' ')}`); for (const target of execution.transferred) console.log(`transferred\t${target}`); } catch (error) { fail(error instanceof Error ? error.message : String(error)); diff --git a/.github/scripts/resolve-planned-moon-execution.test.mts b/.github/scripts/resolve-planned-moon-execution.test.mts index 8c1e5bee8..67c000619 100644 --- a/.github/scripts/resolve-planned-moon-execution.test.mts +++ b/.github/scripts/resolve-planned-moon-execution.test.mts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import { resolveExecution } from './resolve-planned-moon-execution.mts'; +import { executionBatches, resolveExecution } from './resolve-planned-moon-execution.mts'; const tasks = new Map([ [ @@ -26,6 +26,21 @@ const tasks = new Map([ ], ]); +test('batches independent artifact consumers without starting their dependents early', () => { + const graph = new Map([ + ['import:bytes', { deps: [] }], + ['a:build', { deps: ['import:bytes'] }], + ['b:build', { deps: ['import:bytes'] }], + ['c:test', { deps: ['a:build', 'b:build'] }], + ['d:test', { deps: ['a:build'] }], + ]); + const execution = resolveExecution(['c:test', 'd:test'], ['import:bytes'], graph); + assert.deepEqual(executionBatches(execution.targets, graph), [ + ['a:build', 'b:build'], + ['c:test', 'd:test'], + ]); +}); + test('leaves ordinary Moon execution intact when no dependency was transferred', () => { assert.deepEqual(resolveExecution(['release:package'], [], tasks), { localDependencies: [], diff --git a/.github/scripts/resolve-planned-moon-execution.test.sh b/.github/scripts/resolve-planned-moon-execution.test.sh index 806b3b28c..be6ed1f88 100644 --- a/.github/scripts/resolve-planned-moon-execution.test.sh +++ b/.github/scripts/resolve-planned-moon-execution.test.sh @@ -8,7 +8,7 @@ bash tools/dev/bun.sh test ./.github/scripts/resolve-planned-moon-execution.test resolver=.github/scripts/resolve-planned-moon-execution.mts export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"wasix-ts-sdk-package":["oliphaunt-wasix-ts:package","oliphaunt-wasix-ts:test-consumer","oliphaunt-wasix-ts:test-browser"]}' export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["liboliphaunt-wasix:runtime-portable","oliphaunt-wasix-napi:build-release-assets","extension-artifacts-wasix:compiler-output","database-resources:build-wasix-standard","database-resources:build-wasix-icu","database-resources:package-icu"]' -bash tools/dev/bun.sh "$resolver" wasix-ts-sdk-package >"$scratch/output" +bash tools/dev/bun.sh "$resolver" wasix-ts-sdk-package | awk -F '\t' '{n=split($2, targets, " "); for(i=1;i<=n;i++) print $1 "\t" targets[i]}' >"$scratch/output" for task in package test-consumer test-browser; do grep -Fx "$(printf 'target\toliphaunt-wasix-ts:%s' "$task")" "$scratch/output"; done grep -Fx $'target\tdatabase-resources:package-wasix' "$scratch/output" for variant in standard icu; do @@ -29,22 +29,79 @@ for platform in android ios; do printf 'transferred\tliboliphaunt-native:build-runtime-%s\n' "$target" >>"$scratch/expected" done export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON="${transfers%,}]" - bash tools/dev/bun.sh "$resolver" "$job" >"$scratch/output" + bash tools/dev/bun.sh "$resolver" "$job" | awk -F '\t' '{n=split($2, targets, " "); for(i=1;i<=n;i++) print $1 "\t" targets[i]}' >"$scratch/output" cmp "$scratch/expected" "$scratch/output" done export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"react-native-sdk-package":["oliphaunt-react-native:test-consumer","oliphaunt-react-native:package"]}' export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["liboliphaunt-native:finalize-runtime-ios-abi"]' -bash tools/dev/bun.sh "$resolver" react-native-sdk-package >"$scratch/output" +bash tools/dev/bun.sh "$resolver" react-native-sdk-package | awk -F '\t' '{n=split($2, targets, " "); for(i=1;i<=n;i++) print $1 "\t" targets[i]}' >"$scratch/output" grep $'^target\t' "$scratch/output" >"$scratch/targets" printf 'target\toliphaunt-react-native:package\ntarget\toliphaunt-react-native:test-consumer\n' >"$scratch/expected" cmp "$scratch/expected" "$scratch/targets" if grep -E $'^local\t(oliphaunt-react-native:package|liboliphaunt-native:finalize-runtime-ios-abi)$' "$scratch/output"; then exit 1; fi unset OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"liboliphaunt-native-android":["liboliphaunt-native:package-runtime-android-arm64-v8a","liboliphaunt-native:package-runtime-android-x86_64"]}' -bash tools/dev/bun.sh "$resolver" liboliphaunt-native-android liboliphaunt-native:package-runtime-android-x86_64 >"$scratch/output" +bash tools/dev/bun.sh "$resolver" liboliphaunt-native-android liboliphaunt-native:package-runtime-android-x86_64 | awk -F '\t' '{n=split($2, targets, " "); for(i=1;i<=n;i++) print $1 "\t" targets[i]}' >"$scratch/output" printf 'target\tliboliphaunt-native:package-runtime-android-x86_64\n' >"$scratch/expected" cmp "$scratch/expected" "$scratch/output" if bash tools/dev/bun.sh "$resolver" liboliphaunt-native-android liboliphaunt-native:package-runtime-ios-xcframework >"$scratch/output" 2>"$scratch/error"; then echo 'accepted a target outside the job plan' >&2; exit 1 fi grep -q 'is not planned' "$scratch/error" + + +# Prove the scheduling boundary with the pinned Moon binary: siblings overlap, +# their consumer waits for both, and a transferred producer must never execute. +mkdir -p "$scratch/parallel/.moon" "$scratch/parallel/.github/scripts" +cp .github/scripts/{run-planned-moon-job.sh,run-moon-targets.sh,resolve-planned-moon-execution.mts,select-planned-moon-targets.mts} "$scratch/parallel/.github/scripts/" +cat > "$scratch/parallel/.moon/workspace.yml" <<'YAML' +projects: + fixture: . +vcs: + defaultBranch: main +YAML +cat > "$scratch/parallel/moon.yml" <<'YAML' +id: fixture +language: unknown +tasks: + downloaded: + script: exit 99 + a: + command: bash sibling.sh a b + deps: [downloaded] + b: + command: bash sibling.sh b a + deps: [downloaded] + joined: + script: test -f a.done && test -f b.done && touch joined.done + deps: [a, b] +YAML +cat > "$scratch/parallel/sibling.sh" <<'SH' +#!/usr/bin/env bash +set -eu +touch "$1.started" +for attempt in {1..100}; do + [ ! -e "$2.started" ] || break + sleep 0.1 +done +test -f "$2.started" +touch "$1.done" +SH +( + cd "$scratch/parallel" + git init -q --initial-branch=fixture + git add . + git -c user.name=fixture -c user.email=fixture@example.invalid commit -q -m fixture + git clone -q --depth 1 "file://$PWD" "$scratch/shallow" + cd "$scratch/shallow" + # Planned execution needs only the candidate, with no main ref or history. + # Moon tasks inherit their parent's root; this is a separate fixture workspace. + export MOON_WORKSPACE_ROOT="$PWD" + test "$(git rev-parse --is-shallow-repository)" = true + export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"parallel":["fixture:joined"]}' + export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["fixture:downloaded"]' + # Exercise CI's base-ref lookup locally as well as on GitHub. + CI=true GITHUB_ACTIONS=true GITHUB_REF=refs/pull/1/merge GITHUB_BASE_REF=main \ + MOON_CONCURRENCY=2 bash .github/scripts/run-planned-moon-job.sh parallel + test -f joined.done +) diff --git a/.github/scripts/run-moon-targets.sh b/.github/scripts/run-moon-targets.sh index f3501fa00..ee04ad4c2 100755 --- a/.github/scripts/run-moon-targets.sh +++ b/.github/scripts/run-moon-targets.sh @@ -5,13 +5,19 @@ unset MOON_BASE unset MOON_HEAD moon_bin="${MOON_BIN:-moon}" +# Planning already selected these targets. Execute the candidate without +# requiring a second changed-file comparison or a fetched base branch. if [ "${1:-}" = --matrix ]; then groups="$(bun .github/scripts/select-moon-target-groups.mts)" while IFS=$'\t' read -r upstream targets; do read -r -a target_args <<<"$targets" - "$moon_bin" run --upstream "$upstream" "${target_args[@]}" + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + printf '\nSelected Moon tasks:\n\n' >> "$GITHUB_STEP_SUMMARY" + printf -- '- `%s`\n' "${target_args[@]}" >> "$GITHUB_STEP_SUMMARY" + fi + "$moon_bin" run --base HEAD --head HEAD --upstream "$upstream" "${target_args[@]}" done <<<"$groups" else - exec "$moon_bin" run "$@" + exec "$moon_bin" run --base HEAD --head HEAD "$@" fi diff --git a/.github/scripts/run-moon-targets.test.sh b/.github/scripts/run-moon-targets.test.sh index 179ede0b1..adc656c3a 100644 --- a/.github/scripts/run-moon-targets.test.sh +++ b/.github/scripts/run-moon-targets.test.sh @@ -5,7 +5,7 @@ trap 'rm -rf "$fixture"' EXIT export MOON_BIN="$fixture/moon" MOON_CALLS="$fixture/calls" cat >"$MOON_BIN" <<'MOON' #!/usr/bin/env bash -if [[ -v MOON_BASE || -v MOON_HEAD ]]; then +if [[ -n "${MOON_BASE+x}${MOON_HEAD+x}" ]]; then echo 'explicit task execution inherited affectedness revisions' >&2 exit 8 fi @@ -15,19 +15,19 @@ MOON chmod +x "$MOON_BIN" MOON_BASE=missing-base MOON_HEAD=missing-head \ bash .github/scripts/run-moon-targets.sh ci-workflows:verify-bash -printf 'run ci-workflows:verify-bash\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD ci-workflows:verify-bash\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" export MOON_TARGET_MATRIX_JSON='{"include":[{"target":"sdk:b"},{"target":"sdk:a"},{"target":"sdk:a"},{"target":"native:c","upstream":"none"}]}' bash .github/scripts/run-moon-targets.sh --matrix -printf 'run --upstream deep sdk:a sdk:b\nrun --upstream none native:c\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD --upstream deep sdk:a sdk:b\nrun --base HEAD --head HEAD --upstream none native:c\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" if MOON_TEST_EXIT=7 bash .github/scripts/run-moon-targets.sh --matrix; then echo 'Moon failure was ignored' >&2 exit 1 fi -printf 'run --upstream deep sdk:a sdk:b\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD --upstream deep sdk:a sdk:b\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" if MOON_TARGET_MATRIX_JSON='{"include":[{"target":"sdk:a"},{"target":"-bad target"}]}' \ @@ -45,7 +45,7 @@ export MOON_TEST_GRAPH="$fixture/graph.json" cat >"$MOON_BIN" <<'MOON' #!/usr/bin/env bash if [ "$1" = task-graph ]; then cat "$MOON_TEST_GRAPH"; exit; fi -if [[ "$*" == 'run --upstream none '* && "${MOON_CACHE:-}" != off ]]; then +if [[ "$*" == 'run --base HEAD --head HEAD --upstream none '* && "${MOON_CACHE:-}" != off ]]; then echo 'transferred consumers must execute without incomplete dependency cache keys' >&2 exit 9 fi @@ -55,26 +55,26 @@ MOON export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"fixture":["sdk:a-consumer","sdk:z-package"]}' export OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON='["native:ios"]' bash .github/scripts/run-planned-moon-job.sh fixture -printf 'run sdk:build\nrun --upstream none sdk:z-package\nrun --upstream none sdk:a-consumer\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD sdk:build\nrun --base HEAD --head HEAD --upstream none sdk:z-package\nrun --base HEAD --head HEAD --upstream none sdk:a-consumer\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" if MOON_FAIL_TARGET=sdk:z-package bash .github/scripts/run-planned-moon-job.sh fixture; then echo 'failed package reached its consumer' >&2 exit 1 fi -printf 'run sdk:build\nrun --upstream none sdk:z-package\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD sdk:build\nrun --base HEAD --head HEAD --upstream none sdk:z-package\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" # A narrowed consumer still needs its intermediate package, but never its transferred producer. : >"$MOON_CALLS" export OLIPHAUNT_CI_JOB_TARGETS_JSON='{"fixture":["sdk:a-consumer"]}' bash .github/scripts/run-planned-moon-job.sh fixture -printf 'run sdk:build\nrun --upstream none sdk:z-package\nrun --upstream none sdk:a-consumer\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD sdk:build\nrun --base HEAD --head HEAD --upstream none sdk:z-package\nrun --base HEAD --head HEAD --upstream none sdk:a-consumer\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" : >"$MOON_CALLS" if MOON_FAIL_TARGET=sdk:z-package bash .github/scripts/run-planned-moon-job.sh fixture; then echo 'failed intermediate package reached its consumer' >&2 exit 1 fi -printf 'run sdk:build\nrun --upstream none sdk:z-package\n' >"$fixture/expected" +printf 'run --base HEAD --head HEAD sdk:build\nrun --base HEAD --head HEAD --upstream none sdk:z-package\n' >"$fixture/expected" cmp "$MOON_CALLS" "$fixture/expected" diff --git a/.github/scripts/run-planned-moon-job.sh b/.github/scripts/run-planned-moon-job.sh index efd05cbc8..e55343298 100755 --- a/.github/scripts/run-planned-moon-job.sh +++ b/.github/scripts/run-planned-moon-job.sh @@ -52,14 +52,18 @@ if [[ "${#transferred_dependencies[@]}" -gt 0 ]]; then if [[ "${#local_dependencies[@]}" -gt 0 ]]; then .github/scripts/run-moon-targets.sh "${local_dependencies[@]}" fi - for target in "${targets[@]}"; do + for batch in "${targets[@]}"; do # Omitting transferred producers also omits their source hashes in Moon. # Execute intermediate prerequisites and roots against the downloaded bytes. - MOON_CACHE=off .github/scripts/run-moon-targets.sh --upstream none "$target" + read -r -a batch_targets <<<"$batch" + MOON_CACHE=off .github/scripts/run-moon-targets.sh --upstream none "${batch_targets[@]}" done exit 0 fi +# Without transfers the resolver emits one batch and Moon owns the whole DAG. +read -r -a targets <<<"${targets[0]}" + if [[ "${#moon_args[@]}" -gt 0 ]]; then exec .github/scripts/run-moon-targets.sh "${moon_args[@]}" "${targets[@]}" fi diff --git a/.github/scripts/verify-release-candidate.mts b/.github/scripts/verify-release-candidate.mts index 9a9ba41fd..dbf9e7b93 100644 --- a/.github/scripts/verify-release-candidate.mts +++ b/.github/scripts/verify-release-candidate.mts @@ -1,15 +1,20 @@ #!/usr/bin/env bun import { readFileSync } from 'node:fs'; import process from 'node:process'; +import { + extensionArtifactProductsForReleaseProducts, + extensionSqlNamesForProducts, +} from '../../tools/release/release-artifact-targets.mts'; import { affectedPlanBinding, assertBindingMatches, assertCandidateBindingShape, + assertQualificationProductCoverage, candidateQualificationMode, FULL_PAYLOAD_QUALIFICATION_MODE, + nativeEvidenceBinding, PRODUCT_QUALIFICATION_MODE, - assertQualificationProductCoverage, wasixEvidenceBinding, } from './release-candidate-lib.mts'; @@ -36,6 +41,8 @@ function parseArgs(argv) { '--plan', '--qualification-mode', '--products-json', + '--native-evidence-required', + '--native-evidence-root', '--wasix-evidence-required', '--wasix-evidence-root', ].includes(name) @@ -52,7 +59,8 @@ function parseArgs(argv) { fail( 'usage: verify-release-candidate.mts --plan ' + '--wasix-evidence-required true|false [--qualification-mode full-payload] ' + - '[--wasix-evidence-root ]', + '[--wasix-evidence-root ] [--native-evidence-required true|false] ' + + '[--native-evidence-root ]', ); } const required = values.get('wasix-evidence-required'); @@ -62,6 +70,11 @@ function parseArgs(argv) { if (required === 'true' && !values.has('wasix-evidence-root')) { fail('--wasix-evidence-root is required when WASIX evidence is required'); } + const nativeRequired = values.get('native-evidence-required') ?? 'false'; + if (!['true', 'false'].includes(nativeRequired)) + fail('--native-evidence-required must be true or false'); + if (nativeRequired === 'true' && !values.has('native-evidence-root')) + fail('--native-evidence-root is required when native evidence is required'); const qualificationMode = values.get('qualification-mode') ?? FULL_PAYLOAD_QUALIFICATION_MODE; if ( ![FULL_PAYLOAD_QUALIFICATION_MODE, PRODUCT_QUALIFICATION_MODE, 'release'].includes( @@ -79,6 +92,8 @@ function parseArgs(argv) { candidatePath, planPath: values.get('plan'), wasixEvidenceRequired: required === 'true', + nativeEvidenceRequired: nativeRequired === 'true', + nativeEvidenceRoot: values.get('native-evidence-root'), wasixEvidenceRoot: values.get('wasix-evidence-root'), qualificationMode, products, @@ -195,4 +210,36 @@ if (args.wasixEvidenceRequired) { } } +if (args.nativeEvidenceRequired && !candidate.evidenceRequirements.nativeExtensionLifecycle) + fail( + 'selected release products require native evidence, but the qualified CI plan did not require it', + ); +if (args.nativeEvidenceRequired) { + try { + if (args.products) { + const requiredExtensions = extensionSqlNamesForProducts( + extensionArtifactProductsForReleaseProducts(args.products, { family: 'native' }), + ); + for (const extension of requiredExtensions) { + if (!expectedPlan.nativeExtensionLifecycle.extensions.includes(extension)) + fail(`native evidence is missing published extension ${extension}`); + } + } + assertBindingMatches( + candidate.evidence.nativeExtensionLifecycle, + nativeEvidenceBinding(args.nativeEvidenceRoot, { + repository: expected.repository, + runId: expected.runId, + runAttempt: candidate.runAttempt, + sha: expected.sha, + tree: expectedTree, + selection: expectedPlan.nativeExtensionLifecycle, + }), + 'release candidate native evidence', + ); + } catch (error) { + fail(error.message); + } +} + console.log(`verified qualified CI run ${candidate.runId} for ${candidate.sha}`); diff --git a/.github/scripts/write-release-candidate.mts b/.github/scripts/write-release-candidate.mts index 1db3643e5..0a606e618 100644 --- a/.github/scripts/write-release-candidate.mts +++ b/.github/scripts/write-release-candidate.mts @@ -8,6 +8,7 @@ import { assertCandidateBindingShape, candidateQualificationMode, FULL_PAYLOAD_QUALIFICATION_MODE, + nativeEvidenceBinding, PRODUCT_QUALIFICATION_MODE, wasixEvidenceBinding, } from './release-candidate-lib.mts'; @@ -81,6 +82,23 @@ if (wasixRequired) { } } +const nativeRequired = affectedPlan.nativeExtensionLifecycle !== undefined; +let nativeEvidence = null; +if (nativeRequired) { + try { + nativeEvidence = nativeEvidenceBinding(requiredEnv('NATIVE_EVIDENCE_ROOT'), { + repository: requiredEnv('GITHUB_REPOSITORY'), + runId: requiredEnv('GITHUB_RUN_ID'), + runAttempt, + sha: checkedOutSha, + tree, + selection: affectedPlan.nativeExtensionLifecycle, + }); + } catch (error) { + fail(error.message); + } +} + const candidate = { schemaVersion: 2, repository: requiredEnv('GITHUB_REPOSITORY'), @@ -96,10 +114,15 @@ const candidate = { affectedPlan, evidenceRequirements: { wasixReleaseRegression: wasixRequired, - artifacts: wasixRequired ? ['wasix-release-regression-evidence'] : [], + nativeExtensionLifecycle: nativeRequired, + artifacts: [ + ...(nativeRequired ? ['native-extension-lifecycle-evidence'] : []), + ...(wasixRequired ? ['wasix-release-regression-evidence'] : []), + ], }, evidence: { wasixReleaseRegression: wasixEvidence, + nativeExtensionLifecycle: nativeEvidence, }, }; diff --git a/.github/workflows/broker-runtime.yml b/.github/workflows/broker-runtime.yml index 5bdd279b4..4883f0527 100644 --- a/.github/workflows/broker-runtime.yml +++ b/.github/workflows/broker-runtime.yml @@ -2,6 +2,10 @@ name: broker-runtime on: workflow_call: inputs: + cache-save-if: + description: Resolved cache-save policy from the calling CI workflow. + required: true + type: boolean matrix: required: true type: string @@ -14,7 +18,6 @@ env: OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ inputs.job-targets }} CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - HEAVY_CACHE_SAVE_IF: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} defaults: run: shell: bash @@ -39,6 +42,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} - name: Set up MSVC if: ${{ runner.os == 'Windows' }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34e5d8cd5..1843c01db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,10 +70,7 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - NPM_VERSION: 11.18.0 - DENO_VERSION: v2.8.1 ACTIONLINT_VERSION: 1.7.12 - ASSET_PROFILE: release WASMER_LLVM_VERSION: "22.1" WASMER_LLVM_LINUX_X64_URL: https://github.com/wasmerio/llvm-custom-builds/releases/download/22.x/llvm-linux-amd64.tar.xz WASMER_LLVM_LINUX_X64_SHA256: 5fb1c687c5e895d517a23e7aabea9ec3557e3a3e33f8a8d3a8d21395157b3906 @@ -98,6 +95,12 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: + liboliphaunt_wasix_aot_runtime_matrix_linux: ${{ steps.plan.outputs.liboliphaunt_wasix_aot_runtime_matrix_linux }} + liboliphaunt_wasix_aot_runtime_matrix_other: ${{ steps.plan.outputs.liboliphaunt_wasix_aot_runtime_matrix_other }} + wasix_napi_targets: ${{ steps.plan.outputs.wasix_napi_targets }} + reuse_ios_carrier: ${{ steps.plan.outputs.reuse_ios_carrier }} + cache_save_if: ${{ env.HEAVY_CACHE_SAVE_IF }} + llvm_version: ${{ env.WASMER_LLVM_VERSION }} published_dependencies: ${{ steps.plan.outputs.published_dependencies }} mobile_extension_package_native_targets_ios_csv: ${{ steps.plan.outputs.mobile_extension_package_native_targets_ios_csv }} mobile_extension_package_native_targets_android_csv: ${{ steps.plan.outputs.mobile_extension_package_native_targets_android_csv }} @@ -155,7 +158,7 @@ jobs: qualification_mode: ${{ steps.plan.outputs.qualification_mode }} qualification_base_sha: ${{ steps.plan.outputs.qualification_base_sha }} qualification_head_sha: ${{ steps.plan.outputs.qualification_head_sha }} - wasix_release_regression_required: ${{ contains(fromJson(steps.plan.outputs.jobs), 'liboliphaunt-wasix-runtime') && (github.event_name != 'workflow_dispatch' || inputs.wasm_target == 'all' || inputs.wasm_target == 'linux-x64-gnu') }} + wasix_release_regression_required: ${{ contains(fromJson(steps.plan.outputs.jobs), 'wasix-release-regression') && (github.event_name != 'workflow_dispatch' || inputs.wasm_target == 'all' || inputs.wasm_target == 'linux-x64-gnu') }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -167,8 +170,17 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon with: + task-cache: "false" install-workspace: "false" + - name: Restore frozen iOS carrier metadata + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/ci/ios-carrier + key: ios-carrier-v1-${{ hashFiles('src/native/runtime/VERSION', 'src/native/sdks/swift/tools/ios-carrier-manifest.mts', 'tools/packaging/**', 'tools/release/**') }}-${{ github.event.pull_request.head.sha || github.sha }} + restore-keys: | + ios-carrier-v1-${{ hashFiles('src/native/runtime/VERSION', 'src/native/sdks/swift/tools/ios-carrier-manifest.mts', 'tools/packaging/**', 'tools/release/**') }}- + - name: Plan artifact builder jobs id: plan env: @@ -192,6 +204,15 @@ jobs: MOON_HEAD: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }} run: bash .github/scripts/write-affected-moon-target-matrices.sh + - name: Transfer verified unchanged iOS carrier metadata + if: ${{ steps.plan.outputs.reuse_ios_carrier == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: frozen-ios-carrier + path: target/ci/ios-carrier + if-no-files-found: error + retention-days: 30 + - name: Upload build plan uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: @@ -218,6 +239,7 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon with: + task-cache: "false" install-workspace: "false" - name: Check release intent @@ -266,6 +288,8 @@ jobs: - name: Set up Rust if: ${{ matrix.requires_rust }} uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Install Android Rust targets if: ${{ matrix.requires_rust && matrix.requires_android_sdk }} @@ -284,11 +308,15 @@ jobs: - name: Set up Android if: ${{ matrix.requires_android_sdk }} uses: ./.github/actions/setup-android + with: + native-tools: ${{ matrix.requires_rust && 'true' || 'false' }} + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Wasmer LLVM if: ${{ matrix.requires_wasmer_llvm }} uses: ./.github/actions/setup-wasmer-llvm with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} url: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_URL || env.WASMER_LLVM_LINUX_X64_URL }} sha256: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_SHA256 || env.WASMER_LLVM_LINUX_X64_SHA256 }} bytes: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_BYTES || env.WASMER_LLVM_LINUX_X64_BYTES }} @@ -326,6 +354,8 @@ jobs: - name: Set up Rust if: ${{ needs.affected.outputs.policy_requires_rust == 'true' }} uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Swift if: ${{ needs.affected.outputs.policy_requires_swift == 'true' }} @@ -334,11 +364,14 @@ jobs: - name: Set up Android if: ${{ needs.affected.outputs.policy_requires_android_sdk == 'true' }} uses: ./.github/actions/setup-android + with: + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Wasmer LLVM if: ${{ needs.affected.outputs.policy_requires_wasmer_llvm == 'true' }} uses: ./.github/actions/setup-wasmer-llvm with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} url: ${{ env.WASMER_LLVM_LINUX_X64_URL }} sha256: ${{ env.WASMER_LLVM_LINUX_X64_SHA256 }} bytes: ${{ env.WASMER_LLVM_LINUX_X64_BYTES }} @@ -408,6 +441,7 @@ jobs: if: ${{ matrix.requires_rust }} uses: ./.github/actions/setup-rust with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} tools: cargo-nextest@0.9.137 - name: Set up Swift @@ -423,11 +457,15 @@ jobs: - name: Set up Android if: ${{ matrix.requires_android_sdk }} uses: ./.github/actions/setup-android + with: + native-tools: ${{ matrix.requires_rust && 'true' || 'false' }} + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Wasmer LLVM if: ${{ matrix.requires_wasmer_llvm }} uses: ./.github/actions/setup-wasmer-llvm with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} url: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_URL || env.WASMER_LLVM_LINUX_X64_URL }} sha256: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_SHA256 || env.WASMER_LLVM_LINUX_X64_SHA256 }} bytes: ${{ runner.os == 'macOS' && env.WASMER_LLVM_MACOS_ARM64_BYTES || env.WASMER_LLVM_LINUX_X64_BYTES }} @@ -470,39 +508,59 @@ jobs: run: bun .github/scripts/check-ci-gate.mts selected extension-artifacts-native-linux: + name: Native Extensions (Linux) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_linux).include[0] != null }} uses: ./.github/workflows/extension-artifacts-native.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_linux }} job-targets: ${{ needs.affected.outputs.job_targets }} - extension-artifacts-native-android: + extension-artifacts-native-android-static: + name: Native Extensions (Android Compilation) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_android).include[0] != null }} uses: ./.github/workflows/extension-artifacts-native.yml with: + phase: android-static + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} + matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_android }} + job-targets: ${{ needs.affected.outputs.job_targets }} + + extension-artifacts-native-android: + name: Native Extensions (Android Packaging) + needs: [affected, extension-artifacts-native-android-static, extension-artifacts-native-linux] + if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_android).include[0] != null }} + uses: ./.github/workflows/extension-artifacts-native.yml + with: + phase: android-package + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_android }} job-targets: ${{ needs.affected.outputs.job_targets }} extension-artifacts-native-ios: + name: Native Extensions (iOS) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_ios).include[0] != null }} uses: ./.github/workflows/extension-artifacts-native.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_ios }} job-targets: ${{ needs.affected.outputs.job_targets }} extension-artifacts-native-other: + name: Native Extensions (Desktop) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.extension_artifacts_native_matrix_other).include[0] != null }} uses: ./.github/workflows/extension-artifacts-native.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.extension_artifacts_native_matrix_other }} job-targets: ${{ needs.affected.outputs.job_targets }} extension-artifacts-native: - name: Builds / extension-artifacts-native + name: Builds / Native Extensions needs: [affected, extension-artifacts-native-linux, extension-artifacts-native-android, extension-artifacts-native-ios, extension-artifacts-native-other] if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'extension-artifacts-native') }} runs-on: ubuntu-24.04 @@ -517,7 +575,7 @@ jobs: esac extension-artifacts-wasix: - name: Builds / WASIX Extension Artifacts (${{ matrix.target }}) + name: Builds / WASIX Extensions (${{ matrix.target }}) needs: - affected - liboliphaunt-wasix-runtime @@ -540,6 +598,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download portable WASIX runtime outputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -568,7 +628,7 @@ jobs: if-no-files-found: error extension-packages: - name: Builds / Extension Packages + name: Packages / Extensions needs: - affected - extension-artifacts-native @@ -590,6 +650,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download native exact-extension artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -626,20 +688,24 @@ jobs: if-no-files-found: error mobile-extension-packages-android: + name: Extension Packages (Android) needs: [affected, extension-artifacts-native-android] if: ${{ needs.affected.outputs.mobile_extension_package_native_targets_android_csv != '' }} uses: ./.github/workflows/mobile-extension-packages.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} family: android targets: ${{ needs.affected.outputs.mobile_extension_package_native_targets_android_csv }} products: ${{ needs.affected.outputs.extension_package_products_csv }} job-targets: ${{ needs.affected.outputs.job_targets }} mobile-extension-packages-ios: + name: Extension Packages (iOS) needs: [affected, extension-artifacts-native-ios] if: ${{ needs.affected.outputs.mobile_extension_package_native_targets_ios_csv != '' }} uses: ./.github/workflows/mobile-extension-packages.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} family: ios targets: ${{ needs.affected.outputs.mobile_extension_package_native_targets_ios_csv }} products: ${{ needs.affected.outputs.extension_package_products_csv }} @@ -660,23 +726,27 @@ jobs: esac liboliphaunt-native-desktop-linux: + name: Native Runtime (Linux) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.liboliphaunt_native_desktop_runtime_matrix_linux).include[0] != null }} uses: ./.github/workflows/liboliphaunt-native-desktop.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.liboliphaunt_native_desktop_runtime_matrix_linux }} job-targets: ${{ needs.affected.outputs.job_targets }} liboliphaunt-native-desktop-other: + name: Native Runtime (Desktop) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.liboliphaunt_native_desktop_runtime_matrix_other).include[0] != null }} uses: ./.github/workflows/liboliphaunt-native-desktop.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.liboliphaunt_native_desktop_runtime_matrix_other }} job-targets: ${{ needs.affected.outputs.job_targets }} liboliphaunt-native-desktop: - name: Builds / liboliphaunt-native-desktop + name: Builds / Native Desktop Runtime needs: [affected, liboliphaunt-native-desktop-linux, liboliphaunt-native-desktop-other] if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-native-desktop') }} runs-on: ubuntu-24.04 @@ -715,6 +785,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up Android uses: ./.github/actions/setup-android @@ -807,6 +879,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Prepare native build paths env: @@ -866,7 +940,7 @@ jobs: if-no-files-found: ignore liboliphaunt-native-android-abi: - name: Builds / Native Runtime ABI Compatibility (Android) + name: Builds / Android ABI + Seeds needs: - affected - liboliphaunt-native-android @@ -883,6 +957,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download Android arm64 ABI receipts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -938,7 +1014,7 @@ jobs: if-no-files-found: error liboliphaunt-native-ios-abi: - name: Builds / Native Runtime ABI Compatibility (iOS) + name: Builds / iOS ABI + Seeds needs: - affected - liboliphaunt-native-ios @@ -955,6 +1031,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download iOS ABI receipts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -998,7 +1076,7 @@ jobs: if-no-files-found: error liboliphaunt-native-release-assets: - name: Builds / Native Runtime Release Assets + name: Packages / Native Runtime needs: - affected - liboliphaunt-native-android @@ -1019,9 +1097,13 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download liboliphaunt target release assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1070,7 +1152,7 @@ jobs: if-no-files-found: error rust-sdk-package: - name: Builds / Rust SDK Package + name: Packages / Rust SDK needs: - affected - checks @@ -1091,6 +1173,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Build Rust SDK package artifacts run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh rust-sdk-package @@ -1152,23 +1236,27 @@ jobs: if-no-files-found: error broker-runtime-linux: + name: Broker Runtime (Linux) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.broker_runtime_matrix_linux).include[0] != null }} uses: ./.github/workflows/broker-runtime.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.broker_runtime_matrix_linux }} job-targets: ${{ needs.affected.outputs.job_targets }} broker-runtime-other: + name: Broker Runtime (Desktop) needs: [affected, checks, tests] if: ${{ fromJson(needs.affected.outputs.broker_runtime_matrix_other).include[0] != null }} uses: ./.github/workflows/broker-runtime.yml with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} matrix: ${{ needs.affected.outputs.broker_runtime_matrix_other }} job-targets: ${{ needs.affected.outputs.job_targets }} broker-runtime: - name: Builds / broker-runtime + name: Builds / Broker Runtime needs: [affected, broker-runtime-linux, broker-runtime-other] if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'broker-runtime') }} runs-on: ubuntu-24.04 @@ -1207,6 +1295,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Set up MSVC if: ${{ runner.os == 'Windows' }} @@ -1230,7 +1320,7 @@ jobs: if-no-files-found: error broker-release-assets: - name: Builds / Broker Release Assets + name: Packages / Broker Runtime needs: - affected - broker-runtime @@ -1247,6 +1337,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download broker target release assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1262,7 +1354,7 @@ jobs: run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh broker-release-assets node-direct-release-assets: - name: Builds / Node.js Direct Release Assets + name: Packages / Node.js Direct needs: - affected - node-direct @@ -1279,6 +1371,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download Node direct target release assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1301,115 +1395,22 @@ jobs: run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh node-direct-release-assets wasix-napi: - name: Builds / WASIX Node-API (${{ matrix.target }}) - needs: - - affected - - extension-artifacts-wasix - - liboliphaunt-wasix-aot - - liboliphaunt-wasix-runtime - if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.extension-artifacts-wasix.result == 'success' && needs.liboliphaunt-wasix-aot.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && contains(fromJson(needs.affected.outputs.jobs), 'wasix-napi') }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.affected.outputs.wasix_napi_runtime_matrix) }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 + name: Builds / WASIX Node-API + needs: [affected, wasix-host-linux, wasix-host-other] + if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'wasix-napi') }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha || github.sha }} - persist-credentials: false - - - name: Set up Moon - uses: ./.github/actions/setup-moon - - - name: Set up Rust - uses: ./.github/actions/setup-rust - - - name: Set up MSVC - if: ${{ runner.os == 'Windows' }} - uses: ./.github/actions/setup-msvc - - - name: Download same-run portable WASIX outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: liboliphaunt-wasix-runtime-portable - path: . - - - name: Download same-run WASIX exact-extension outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - pattern: liboliphaunt-wasix-extension-artifacts-* - path: target/extensions/wasix/release-assets - merge-multiple: true - - - name: Download same-run target extension AOT outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: liboliphaunt-wasix-extension-aot-${{ matrix.target }} - path: target/extensions/wasix/aot-artifacts - - - name: Download same-run target core and tool AOT outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: liboliphaunt-wasix-runtime-aot-${{ matrix.target }} - path: target/oliphaunt-wasix/napi-aot-download - - - name: Restore exact target core and tool AOT layout - shell: bash + - name: Check selected hosts env: - EXPECTED_TARGET_TRIPLE: ${{ matrix.target_triple }} + RESULTS: ${{ join(needs.*.result, ',') }} run: | - artifact_dir=target/oliphaunt-wasix/napi-aot-download - marker="$artifact_dir/target-triple.txt" - raw_target_dir="$artifact_dir/files" - if [[ ! -f "$marker" || ! -d "$raw_target_dir" ]]; then - echo "invalid WASIX N-API AOT artifact envelope in $artifact_dir" >&2 - exit 1 - fi - target="$(tr -d '\r\n' < "$marker")" - if [[ "$target" != "${EXPECTED_TARGET_TRIPLE:?EXPECTED_TARGET_TRIPLE is required}" ]]; then - echo "WASIX N-API AOT artifact targets $target, expected $EXPECTED_TARGET_TRIPLE" >&2 - exit 1 - fi - destination="target/oliphaunt-wasix/aot/$target" - mkdir -p "$destination" - cp -R "$raw_target_dir/." "$destination/" - - - name: Set up macOS smoke timeout - if: ${{ runner.os == 'macOS' }} - shell: bash - run: brew list coreutils >/dev/null 2>&1 || HOMEBREW_NO_AUTO_UPDATE=1 brew install coreutils - - - name: Build WASIX Node-API release assets - shell: bash - env: - OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD: "1" - OLIPHAUNT_ICU_DATA_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/wasix-build/work/icu-wasix/share/icu - OLIPHAUNT_WASM_GENERATED_AOT_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/aot - OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT: ${{ github.workspace }}/target/extension-artifacts - OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/assets - OLIPHAUNT_WASIX_NAPI_ARTIFACT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["extension-artifacts-wasix:build-target", "extension-artifacts-wasix:build-aot", "liboliphaunt-wasix:runtime-aot"]' - run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-napi - - - name: Upload WASIX Node-API release assets - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: oliphaunt-wasix-napi-release-assets-${{ matrix.target }} - path: target/oliphaunt-wasix-napi/release-assets - if-no-files-found: error - - - name: Upload WASIX Node-API optional npm package - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: oliphaunt-wasix-napi-npm-package-${{ matrix.target }} - path: target/oliphaunt-wasix-napi/npm-packages/*.tgz - if-no-files-found: error + case ",$RESULTS," in + *,failure,*|*,cancelled,*) exit 1 ;; + esac wasix-napi-release-assets: - name: Builds / WASIX Node-API Release Assets + name: Packages / WASIX Node-API needs: - affected - wasix-napi @@ -1426,6 +1427,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download WASIX Node-API target release assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1464,6 +1467,8 @@ jobs: uses: ./.github/actions/setup-moon - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Install Apple Rust targets run: rustup target add aarch64-apple-ios aarch64-apple-ios-sim aarch64-apple-darwin - name: Select Apple framework toolchain @@ -1482,7 +1487,7 @@ jobs: if-no-files-found: error swift-sdk-package: - name: Builds / Swift SDK Package + name: Packages / Swift SDK needs: - affected - liboliphaunt-native-ios-abi @@ -1527,7 +1532,7 @@ jobs: if-no-files-found: error kotlin-sdk-package: - name: Builds / Kotlin SDK Package + name: Packages / Kotlin SDK needs: - affected - checks @@ -1548,6 +1553,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Install Android Rust targets run: rustup target add aarch64-linux-android x86_64-linux-android @@ -1568,7 +1575,7 @@ jobs: if-no-files-found: error kotlin-maven-staging: - name: Builds / Kotlin SDK Maven Staging + name: Packages / Kotlin Maven needs: - affected - kotlin-sdk-package @@ -1585,6 +1592,8 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Download exact same-run Kotlin SDK package artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1599,11 +1608,13 @@ jobs: run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh kotlin-maven-staging react-native-sdk-package: - name: Builds / React Native SDK Package + name: Packages / React Native SDK needs: - affected + - checks + - tests - liboliphaunt-native-ios-abi - if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'react-native-sdk-package') }} + if: ${{ !cancelled() && !failure() && needs.checks.result == 'success' && needs.tests.result == 'success' && (needs.affected.outputs.reuse_ios_carrier == 'true' || needs.liboliphaunt-native-ios-abi.result == 'success') && contains(fromJson(needs.affected.outputs.jobs), 'react-native-sdk-package') }} runs-on: ubuntu-24.04 timeout-minutes: 90 steps: @@ -1620,13 +1631,22 @@ jobs: install-workspace: "true" - name: Download Apple liboliphaunt release assets + if: ${{ needs.affected.outputs.reuse_ios_carrier != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: liboliphaunt-native-abi-compatible-release-assets-ios-datum64 path: target/liboliphaunt/abi-compatible-release-assets/ios-datum64 + - name: Download frozen iOS carrier metadata + if: ${{ needs.affected.outputs.reuse_ios_carrier == 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: frozen-ios-carrier + path: target/ci/ios-carrier + - name: Build React Native SDK package artifacts env: + OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER: ${{ needs.affected.outputs.reuse_ios_carrier == 'true' && 'target/ci/ios-carrier/manifest.json' || '' }} OLIPHAUNT_REACT_NATIVE_IOS_RELEASE_ASSET_DIR: ${{ github.workspace }}/target/liboliphaunt/abi-compatible-release-assets/ios-datum64 OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-native:finalize-runtime-ios-abi"]' run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh react-native-sdk-package @@ -1638,8 +1658,24 @@ jobs: path: target/sdk-artifacts/oliphaunt-react-native if-no-files-found: error + - name: Freeze current iOS carrier metadata + if: ${{ env.HEAVY_CACHE_SAVE_IF == 'true' }} + env: + SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p target/ci/ios-carrier + cp target/sdk-artifacts/oliphaunt-react-native/ios-carriers/oliphaunt-react-native-ios-carriers.json target/ci/ios-carrier/manifest.json + printf '%s\n' "$SOURCE_SHA" > target/ci/ios-carrier/source-sha + + - name: Save frozen iOS carrier metadata + if: ${{ env.HEAVY_CACHE_SAVE_IF == 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/ci/ios-carrier + key: ios-carrier-v1-${{ hashFiles('src/native/runtime/VERSION', 'src/native/sdks/swift/tools/ios-carrier-manifest.mts', 'tools/packaging/**', 'tools/release/**') }}-${{ github.event.pull_request.head.sha || github.sha }} + js-sdk-package: - name: Builds / JavaScript SDK Package + name: Packages / JavaScript SDK + ICU permissions: contents: read actions: read @@ -1667,8 +1703,6 @@ jobs: - name: Set up Deno uses: ./.github/actions/setup-deno - with: - deno-version: ${{ env.DENO_VERSION }} - name: Build TypeScript SDK package artifacts run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh js-sdk-package @@ -1728,11 +1762,11 @@ jobs: bun .github/scripts/moon-producer-receipt.mts oliphaunt-query-ts:package oliphaunt-query-ts-sdk-package-artifacts src/query/ts native-consumers: - name: Builds / Native Product Consumers - needs: [affected, js-sdk-package, rust-sdk-package, liboliphaunt-native-desktop, broker-runtime, node-direct] + name: Tests / Native Consumers + needs: [affected, js-sdk-package, rust-sdk-package, liboliphaunt-native-desktop-linux, broker-runtime-linux, node-direct] if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && contains(fromJson(needs.affected.outputs.jobs), 'native-consumers') }} runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 45 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -1746,11 +1780,21 @@ jobs: install-workspace: "true" - name: Set up Deno uses: ./.github/actions/setup-deno - with: - deno-version: ${{ env.DENO_VERSION }} - name: Set up Rust - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-integration') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-swift:test-native') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-kotlin:test-native-bindings') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-mobile-bindings:test-native') }} uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + tools: nextest + - name: Set up Swift for native SDK tests + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-swift:test-native') }} + uses: ./.github/actions/setup-swift + - name: Set up Android SDK for Kotlin host tests + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-kotlin:test-native-bindings') }} + uses: ./.github/actions/setup-android + with: + native-tools: "false" + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download Rust SDK package for installed consumer if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1788,13 +1832,13 @@ jobs: name: oliphaunt-query-ts-sdk-package-artifacts path: target/sdk-artifacts/oliphaunt-query-ts - name: Download Linux runtime archive - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-js:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-js:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-integration') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-swift:test-native') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-kotlin:test-native-bindings') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-mobile-bindings:test-native') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: liboliphaunt-native-release-assets-linux-x64-gnu path: target/liboliphaunt/desktop-release-assets/linux-x64-gnu - name: Download Linux broker archive - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-js:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-js:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-broker:test-consumer') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-integration') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: oliphaunt-broker-release-assets-linux-x64-gnu @@ -1809,7 +1853,7 @@ jobs: if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} run: chmod 0755 target/oliphaunt-rust/release-consumer/oliphaunt-rust-release-consumer - name: Download Linux tools archive - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') }} + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-consumer-runtime') || contains(fromJson(needs.affected.outputs.job_targets)['native-consumers'], 'oliphaunt-rust:test-integration') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: postgres-tools-native-release-assets-linux-x64-gnu @@ -1826,15 +1870,78 @@ jobs: OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["oliphaunt-rust:test-consumer", "postgres-tools-native:package-assets", "oliphaunt-js:package", "oliphaunt-query-ts:package", "liboliphaunt-native:package-runtime-desktop-target", "oliphaunt-broker:build-release-assets", "oliphaunt-node-direct:build-release-assets"]' run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh native-consumers + wasix-ts-package: + name: Packages / WASIX TypeScript SDK + needs: [affected, checks, tests] + if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'wasix-ts-package') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Moon + uses: ./.github/actions/setup-moon + with: + install-workspace: "true" + + - name: Set up Rust and wasm-pack + uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + tools: wasm-pack@0.15.0 + + - name: Restore browser host compiler cache + id: browser-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/oliphaunt-wasix-ts/host/cargo + key: wasix-browser-${{ runner.os }}-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('src/wasix/browser-host/**') }} + restore-keys: | + wasix-browser-${{ runner.os }}-${{ hashFiles('rust-toolchain.toml') }}- + + - name: Package WASIX TypeScript SDK + run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-ts-package + + - name: Save browser host compiler cache + if: ${{ env.HEAVY_CACHE_SAVE_IF == 'true' && steps.browser-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/oliphaunt-wasix-ts/host/cargo + key: ${{ steps.browser-cache.outputs.cache-primary-key }} + + - name: Upload WASIX TypeScript SDK package artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: oliphaunt-wasix-ts-sdk-package-artifacts + path: target/sdk-artifacts/oliphaunt-wasix-ts + if-no-files-found: error + + - name: Upload WASIX TypeScript consumer inputs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: oliphaunt-wasix-ts-consumer-inputs + path: | + target/sdk-artifacts/oliphaunt-wasix-ts + target/oliphaunt-wasix-ts/package + target/oliphaunt-wasix-ts/host/wasmer-sdk + src/wasix/sdks/ts/lib + if-no-files-found: error + retention-days: 30 + wasix-ts-sdk-package: - name: Builds / WASIX TypeScript SDK + name: Tests / WASIX TypeScript Consumers needs: - affected + - wasix-ts-package - liboliphaunt-wasix-runtime - - wasix-napi - js-sdk-package - - liboliphaunt-wasix-aot - if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && (!contains(fromJson(needs.affected.outputs.jobs), 'wasix-napi') || needs.wasix-napi.result == 'success') && (!contains(fromJson(needs.affected.outputs.jobs), 'js-sdk-package') || needs.js-sdk-package.result == 'success') && (!contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-aot') || needs.liboliphaunt-wasix-aot.result == 'success') && contains(fromJson(needs.affected.outputs.jobs), 'wasix-ts-sdk-package') }} + - wasix-host-linux + if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.wasix-ts-package.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && (!contains(fromJson(needs.affected.outputs.jobs), 'wasix-napi') || needs.wasix-host-linux.result == 'success') && (!contains(fromJson(needs.affected.outputs.jobs), 'js-sdk-package') || needs.js-sdk-package.result == 'success') && (!contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-aot') || needs.wasix-host-linux.result == 'success') && contains(fromJson(needs.affected.outputs.jobs), 'wasix-ts-sdk-package') }} runs-on: ubuntu-24.04 timeout-minutes: 120 steps: @@ -1850,15 +1957,14 @@ jobs: with: install-workspace: "true" - - name: Set up Rust and wasm-pack - uses: ./.github/actions/setup-rust + - name: Download WASIX TypeScript consumer inputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: - tools: wasm-pack@0.15.0 + name: oliphaunt-wasix-ts-consumer-inputs + path: . - name: Set up Deno uses: ./.github/actions/setup-deno - with: - deno-version: ${{ env.DENO_VERSION }} - name: Download same-run portable WASIX outputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -1915,21 +2021,14 @@ jobs: name: database-resources-icu-npm path: target/release/npm-packages/oliphaunt-icu - - name: Build, test, and package the WASIX TypeScript SDK + - name: Test WASIX TypeScript consumers env: - OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-portable", "extension-artifacts-wasix:compiler-output", "database-resources:build-wasix-standard", "database-resources:build-wasix-icu", "database-resources:package-icu", "oliphaunt-wasix-napi:build-release-assets", "oliphaunt-wasix-tools-ts:package", "postgres-tools-wasix:package-portable", "postgres-tools-wasix:package-aot"]' + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["oliphaunt-wasix-ts:build", "wasix-browser-host:build", "oliphaunt-wasix-ts:package", "liboliphaunt-wasix:runtime-portable", "extension-artifacts-wasix:compiler-output", "database-resources:build-wasix-standard", "database-resources:build-wasix-icu", "database-resources:package-icu", "oliphaunt-wasix-napi:build-release-assets", "oliphaunt-wasix-tools-ts:package", "postgres-tools-wasix:package-portable", "postgres-tools-wasix:package-aot"]' run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-ts-sdk-package - - name: Upload WASIX TypeScript SDK package artifacts - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['wasix-ts-sdk-package'], 'oliphaunt-wasix-ts:package') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: oliphaunt-wasix-ts-sdk-package-artifacts - path: target/sdk-artifacts/oliphaunt-wasix-ts - if-no-files-found: error wasix-rust-package: - name: Builds / WASIX Rust Binding Package + name: Packages / WASIX Rust Binding needs: - affected - checks @@ -1952,6 +2051,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Build Rust WASIX binding package artifacts run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh wasix-rust-package @@ -1973,7 +2074,7 @@ jobs: if-no-files-found: error liboliphaunt-wasix-runtime: - name: Builds / liboliphaunt WASIX Runtime + name: Builds / WASIX Portable Runtime needs: - affected - checks @@ -1981,6 +2082,8 @@ jobs: if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-runtime') }} runs-on: ubuntu-24.04 timeout-minutes: 360 + env: + ASSET_PROFILE: release steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -2023,33 +2126,14 @@ jobs: target/oliphaunt-wasix/wasix-build/work/geos-wasix target/oliphaunt-wasix/wasix-build/work/geos-wasix-build target/oliphaunt-wasix/wasix-build/build - key: wasix-build-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ env.WASMER_LLVM_VERSION }}-${{ env.WASMER_LLVM_LINUX_X64_BYTES }}-${{ env.WASMER_LLVM_LINUX_X64_SHA256 }}-${{ github.event.pull_request.head.sha || github.sha }} + key: wasix-build-v2-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ hashFiles('tools/dev/acquisition.sh', 'src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore', 'src/wasix/runtime/toolchain.toml', 'src/wasix/runtime/assets/build/docker/Dockerfile', 'src/wasix/runtime/assets/build/docker/*.sh', '!src/wasix/runtime/assets/build/docker/*.test.sh') }}-${{ github.event.pull_request.head.sha || github.sha }} restore-keys: | - wasix-build-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ env.WASMER_LLVM_VERSION }}-${{ env.WASMER_LLVM_LINUX_X64_BYTES }}-${{ env.WASMER_LLVM_LINUX_X64_SHA256 }}-${{ github.event.pull_request.base.sha || github.event.before || github.sha }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - - name: Build WASIX builder image and save cache - if: ${{ env.HEAVY_CACHE_SAVE_IF == 'true' }} - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f - with: - context: src/wasix/runtime/assets/build/docker - file: src/wasix/runtime/assets/build/docker/Dockerfile - tags: oliphaunt-wasix-wasix-build:ci - load: true - cache-from: type=gha,scope=wasix-builder - cache-to: type=gha,mode=max,scope=wasix-builder,ignore-error=true + wasix-build-v2-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ hashFiles('tools/dev/acquisition.sh', 'src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore', 'src/wasix/runtime/toolchain.toml', 'src/wasix/runtime/assets/build/docker/Dockerfile', 'src/wasix/runtime/assets/build/docker/*.sh', '!src/wasix/runtime/assets/build/docker/*.test.sh') }}- - - name: Build WASIX builder image - if: ${{ env.HEAVY_CACHE_SAVE_IF != 'true' }} - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f + - name: Set up WASIX builder + uses: ./.github/actions/setup-wasix-builder with: - context: src/wasix/runtime/assets/build/docker - file: src/wasix/runtime/assets/build/docker/Dockerfile - tags: oliphaunt-wasix-wasix-build:ci - load: true - cache-from: type=gha,scope=wasix-builder + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Install Wasmer LLVM 22.1 for WASIX cluster-seed generation uses: ./.github/actions/setup-wasmer-llvm @@ -2093,7 +2177,7 @@ jobs: target/oliphaunt-wasix/wasix-build/work/geos-wasix target/oliphaunt-wasix/wasix-build/work/geos-wasix-build target/oliphaunt-wasix/wasix-build/build - key: wasix-build-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ env.WASMER_LLVM_VERSION }}-${{ env.WASMER_LLVM_LINUX_X64_BYTES }}-${{ env.WASMER_LLVM_LINUX_X64_SHA256 }}-${{ github.event.pull_request.head.sha || github.sha }} + key: wasix-build-v2-${{ runner.os }}-${{ env.ASSET_PROFILE }}-${{ hashFiles('tools/dev/acquisition.sh', 'src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore', 'src/wasix/runtime/toolchain.toml', 'src/wasix/runtime/assets/build/docker/Dockerfile', 'src/wasix/runtime/assets/build/docker/*.sh', '!src/wasix/runtime/assets/build/docker/*.test.sh') }}-${{ github.event.pull_request.head.sha || github.sha }} - name: Upload portable WASIX tools compiler outputs if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-runtime'], 'postgres-tools-wasix:compiler-output') }} @@ -2139,121 +2223,47 @@ jobs: src/wasix/runtime/assets/generated/** if-no-files-found: error + wasix-host-linux: + name: WASIX (Linux) + needs: [affected, liboliphaunt-wasix-runtime, extension-artifacts-wasix] + if: ${{ !cancelled() && !failure() && needs.liboliphaunt-wasix-runtime.result == 'success' && (!contains(fromJson(needs.affected.outputs.jobs), 'extension-artifacts-wasix') || needs.extension-artifacts-wasix.result == 'success') && fromJson(needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix_linux).include[0] != null }} + uses: ./.github/workflows/wasix-host.yml + with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} + matrix: ${{ needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix_linux }} + job-targets: ${{ needs.affected.outputs.job_targets }} + napi-targets: ${{ needs.affected.outputs.wasix_napi_targets }} + llvm-version: ${{ needs.affected.outputs.llvm_version }} + + wasix-host-other: + name: WASIX (Desktop) + needs: [affected, liboliphaunt-wasix-runtime, extension-artifacts-wasix] + if: ${{ !cancelled() && !failure() && needs.liboliphaunt-wasix-runtime.result == 'success' && (!contains(fromJson(needs.affected.outputs.jobs), 'extension-artifacts-wasix') || needs.extension-artifacts-wasix.result == 'success') && fromJson(needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix_other).include[0] != null }} + uses: ./.github/workflows/wasix-host.yml + with: + cache-save-if: ${{ needs.affected.outputs.cache_save_if == 'true' }} + matrix: ${{ needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix_other }} + job-targets: ${{ needs.affected.outputs.job_targets }} + napi-targets: ${{ needs.affected.outputs.wasix_napi_targets }} + llvm-version: ${{ needs.affected.outputs.llvm_version }} + liboliphaunt-wasix-aot: - name: Builds / liboliphaunt WASIX AOT (${{ matrix.target_id }}) - needs: - - affected - - liboliphaunt-wasix-runtime - if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-aot') }} - runs-on: ${{ matrix.os }} - timeout-minutes: 180 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.affected.outputs.liboliphaunt_wasix_aot_runtime_matrix || '{"include":[]}') }} + name: Builds / WASIX AOT + needs: [affected, wasix-host-linux, wasix-host-other] + if: ${{ !cancelled() && !failure() && contains(fromJson(needs.affected.outputs.jobs), 'liboliphaunt-wasix-aot') }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha || github.sha }} - persist-credentials: false - - - name: Set up Moon - uses: ./.github/actions/setup-moon - - - name: Set up Rust - uses: ./.github/actions/setup-rust - - - name: Set up MSVC - if: ${{ runner.os == 'Windows' }} - uses: ./.github/actions/setup-msvc - - - name: Download portable WASIX build outputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: liboliphaunt-wasix-runtime-portable - path: . - - - name: Download portable WASIX tools compiler outputs - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:build-aot') }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: postgres-tools-wasix-compiler-output - path: target/postgres-tools/wasix/assets - - - name: Download portable WASIX extension compiler outputs - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'extension-artifacts-wasix:build-aot') }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c - with: - name: extensions-wasix-compiler-output - path: target/extensions/wasix/assets - - - name: Install Wasmer LLVM 22.1 for AOT generation - uses: ./.github/actions/setup-wasmer-llvm - with: - url: ${{ matrix.llvm_url }} - sha256: ${{ matrix.llvm_sha256 }} - bytes: ${{ matrix.llvm_bytes }} - version: ${{ env.WASMER_LLVM_VERSION }} - - - name: Build, validate, and smoke target AOT artifacts - env: - AOT_TARGET: ${{ matrix.target }} - OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-portable", "postgres-tools-wasix:compiler-output", "extension-artifacts-wasix:compiler-output"]' - run: OLIPHAUNT_CI_JOB_TARGETS_JSON='${{ needs.affected.outputs.job_targets }}' .github/scripts/run-planned-moon-job.sh liboliphaunt-wasix-aot - - - name: Upload target PostgreSQL WASIX tools - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:package-aot') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: postgres-tools-wasix-release-assets-${{ matrix.target_id }} - path: target/postgres-tools/wasix/release-assets/*-aot-${{ matrix.target_id }}.tar.gz - if-no-files-found: error - - - name: Stage target AOT artifact envelope - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'liboliphaunt-wasix:runtime-aot') }} + - name: Check selected hosts env: - AOT_TARGET: ${{ matrix.target }} + RESULTS: ${{ join(needs.*.result, ',') }} run: | - target="${AOT_TARGET:?AOT_TARGET is required}" - source="target/oliphaunt-wasix/aot/$target" - if [ ! -d "$source" ]; then - echo "missing AOT output directory: $source" >&2 - exit 1 - fi - upload="target/oliphaunt-wasix/aot-upload" - rm -rf "$upload" - mkdir -p "$upload/files" - cp -R "$source/." "$upload/files/" - printf '%s\n' "$target" >"$upload/target-triple.txt" - - - name: Upload target artifacts - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'liboliphaunt-wasix:runtime-aot') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: liboliphaunt-wasix-runtime-aot-${{ matrix.target_id }} - path: | - target/oliphaunt-wasix/aot-upload/** - if-no-files-found: error - - - name: Upload target WASIX tools compiler outputs - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:build-aot') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: postgres-tools-wasix-aot-${{ matrix.target_id }} - path: target/postgres-tools/wasix/aot - if-no-files-found: error - - - name: Upload target extension AOT artifacts - if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['liboliphaunt-wasix-aot'], 'extension-artifacts-wasix:build-aot') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - with: - name: liboliphaunt-wasix-extension-aot-${{ matrix.target_id }} - path: target/extensions/wasix/aot-artifacts - if-no-files-found: error + case ",$RESULTS," in + *,failure,*|*,cancelled,*) exit 1 ;; + esac liboliphaunt-wasix-release-assets: - name: Builds / liboliphaunt WASIX Release Assets + name: Packages / WASIX Runtime needs: - affected - liboliphaunt-wasix-runtime @@ -2271,9 +2281,13 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + with: + task-cache: "false" - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download portable WASIX runtime outputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -2320,7 +2334,7 @@ jobs: if-no-files-found: error wasix-postmaster-portable: - name: Builds / WASIX Postmaster / Portable + qualification + name: Builds / WASIX Postmaster (Portable + Tests) needs: - affected - checks @@ -2339,8 +2353,21 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon + - name: Set up WASIX builder + uses: ./.github/actions/setup-wasix-builder + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + # Runtime Cargo commands use the executor workspace's patched dependency config. + cache-workspaces: | + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/wasmer/target + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/postmaster-executor-target + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/postmaster-compiler-target + src/wasix/postmaster/tools/sealed-export-closure -> ../../../../../target/oliphaunt-wasix-postmaster/runtime/sealed-export-closure-target - name: Install Wasmer LLVM 22.1 uses: ./.github/actions/setup-wasmer-llvm @@ -2395,7 +2422,7 @@ jobs: retention-days: 3 wasix-postmaster-target: - name: Builds / WASIX Postmaster / ${{ matrix.target_id }} + qualification + name: Builds / WASIX Postmaster (${{ matrix.target_id }} + Tests) needs: - affected - wasix-postmaster-portable @@ -2418,6 +2445,15 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + cache: ${{ matrix.target_id != 'linux-x64-gnu' }} + # Runtime Cargo commands use the executor workspace's patched dependency config. + cache-workspaces: | + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/wasmer/target + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/postmaster-executor-target + src/wasix/postmaster/executor -> ../../../../target/oliphaunt-wasix-postmaster/runtime/postmaster-compiler-target + src/wasix/postmaster/tools/sealed-export-closure -> ../../../../../target/oliphaunt-wasix-postmaster/runtime/sealed-export-closure-target - name: Install portable shell utilities if: ${{ runner.os == 'macOS' }} @@ -2433,6 +2469,7 @@ jobs: - name: Install Wasmer LLVM 22.1 uses: ./.github/actions/setup-wasmer-llvm with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} url: ${{ matrix.llvm_url }} sha256: ${{ matrix.llvm_sha256 }} bytes: ${{ matrix.llvm_bytes }} @@ -2479,7 +2516,7 @@ jobs: if-no-files-found: error wasix-postmaster: - name: Builds / WASIX Postmaster / Aggregate release assets + name: Packages / WASIX Postmaster needs: - affected - wasix-postmaster-target @@ -2547,8 +2584,6 @@ jobs: - name: Set up Deno for packed native tools smoke if: ${{ matrix.shard == 0 }} uses: ./.github/actions/setup-deno - with: - deno-version: ${{ env.DENO_VERSION }} - name: Download same-run Linux native runtime uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -2600,6 +2635,7 @@ jobs: name: native-extension-lifecycle-evidence-${{ matrix.shard }} path: target/native-extension-lifecycle/evidence if-no-files-found: error + overwrite: true retention-days: 30 native-extension-lifecycle-aggregate: @@ -2621,6 +2657,7 @@ jobs: - name: Set up Moon uses: ./.github/actions/setup-moon with: + task-cache: "false" install-workspace: "false" - name: Download all same-run native lifecycle shard receipts @@ -2650,6 +2687,9 @@ jobs: --candidate-tree "$candidate_tree" \ --expected-extensions-csv "$OLIPHAUNT_NATIVE_EXTENSION_PROOF_SQL_NAMES" \ --expected-shard-count "$OLIPHAUNT_NATIVE_EXTENSION_PROOF_SHARD_COUNT" \ + --repository "$GITHUB_REPOSITORY" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ --output target/native-extension-lifecycle/aggregate/output/aggregate-receipt.json - name: Upload aggregate native extension lifecycle evidence @@ -2660,16 +2700,18 @@ jobs: name: native-extension-lifecycle-evidence path: target/native-extension-lifecycle/aggregate if-no-files-found: error - retention-days: 30 + overwrite: true + retention-days: 90 wasix-release-regression: - name: E2E / WASIX Release Regression + name: E2E / WASIX Extension Lifecycle needs: - affected - extension-artifacts-wasix - liboliphaunt-wasix-runtime - - liboliphaunt-wasix-aot - if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.extension-artifacts-wasix.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && needs.liboliphaunt-wasix-aot.result == 'success' && needs.affected.outputs.wasix_release_regression_required == 'true' }} + - wasix-host-linux + - js-sdk-package + if: ${{ !cancelled() && !failure() && needs.affected.result == 'success' && needs.extension-artifacts-wasix.result == 'success' && needs.liboliphaunt-wasix-runtime.result == 'success' && needs.wasix-host-linux.result == 'success' && needs.affected.outputs.wasix_release_regression_required == 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 240 steps: @@ -2688,6 +2730,7 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} components: rustfmt - name: Download same-run portable WASIX outputs @@ -2751,20 +2794,31 @@ jobs: mkdir -p "$destination" rsync -a "$raw_target_dir/" "$destination/" + - name: Download same-run WASIX cluster seeds + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['wasix-release-regression'], 'oliphaunt-wasix-rust:test-integration') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: database-resources-wasix-seeds-portable + path: target/database-resources/release-assets + + - name: Download same-run ICU data + if: ${{ contains(fromJson(needs.affected.outputs.job_targets)['wasix-release-regression'], 'oliphaunt-wasix-rust:test-integration') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: database-resources-icu-data-portable + path: target/database-resources/release-assets + - name: Stage exact-extension WASIX evidence inputs run: tools/dev/bun.sh src/extensions/artifacts/packages/tools/build-extension-ci-artifacts.mts --all --family wasix --require-wasix - name: Collect WASIX extension evidence id: regression env: + OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ needs.affected.outputs.job_targets }} + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-aot", "extension-artifacts-wasix:build-target", "extension-artifacts-wasix:build-aot", "postgres-tools-wasix:build-aot", "database-resources:build-wasix-standard", "database-resources:build-wasix-icu", "database-resources:package-icu"]' CI_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT: ${{ github.workspace }}/target/extension-artifacts run: | - actual_sha="$(git rev-parse HEAD)" - if [[ "$actual_sha" != "$CI_HEAD_SHA" ]]; then - echo "checked-out candidate $actual_sha does not match requested SHA $CI_HEAD_SHA" >&2 - exit 1 - fi run_id="$(date -u +%Y-%m-%dT%H%M%SZ)-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${CI_HEAD_SHA}" run_path="src/extensions/evidence/runs/$run_id.json" echo "run_path=$run_path" >> "$GITHUB_OUTPUT" @@ -2827,10 +2881,14 @@ jobs: - name: Set up Android uses: ./.github/actions/setup-android with: + gradle-cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} + expo: "true" native-ccache: "true" - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Reclaim Android mobile build disk run: bash .github/scripts/reclaim-android-mobile-build-disk.sh @@ -2970,6 +3028,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ env.HEAVY_CACHE_SAVE_IF }} - name: Download iOS liboliphaunt target uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c @@ -3209,6 +3269,7 @@ jobs: - kotlin-maven-staging - react-native-sdk-package - js-sdk-package + - wasix-ts-package - wasix-ts-sdk-package - native-consumers - wasix-rust-package @@ -3216,7 +3277,6 @@ jobs: - liboliphaunt-wasix-aot - liboliphaunt-wasix-release-assets - wasix-postmaster - - wasix-release-regression - mobile-build-android - mobile-build-ios runs-on: ubuntu-24.04 @@ -3240,13 +3300,6 @@ jobs: GATE_LABEL: selected build jobs run: bun .github/scripts/check-ci-gate.mts selected - - name: Check WASIX release regression - id: wasix_regression_gate - env: - NEEDS_JSON: ${{ toJson(needs) }} - SELECTED_JOBS_JSON: ${{ needs.affected.outputs.wasix_release_regression_required == 'true' && '["wasix-release-regression"]' || '[]' }} - GATE_LABEL: WASIX release regression - run: bun .github/scripts/check-ci-gate.mts selected mobile-e2e-android: name: E2E / Android App @@ -3264,66 +3317,17 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - name: Set up Node and Bun - id: setup_android_e2e_node - uses: ./.github/actions/setup-node-bun - - - name: Reclaim Android emulator disk - id: reclaim_android_emulator_disk - run: bash .github/scripts/reclaim-android-mobile-build-disk.sh - - - name: Set up Android - id: setup_android_e2e - uses: ./.github/actions/setup-android - with: - gradle-cache: "false" - - - name: Set up Maestro - uses: ./.github/actions/setup-maestro - - name: Download Android app artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: react-native-mobile-android-app-android-x86_64 path: target/mobile-build/react-native/android - - name: List Android app artifact - run: find target/mobile-build/react-native/android -maxdepth 2 -type f -print - - - name: Provision runner KVM access - run: | - test -c /dev/kvm - if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then - sudo chmod a+rw /dev/kvm - fi - - - name: Start Android emulator - id: start_android_emulator - env: - OLIPHAUNT_ANDROID_EMULATOR_API: "35" - OLIPHAUNT_ANDROID_EMULATOR_DISK_HEADROOM_MB: "2048" - OLIPHAUNT_ANDROID_EMULATOR_PARTITION_SIZE_MB: "6144" - run: tools/ci/start-android-emulator-ci.sh - - - name: Run Android installed-app E2E - env: - CI_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - OLIPHAUNT_EXPO_ANDROID_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/android - OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE: release - OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE: "0" - OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER: maestro - run: bash src/native/sdks/react-native/tools/mobile-e2e.sh android - - - name: Upload Android E2E reports - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + - name: Test installed app + uses: ./.github/actions/run-mobile-e2e with: - name: react-native-mobile-android-e2e-reports - path: | - target/mobile/react-native/android-e2e/reports - target/mobile/react-native/android-e2e/logs - ${{ runner.temp }}/oliphaunt-android-emulator.log - if-no-files-found: ignore + platform: android + source-sha: ${{ github.event.pull_request.head.sha || github.sha }} mobile-e2e-ios: name: E2E / iOS App @@ -3341,59 +3345,24 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - - name: Set up Node and Bun - uses: ./.github/actions/setup-node-bun - - - name: Set up Apple - uses: ./.github/actions/setup-apple - - - name: Set up Maestro - uses: ./.github/actions/setup-maestro - - name: Download iOS app artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: react-native-mobile-ios-app path: target/mobile-build/react-native/ios-transport - - name: Verify and extract iOS app artifact - id: ios_app_transport_verify - run: | - rm -rf target/mobile-build/react-native/ios - bash src/native/sdks/react-native/tools/ios-app-transport.sh verify-extract \ - --transport-dir target/mobile-build/react-native/ios-transport \ - --output-dir target/mobile-build/react-native/ios - - - name: List iOS app artifact - run: find target/mobile-build/react-native/ios -maxdepth 2 -print - - - name: Run iOS installed-app E2E - env: - CI_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - OLIPHAUNT_EXPO_IOS_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/ios - OLIPHAUNT_EXPO_IOS_CONFIGURATION: Release - OLIPHAUNT_EXPO_IOS_SDK: iphonesimulator - OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE: "0" - OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER: maestro - MAESTRO_DRIVER_STARTUP_TIMEOUT: "300000" - run: bash src/native/sdks/react-native/tools/mobile-e2e.sh ios - - - name: Upload iOS E2E reports - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + - name: Test installed app + uses: ./.github/actions/run-mobile-e2e with: - name: react-native-mobile-ios-e2e-reports - path: | - target/mobile/react-native/ios-e2e/reports - target/mobile/react-native/ios-e2e/logs - target/mobile/react-native/ios-e2e/*.log - if-no-files-found: ignore + platform: ios + source-sha: ${{ github.event.pull_request.head.sha || github.sha }} e2e: name: E2E if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.action != 'closed') }} needs: - affected + - wasix-release-regression - native-extension-lifecycle-aggregate - mobile-e2e-android - mobile-e2e-ios @@ -3418,6 +3387,14 @@ jobs: GATE_LABEL: final release execution qualification run: bun .github/scripts/check-ci-gate.mts selected + - name: Check WASIX release regression + id: wasix_regression_gate + env: + NEEDS_JSON: ${{ toJson(needs) }} + SELECTED_JOBS_JSON: ${{ needs.affected.outputs.wasix_release_regression_required == 'true' && '["wasix-release-regression"]' || '[]' }} + GATE_LABEL: WASIX release regression + run: bun .github/scripts/check-ci-gate.mts selected + required: name: Required if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.action != 'closed') }} @@ -3500,6 +3477,13 @@ jobs: name: wasix-release-regression-evidence path: target/qualification/wasix-release-regression-evidence + - name: Download required same-run native evidence + if: ${{ contains(fromJson(needs.affected.outputs.jobs), 'native-extension-lifecycle') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: native-extension-lifecycle-evidence + path: target/qualification/native-extension-lifecycle-evidence + - name: Write exact-SHA qualification record id: qualification_record env: @@ -3508,6 +3492,7 @@ jobs: CI_PLAN_PATH: target/qualification/affected-plan/ci-plan.json WASIX_RELEASE_REGRESSION_REQUIRED: ${{ needs.affected.outputs.wasix_release_regression_required }} WASIX_EVIDENCE_ROOT: target/qualification/wasix-release-regression-evidence + NATIVE_EVIDENCE_ROOT: target/qualification/native-extension-lifecycle-evidence PRODUCER_RECEIPTS_JSON: ${{ format('[{0}]', needs.js-sdk-package.outputs.producer_receipt || '') }} run: bash .github/scripts/release-candidate.sh write diff --git a/.github/workflows/extension-artifacts-native.yml b/.github/workflows/extension-artifacts-native.yml index ba17956e0..7e908dd67 100644 --- a/.github/workflows/extension-artifacts-native.yml +++ b/.github/workflows/extension-artifacts-native.yml @@ -2,6 +2,14 @@ name: extension-artifacts-native on: workflow_call: inputs: + cache-save-if: + description: Resolved cache-save policy from the calling CI workflow. + required: true + type: boolean + phase: + description: all, android-static, or android-package + default: all + type: string matrix: required: true type: string @@ -14,7 +22,6 @@ env: OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ inputs.job-targets }} CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - HEAVY_CACHE_SAVE_IF: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} defaults: run: shell: bash @@ -49,7 +56,7 @@ jobs: uses: ./.github/actions/setup-apple - name: Set up Android - if: ${{ startsWith(matrix.target, 'android-') }} + if: ${{ startsWith(matrix.target, 'android-') && inputs.phase != 'android-package' }} uses: ./.github/actions/setup-android with: gradle-cache: "false" @@ -63,46 +70,76 @@ jobs: run: bash tools/dev/bun.sh test ./tools/packaging/windows-vc-runtime-closure.test.mts - name: Set up Rust + if: ${{ inputs.phase != 'android-package' }} uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} - name: Prepare native compiler cache path - if: ${{ matrix.target == 'ios-xcframework' }} + if: ${{ inputs.phase != 'android-package' && (matrix.target == 'ios-xcframework' || startsWith(matrix.target, 'android-')) }} run: mkdir -p "$CCACHE_DIR" - name: Restore native compiler cache - id: restore_ios_extension_ccache - if: ${{ matrix.target == 'ios-xcframework' }} + id: restore_extension_ccache + if: ${{ inputs.phase != 'android-package' && (matrix.target == 'ios-xcframework' || startsWith(matrix.target, 'android-')) }} uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ env.CCACHE_DIR }} - key: liboliphaunt-native-extension-ccache-v2-${{ matrix.target }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-apple/**', '.github/scripts/setup-native-build-tools.sh', 'src/third-party/postgres/**', 'src/third-party/icu/**', 'src/third-party/openssl/**', 'src/native/runtime/sources/**', 'src/extensions/contracts/**', 'src/extensions/catalog/extensions.source.json', 'src/extensions/catalog/native-components.toml', 'src/extensions/external/postgis/tools/preprocess-sql.mts', 'src/extensions/contrib/postgres18.toml', 'src/extensions/external/*/source.toml', 'src/extensions/external/*/recipe.toml', 'src/extensions/external/*/dependencies/**/source.toml', 'src/extensions/external/*/dependencies/**/recipe.toml', 'src/extensions/external/*/patches/**', 'src/extensions/external/*/dependencies/**/patches/**', 'src/extensions/generated/extensions.catalog.json', 'src/extensions/generated/contrib-build.tsv', 'src/extensions/generated/pgxs-build.tsv', 'src/extensions/generated/mobile/static-extensions.tsv', 'src/extensions/generated/mobile/static-registry.json', 'src/native/runtime/bin/build-*.sh', '!src/native/runtime/bin/*.test.sh', 'src/extensions/artifacts/native/tools/build-windows-extensions.sh', 'src/extensions/artifacts/native/tools/windows-extension-sources.mts', 'src/extensions/external/postgis/tools/windows/**', 'src/native/runtime/bin/build-output.bash', 'src/native/runtime/bin/common.sh', 'src/native/runtime/bin/fetch-pinned-git-checkout.sh', 'src/third-party/icu/tools/build.sh', 'src/native/runtime/bin/mobile-*.sh', 'src/native/runtime/bin/postgis-dependency-cache.sh', 'src/native/runtime/bin/postgres-backend-objects.mk', 'src/native/postgres-tools/crates/tools/Cargo.toml', 'src/native/postgres-tools/crates/tools/build.rs', 'src/native/postgres-tools/crates/tools/src/**', 'src/native/runtime/include/**', 'src/native/runtime/patches/**', 'src/extensions/contrib/portable-uuid/**', 'src/native/runtime/postgres18/**', 'src/native/runtime/src/**') }} + key: liboliphaunt-native-extension-ccache-v2-${{ matrix.target }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-android/**', 'tools/dev/android-sdk.toml', '.github/actions/setup-apple/**', '.github/scripts/setup-native-build-tools.sh', 'src/third-party/postgres/**', 'src/third-party/icu/**', 'src/third-party/openssl/**', 'src/native/runtime/sources/**', 'src/extensions/contracts/**', 'src/extensions/catalog/extensions.source.json', 'src/extensions/catalog/native-components.toml', 'src/extensions/external/postgis/tools/preprocess-sql.mts', 'src/extensions/contrib/postgres18.toml', 'src/extensions/external/*/source.toml', 'src/extensions/external/*/recipe.toml', 'src/extensions/external/*/dependencies/**/source.toml', 'src/extensions/external/*/dependencies/**/recipe.toml', 'src/extensions/external/*/patches/**', 'src/extensions/external/*/dependencies/**/patches/**', 'src/extensions/generated/extensions.catalog.json', 'src/extensions/generated/contrib-build.tsv', 'src/extensions/generated/pgxs-build.tsv', 'src/extensions/generated/mobile/static-extensions.tsv', 'src/extensions/generated/mobile/static-registry.json', 'src/native/runtime/bin/build-*.sh', '!src/native/runtime/bin/*.test.sh', 'src/extensions/artifacts/native/tools/build-windows-extensions.sh', 'src/extensions/artifacts/native/tools/windows-extension-sources.mts', 'src/extensions/external/postgis/tools/windows/**', 'src/native/runtime/bin/build-output.bash', 'src/native/runtime/bin/common.sh', 'src/native/runtime/bin/fetch-pinned-git-checkout.sh', 'src/third-party/icu/tools/build.sh', 'src/native/runtime/bin/mobile-*.sh', 'src/native/runtime/bin/postgis-dependency-cache.sh', 'src/native/runtime/bin/postgres-backend-objects.mk', 'src/native/postgres-tools/crates/tools/Cargo.toml', 'src/native/postgres-tools/crates/tools/build.rs', 'src/native/postgres-tools/crates/tools/src/**', 'src/native/runtime/include/**', 'src/native/runtime/patches/**', 'src/extensions/contrib/portable-uuid/**', 'src/native/runtime/postgres18/**', 'src/native/runtime/src/**') }} restore-keys: | liboliphaunt-native-extension-ccache-v2-${{ matrix.target }}-${{ runner.os }}-${{ runner.arch }}- - name: Configure native compiler cache + if: ${{ inputs.phase != 'android-package' }} run: .github/scripts/setup-native-build-tools.sh + - name: Download Android static archives + if: ${{ inputs.phase == 'android-package' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: extension-static-${{ matrix.target }} + path: target/extension-inputs/static + + - name: Download Linux extension support + if: ${{ inputs.phase == 'android-package' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: extension-linux-support + path: target/extension-inputs/host + + - name: Restore extension package inputs + if: ${{ inputs.phase == 'android-package' }} + run: | + tar -xzf target/extension-inputs/static/static.tar.gz + tar -xzf target/extension-inputs/host/support.tar.gz + - name: Build native exact-extension artifacts timeout-minutes: 120 env: + OLIPHAUNT_EXTENSION_PHASE: ${{ inputs.phase }} OLIPHAUNT_EXTENSION_PRODUCTS: ${{ matrix.extensions_csv }} OLIPHAUNT_EXTENSION_TARGET: ${{ matrix.target }} OLIPHAUNT_BISON: /opt/homebrew/opt/bison/bin/bison - run: .github/scripts/run-planned-moon-job.sh extension-artifacts-native + run: | + if [[ "$OLIPHAUNT_EXTENSION_PHASE" == android-static ]]; then + OLIPHAUNT_RELEASE_FETCH_ASSETS=1 bash src/extensions/artifacts/native/tools/package-release-assets.sh + else + .github/scripts/run-planned-moon-job.sh extension-artifacts-native + fi - name: Validate produced iOS extension carriers id: validate_ios_extension_carriers if: ${{ matrix.target == 'ios-xcframework' }} run: bun src/native/runtime/tools/validate-ios-carrier-zips.mts --root target/extensions/native/release-assets/ios-xcframework - - name: Save bounded iOS exact-extension compiler cache - id: save_ios_extension_ccache - if: ${{ matrix.target == 'ios-xcframework' && env.HEAVY_CACHE_SAVE_IF == 'true' && steps.restore_ios_extension_ccache.outputs.cache-hit != 'true' }} + - name: Save native extension compiler cache + id: save_extension_ccache + if: ${{ (matrix.target == 'ios-xcframework' || startsWith(matrix.target, 'android-')) && inputs.phase != 'android-package' && inputs.cache-save-if && steps.restore_extension_ccache.outputs.cache-hit != 'true' }} continue-on-error: true uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: ${{ env.CCACHE_DIR }} - key: liboliphaunt-native-extension-ccache-v2-${{ matrix.target }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/actions/setup-apple/**', '.github/scripts/setup-native-build-tools.sh', 'src/third-party/postgres/**', 'src/third-party/icu/**', 'src/third-party/openssl/**', 'src/native/runtime/sources/**', 'src/extensions/contracts/**', 'src/extensions/catalog/extensions.source.json', 'src/extensions/catalog/native-components.toml', 'src/extensions/external/postgis/tools/preprocess-sql.mts', 'src/extensions/contrib/postgres18.toml', 'src/extensions/external/*/source.toml', 'src/extensions/external/*/recipe.toml', 'src/extensions/external/*/dependencies/**/source.toml', 'src/extensions/external/*/dependencies/**/recipe.toml', 'src/extensions/external/*/patches/**', 'src/extensions/external/*/dependencies/**/patches/**', 'src/extensions/generated/extensions.catalog.json', 'src/extensions/generated/contrib-build.tsv', 'src/extensions/generated/pgxs-build.tsv', 'src/extensions/generated/mobile/static-extensions.tsv', 'src/extensions/generated/mobile/static-registry.json', 'src/native/runtime/bin/build-*.sh', '!src/native/runtime/bin/*.test.sh', 'src/extensions/artifacts/native/tools/build-windows-extensions.sh', 'src/extensions/artifacts/native/tools/windows-extension-sources.mts', 'src/extensions/external/postgis/tools/windows/**', 'src/native/runtime/bin/build-output.bash', 'src/native/runtime/bin/common.sh', 'src/native/runtime/bin/fetch-pinned-git-checkout.sh', 'src/third-party/icu/tools/build.sh', 'src/native/runtime/bin/mobile-*.sh', 'src/native/runtime/bin/postgis-dependency-cache.sh', 'src/native/runtime/bin/postgres-backend-objects.mk', 'src/native/postgres-tools/crates/tools/Cargo.toml', 'src/native/postgres-tools/crates/tools/build.rs', 'src/native/postgres-tools/crates/tools/src/**', 'src/native/runtime/include/**', 'src/native/runtime/patches/**', 'src/extensions/contrib/portable-uuid/**', 'src/native/runtime/postgres18/**', 'src/native/runtime/src/**') }} + key: ${{ steps.restore_extension_ccache.outputs.cache-primary-key }} - name: Show native compiler cache stats if: ${{ always() && runner.os != 'Windows' }} @@ -113,7 +150,40 @@ jobs: echo "ccache was not installed before the build stopped" fi + - name: Pack Android static archives + if: ${{ inputs.phase == 'android-static' }} + env: + EXTENSION_TARGET: ${{ matrix.target }} + run: | + mkdir -p target/extension-inputs + tar -czf target/extension-inputs/static.tar.gz target/liboliphaunt-mobile-extension-release/"$EXTENSION_TARGET"/android-*/out + + - name: Upload Android static archives + if: ${{ inputs.phase == 'android-static' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: extension-static-${{ matrix.target }} + path: target/extension-inputs/static.tar.gz + retention-days: 30 + if-no-files-found: error + + - name: Pack Linux extension support + if: ${{ matrix.target == 'linux-x64-gnu' }} + run: | + mkdir -p target/extension-inputs + tar -czf target/extension-inputs/support.tar.gz target/liboliphaunt-pg18-linux-x64-gnu-extension-release/install + + - name: Upload Linux extension support + if: ${{ matrix.target == 'linux-x64-gnu' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: extension-linux-support + path: target/extension-inputs/support.tar.gz + retention-days: 30 + if-no-files-found: error + - name: Upload native exact-extension artifacts + if: ${{ inputs.phase != 'android-static' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: liboliphaunt-native-extension-artifacts-${{ matrix.target }} diff --git a/.github/workflows/liboliphaunt-native-desktop.yml b/.github/workflows/liboliphaunt-native-desktop.yml index 598118fe4..a84c5ddd1 100644 --- a/.github/workflows/liboliphaunt-native-desktop.yml +++ b/.github/workflows/liboliphaunt-native-desktop.yml @@ -2,6 +2,10 @@ name: liboliphaunt-native-desktop on: workflow_call: inputs: + cache-save-if: + description: Resolved cache-save policy from the calling CI workflow. + required: true + type: boolean matrix: required: true type: string @@ -14,7 +18,6 @@ env: OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ inputs.job-targets }} CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - HEAVY_CACHE_SAVE_IF: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} defaults: run: shell: bash @@ -45,6 +48,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} - name: Set up MSVC if: ${{ runner.os == 'Windows' }} diff --git a/.github/workflows/mobile-e2e.yml b/.github/workflows/mobile-e2e.yml index 341c1ab32..ad5530a98 100644 --- a/.github/workflows/mobile-e2e.yml +++ b/.github/workflows/mobile-e2e.yml @@ -88,22 +88,8 @@ jobs: ref: ${{ needs.resolve.outputs.sha }} persist-credentials: false - - name: Set up Node and Bun - id: setup_android_e2e_node - uses: ./.github/actions/setup-node-bun - - - name: Reclaim Android emulator disk - id: reclaim_android_emulator_disk - run: bash .github/scripts/reclaim-android-mobile-build-disk.sh - - - name: Set up Android - id: setup_android_e2e - uses: ./.github/actions/setup-android - with: - gradle-cache: "false" - - - name: Set up Maestro - uses: ./.github/actions/setup-maestro + - name: Set up Bun + uses: ./.github/actions/setup-bun - name: Download Android app artifact id: download_android_app @@ -122,41 +108,11 @@ jobs: --artifact react-native-mobile-android-app-android-x86_64 find target/mobile-build/react-native/android -maxdepth 2 -type f -print - - name: Provision runner KVM access - run: | - test -c /dev/kvm - if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then - sudo chmod a+rw /dev/kvm - fi - - - name: Start Android emulator - id: start_android_emulator - env: - OLIPHAUNT_ANDROID_EMULATOR_API: "35" - OLIPHAUNT_ANDROID_EMULATOR_DISK_HEADROOM_MB: "2048" - OLIPHAUNT_ANDROID_EMULATOR_PARTITION_SIZE_MB: "6144" - run: tools/ci/start-android-emulator-ci.sh - - - name: Run Android installed-app E2E - id: android_app_e2e - env: - CI_HEAD_SHA: ${{ needs.resolve.outputs.sha }} - OLIPHAUNT_EXPO_ANDROID_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/android - OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE: release - OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE: "0" - OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER: maestro - run: bash src/native/sdks/react-native/tools/mobile-e2e.sh android - - - name: Upload Android E2E reports - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + - name: Test installed app + uses: ./.github/actions/run-mobile-e2e with: - name: react-native-mobile-android-e2e-reports - path: | - target/mobile/react-native/android-e2e/reports - target/mobile/react-native/android-e2e/logs - ${{ runner.temp }}/oliphaunt-android-emulator.log - if-no-files-found: ignore + platform: android + source-sha: ${{ needs.resolve.outputs.sha }} ios: name: ios-installed-app @@ -173,14 +129,8 @@ jobs: ref: ${{ needs.resolve.outputs.sha }} persist-credentials: false - - name: Set up Node and Bun - uses: ./.github/actions/setup-node-bun - - - name: Set up Apple - uses: ./.github/actions/setup-apple - - - name: Set up Maestro - uses: ./.github/actions/setup-maestro + - name: Set up Bun + uses: ./.github/actions/setup-bun - name: Download iOS app artifact id: download_ios_app @@ -198,37 +148,11 @@ jobs: --job Builds \ --artifact react-native-mobile-ios-app - - name: Verify and extract iOS app artifact - id: verify_ios_app_transport - run: | - rm -rf target/mobile-build/react-native/ios - bash src/native/sdks/react-native/tools/ios-app-transport.sh verify-extract \ - --transport-dir target/mobile-build/react-native/ios-transport \ - --output-dir target/mobile-build/react-native/ios - find target/mobile-build/react-native/ios -maxdepth 2 -print - - - name: Run iOS installed-app E2E - id: ios_app_e2e - env: - CI_HEAD_SHA: ${{ needs.resolve.outputs.sha }} - OLIPHAUNT_EXPO_IOS_BUILD_ARTIFACT_DIR: ${{ github.workspace }}/target/mobile-build/react-native/ios - OLIPHAUNT_EXPO_IOS_CONFIGURATION: Release - OLIPHAUNT_EXPO_IOS_SDK: iphonesimulator - OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE: "0" - OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER: maestro - MAESTRO_DRIVER_STARTUP_TIMEOUT: "300000" - run: bash src/native/sdks/react-native/tools/mobile-e2e.sh ios - - - name: Upload iOS E2E reports - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + - name: Test installed app + uses: ./.github/actions/run-mobile-e2e with: - name: react-native-mobile-ios-e2e-reports - path: | - target/mobile/react-native/ios-e2e/reports - target/mobile/react-native/ios-e2e/logs - target/mobile/react-native/ios-e2e/*.log - if-no-files-found: ignore + platform: ios + source-sha: ${{ needs.resolve.outputs.sha }} required: name: E2E diff --git a/.github/workflows/mobile-extension-packages.yml b/.github/workflows/mobile-extension-packages.yml index c814f74df..849260043 100644 --- a/.github/workflows/mobile-extension-packages.yml +++ b/.github/workflows/mobile-extension-packages.yml @@ -2,6 +2,10 @@ name: Mobile extension packages on: workflow_call: inputs: + cache-save-if: + description: Resolved cache-save policy from the calling CI workflow. + required: true + type: boolean family: type: string required: true @@ -23,7 +27,7 @@ defaults: shell: bash jobs: package: - name: Builds / Mobile Extension Packages + name: Packages / Mobile Extensions runs-on: ubuntu-24.04 timeout-minutes: 30 steps: @@ -39,6 +43,8 @@ jobs: - name: Set up Rust uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} - name: Download native exact-extension artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 94d58620e..5996a5663 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -204,6 +204,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} REQUIRES_WASIX_EVIDENCE: ${{ steps.release_plan.outputs.requires_wasix_release_regression_evidence }} + REQUIRES_NATIVE_EVIDENCE: ${{ steps.release_plan.outputs.requires_native_extension_lifecycle_evidence }} PRODUCTS_JSON: ${{ steps.release_plan.outputs.products_json }} run: | qualification_args=( @@ -219,6 +220,9 @@ jobs: --artifact artifact-build-plan --artifact oliphaunt-release-candidate ) + if [[ "$REQUIRES_NATIVE_EVIDENCE" == true ]]; then + qualification_args+=(--artifact native-extension-lifecycle-evidence) + fi if [[ "$REQUIRES_WASIX_EVIDENCE" == true ]]; then qualification_args+=(--artifact wasix-release-regression-evidence) fi @@ -290,6 +294,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} REQUIRES_WASIX_EVIDENCE: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).requires_wasix_release_regression_evidence }} + REQUIRES_NATIVE_EVIDENCE: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).requires_native_extension_lifecycle_evidence }} PRODUCTS_JSON: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).products_json }} run: | qualification_args=( @@ -305,6 +310,9 @@ jobs: --artifact artifact-build-plan --artifact oliphaunt-release-candidate ) + if [[ "$REQUIRES_NATIVE_EVIDENCE" == true ]]; then + qualification_args+=(--artifact native-extension-lifecycle-evidence) + fi if [[ "$REQUIRES_WASIX_EVIDENCE" == true ]]; then qualification_args+=(--artifact wasix-release-regression-evidence) fi @@ -352,8 +360,22 @@ jobs: "$RELEASE_HEAD_SHA" \ target/release-candidate/wasix-evidence \ --run-id "$CI_RUN_ID" \ - --job "E2E / WASIX Release Regression" \ + --job "E2E / WASIX Extension Lifecycle" \ --artifact wasix-release-regression-evidence + - name: Download required exact-SHA native evidence + if: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).has_release_changes == 'true' && inputs.approval_run_id == '' && fromJSON(needs.plan-candidate.outputs.release_plan).requires_native_extension_lifecycle_evidence == 'true' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + CI_RUN_ID: ${{ steps.ci_qualification.outputs.run_id }} + run: | + bash .github/scripts/download-build-artifacts.sh \ + CI \ + "$RELEASE_HEAD_SHA" \ + target/release-candidate/native-evidence \ + --run-id "$CI_RUN_ID" \ + --job "E2E / Native Extension Lifecycle Evidence" \ + --artifact native-extension-lifecycle-evidence - name: Verify exact-SHA qualification record id: verify_qualification if: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).has_release_changes == 'true' && inputs.approval_run_id == '' }} @@ -361,12 +383,15 @@ jobs: PRODUCTS_JSON: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).products_json }} CI_RUN_ID: ${{ steps.ci_qualification.outputs.run_id }} WASIX_EVIDENCE_REQUIRED: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).requires_wasix_release_regression_evidence }} + NATIVE_EVIDENCE_REQUIRED: ${{ fromJSON(needs.plan-candidate.outputs.release_plan).requires_native_extension_lifecycle_evidence }} run: | bash .github/scripts/release-candidate.sh verify \ target/release-candidate/oliphaunt-release-candidate.json \ --plan target/release-candidate/affected-plan/ci-plan.json \ --qualification-mode release \ --products-json "$PRODUCTS_JSON" \ + --native-evidence-required "$NATIVE_EVIDENCE_REQUIRED" \ + --native-evidence-root target/release-candidate/native-evidence \ --wasix-evidence-required "$WASIX_EVIDENCE_REQUIRED" \ --wasix-evidence-root target/release-candidate/wasix-evidence - name: Require the explicitly selected prepared candidate diff --git a/.github/workflows/wasix-host.yml b/.github/workflows/wasix-host.yml new file mode 100644 index 000000000..05fe30b76 --- /dev/null +++ b/.github/workflows/wasix-host.yml @@ -0,0 +1,183 @@ +name: WASIX host +on: + workflow_call: + inputs: + matrix: + required: true + type: string + job-targets: + required: true + type: string + napi-targets: + required: true + type: string + llvm-version: + required: true + type: string + cache-save-if: + required: true + type: boolean +permissions: + contents: read +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + WASMER_LLVM_VERSION: ${{ inputs.llvm-version }} + OLIPHAUNT_CI_JOB_TARGETS_JSON: ${{ inputs.job-targets }} +defaults: + run: + shell: bash +jobs: + build: + name: WASIX host (${{ matrix.target_id }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: ${{ fromJson(inputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Moon + uses: ./.github/actions/setup-moon + + - name: Set up Rust + uses: ./.github/actions/setup-rust + with: + cache-save-if: ${{ inputs.cache-save-if }} + + - name: Set up MSVC + if: ${{ runner.os == 'Windows' }} + uses: ./.github/actions/setup-msvc + + - name: Download portable WASIX build outputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: liboliphaunt-wasix-runtime-portable + path: . + + - name: Download portable WASIX tools compiler outputs + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:build-aot') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: postgres-tools-wasix-compiler-output + path: target/postgres-tools/wasix/assets + + - name: Download portable WASIX extension compiler outputs + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'extension-artifacts-wasix:build-aot') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: extensions-wasix-compiler-output + path: target/extensions/wasix/assets + + - name: Install Wasmer LLVM 22.1 for AOT generation + uses: ./.github/actions/setup-wasmer-llvm + with: + cache-save-if: ${{ inputs.cache-save-if }} + url: ${{ matrix.llvm_url }} + sha256: ${{ matrix.llvm_sha256 }} + bytes: ${{ matrix.llvm_bytes }} + version: ${{ env.WASMER_LLVM_VERSION }} + + - name: Build, validate, and smoke target AOT artifacts + env: + AOT_TARGET: ${{ matrix.target }} + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["liboliphaunt-wasix:runtime-portable", "postgres-tools-wasix:compiler-output", "extension-artifacts-wasix:compiler-output"]' + run: .github/scripts/run-planned-moon-job.sh liboliphaunt-wasix-aot + + - name: Upload target PostgreSQL WASIX tools + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:package-aot') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: postgres-tools-wasix-release-assets-${{ matrix.target_id }} + path: target/postgres-tools/wasix/release-assets/*-aot-${{ matrix.target_id }}.tar.gz + if-no-files-found: error + + - name: Stage target AOT artifact envelope + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'liboliphaunt-wasix:runtime-aot') }} + env: + AOT_TARGET: ${{ matrix.target }} + run: | + target="${AOT_TARGET:?AOT_TARGET is required}" + source="target/oliphaunt-wasix/aot/$target" + if [ ! -d "$source" ]; then + echo "missing AOT output directory: $source" >&2 + exit 1 + fi + upload="target/oliphaunt-wasix/aot-upload" + rm -rf "$upload" + mkdir -p "$upload/files" + cp -R "$source/." "$upload/files/" + printf '%s\n' "$target" >"$upload/target-triple.txt" + + - name: Upload target artifacts + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'liboliphaunt-wasix:runtime-aot') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: liboliphaunt-wasix-runtime-aot-${{ matrix.target_id }} + path: | + target/oliphaunt-wasix/aot-upload/** + if-no-files-found: error + + - name: Upload target WASIX tools compiler outputs + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'postgres-tools-wasix:build-aot') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: postgres-tools-wasix-aot-${{ matrix.target_id }} + path: target/postgres-tools/wasix/aot + if-no-files-found: error + + - name: Upload target extension AOT artifacts + if: ${{ contains(fromJson(inputs.job-targets)['liboliphaunt-wasix-aot'], 'extension-artifacts-wasix:build-aot') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: liboliphaunt-wasix-extension-aot-${{ matrix.target_id }} + path: target/extensions/wasix/aot-artifacts + if-no-files-found: error + + - name: Download same-run WASIX exact-extension outputs + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + pattern: liboliphaunt-wasix-extension-artifacts-* + path: target/extensions/wasix/release-assets + merge-multiple: true + + - name: Set up macOS smoke timeout + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) && runner.os == 'macOS' }} + shell: bash + run: brew list coreutils >/dev/null 2>&1 || HOMEBREW_NO_AUTO_UPDATE=1 brew install coreutils + + - name: Build WASIX Node-API release assets + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) }} + shell: bash + env: + OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD: "1" + OLIPHAUNT_ICU_DATA_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/wasix-build/work/icu-wasix/share/icu + OLIPHAUNT_WASM_GENERATED_AOT_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/aot + OLIPHAUNT_WASIX_EXTENSION_ARTIFACT_ROOT: ${{ github.workspace }}/target/extension-artifacts + OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR: ${{ github.workspace }}/target/oliphaunt-wasix/assets + OLIPHAUNT_WASIX_NAPI_ARTIFACT_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON: '["extension-artifacts-wasix:build-target", "extension-artifacts-wasix:build-aot", "liboliphaunt-wasix:runtime-aot"]' + run: .github/scripts/run-planned-moon-job.sh wasix-napi + + - name: Upload WASIX Node-API release assets + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: oliphaunt-wasix-napi-release-assets-${{ matrix.target_id }} + path: target/oliphaunt-wasix-napi/release-assets + if-no-files-found: error + + - name: Upload WASIX Node-API optional npm package + if: ${{ contains(fromJson(inputs.napi-targets), matrix.target_id) }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: oliphaunt-wasix-napi-npm-package-${{ matrix.target_id }} + path: target/oliphaunt-wasix-napi/npm-packages/*.tgz + if-no-files-found: error diff --git a/src/benchmarks/perf/moon.yml b/src/benchmarks/perf/moon.yml index f491892d3..58823c1f9 100644 --- a/src/benchmarks/perf/moon.yml +++ b/src/benchmarks/perf/moon.yml @@ -118,6 +118,7 @@ tasks: bun src/wasix/sdks/ts/tools/stage-host.mts bash src/wasix/sdks/ts/tools/integration/smoke-browser.sh --benchmark deps: + - "oliphaunt-query-ts:build" - "perf-tools:wasix-plan" - "liboliphaunt-wasix:runtime-portable" - "database-resources:build-wasix-standard" diff --git a/src/benchmarks/wasix/README.md b/src/benchmarks/wasix/README.md index bf2b43d41..75ca62313 100644 --- a/src/benchmarks/wasix/README.md +++ b/src/benchmarks/wasix/README.md @@ -16,7 +16,7 @@ and `sdk-worker` ownership values, so Promise shape is not mistaken for main-thread safety. ```sh -bun run --cwd src/wasix/sdks/ts package:build +moon run oliphaunt-wasix-ts:package bash src/wasix/sdks/ts/tools/integration/smoke-browser.sh --benchmark # or: moon run perf-tools:wasix-browser-measure ``` @@ -43,7 +43,7 @@ the built SDK tree, every harness source, and the installed PGlite closure. For a harness smoke check without a full sample set, run: ```sh -bun run --cwd src/wasix/sdks/ts package:build +moon run oliphaunt-wasix-ts:package bash src/wasix/sdks/ts/tools/integration/smoke-browser.sh --benchmark --quick ``` diff --git a/src/docs/maintainers/assets.md b/src/docs/maintainers/assets.md index 43990109c..e1f8470cf 100644 --- a/src/docs/maintainers/assets.md +++ b/src/docs/maintainers/assets.md @@ -179,7 +179,9 @@ containing only `noble`, `noble-updates`, and `noble-security` with the `main` and `universe` components. Every update and install explicitly binds that source, disabled source-parts discovery (`Dir::Etc::sourceparts=-`), a reset list directory, and the verified CA bundle. A transient failure retries the -complete update/install transaction with a fixed bound; it never falls back to +complete update/install transaction within one 15-minute default deadline, +including APT retries and outer retry waits (see +[acquisition deadlines](testing.md#acquisition-deadlines)); it never falls back to a live mirror or disables TLS verification. `ca-certificates` is installed in the same pinned transaction as the builder packages. diff --git a/src/docs/maintainers/release.md b/src/docs/maintainers/release.md index 1b7b399be..c5ce660f3 100644 --- a/src/docs/maintainers/release.md +++ b/src/docs/maintainers/release.md @@ -13,7 +13,7 @@ input, for example `["oliphaunt-js"]`. Leave all platform selectors at `all`. Moon selects those owners' tasks, downstream compatibility checks and required producer dependencies. An empty product array retains the exhaustive audit. Publication accepts this record only for covered products at the exact candidate -SHA; it still verifies the immutable artifacts and any required WASIX evidence. +SHA; it still verifies the immutable artifacts and required native/WASIX evidence. Generated release PRs and their main merge automatically select the products whose Release Please manifest versions advance. Exact main pushes and eligible main dispatches can produce publishable qualification; PR checks use the same @@ -24,6 +24,21 @@ dispatches are not automatically repeated. Cross-commit producer reuse remains an explicit acceptance item rather than permission to substitute arbitrary older artifacts. +`Qualified` binds every planned native extension lifecycle aggregate and WASIX +regression proof to the candidate's source SHA/tree and CI repository/run. A +successful earlier attempt of that same run can supply proof when only failed +jobs are rerun; a different run or newer attempt cannot. Both aggregate proof +artifacts retain 90 days. The native artifact includes its complete shard +receipts, so it does not depend on the shorter-lived diagnostic shard uploads. + +Publishing native extension carriers, including runtime-owned contrib, requires +the native proof. Release revalidates the aggregate and every shard, checks the +published SQL extension set, and compares the evidence digest with the candidate +record. SDK-only publications continue to qualify their consumers against their +declared payload versions. WASIX publications retain the existing product-derived +regression requirement. Missing, changed, wrong-run or incomplete required proof +blocks publication; an overall successful CI result does not replace it. + ## Model A product owns its SemVer, changelog, source identity, Release Please component, diff --git a/src/docs/maintainers/testing.md b/src/docs/maintainers/testing.md index 0002752e3..c313d8cce 100644 --- a/src/docs/maintainers/testing.md +++ b/src/docs/maintainers/testing.md @@ -27,11 +27,12 @@ validation. - PR: Moon-affected `check` and `test` tasks, release intent, and the selected package, artifact, and E2E jobs. Measured `coverage` is an explicit local/manual lane; it is not part of the `Required` PR gate. -- Main: affected checks, builds, runtime tests, and selected E2E. This does not - currently emit a `Qualified` release record. -- Full manual: the complete selected source/runtime/package/E2E graph. Only - an exhaustive dispatch with all target selectors produces the exact-SHA - `Qualified` release record. Coverage and benchmarks remain optional. +- Main: affected checks, builds, runtime tests, and selected E2E. A release + version change selects its product qualification closure and can emit an + exact-SHA `Qualified` record. +- Manual: the selected product or full source/runtime/package/E2E graph. All + platform selectors must remain `all` for publishable qualification on main. + Coverage and benchmarks remain optional. - Release: package-native dry-runs, artifact manifests, checksums, attestations, registry checks, exact-extension evidence, binary compatibility-floor inspection, and selected artifact behavior evidence. @@ -119,6 +120,83 @@ Reusable benchmark datasets, benchmark plans, and published reports belong in `src/benchmarks/`. Executable benchmark harnesses belong in `src/benchmarks/perf/` unless the harness is intentionally part of a product's public developer API. +## Acquisition deadlines + +Repository-owned downloads use `tools/dev/acquisition.sh`. Start one budget at +an acquisition's entry point, before lock waits or transport, and reuse it for +all retries, mirrors and dependent requests. Nested operations can shorten the +budget but cannot restart it. Compilation and test execution have their own +budgets; do not wrap an entire build in an acquisition timeout. + +```sh +. "$repo_root/tools/dev/acquisition.sh" +oliphaunt_acquisition_start 'example sources' 900 +# Per-endpoint cap, attempts, retry delay, curl command, existing secure arguments. +oliphaunt_acquisition_curl 300 3 5 curl --fail --location \ + --proto '=https' --proto-redir '=https' --tlsv1.2 --output "$partial" "$url" +# For Git, package managers and source validation processes: +oliphaunt_acquisition_run 300 git fetch --depth=1 "$url" "$commit" +``` + +The caller owns URL/pin validation, TLS policy, archive limits, checksums, +validation and atomic promotion. Downloads must target a staging file with +`--output`, never append retry responses to stdout. Pass curl's connection, +size and low-speed limits normally, but let the helper own `--retry` and +`--max-time`. Its single attempts each receive the remaining time. Curl's +[`--retry-max-time`](https://curl.se/docs/manpage.html#--retry-max-time) alone +allows its final attempt to run past the retry timer. + +| Acquisition | Default total | Maximum per endpoint/command | +| --- | --- | --- | +| Source scope / individual source pin | 30 / 15 min | Git fetch and archive endpoint: 5 min | +| PostgreSQL source archive, both origins | 3 min | 90 sec per origin | +| WASIX builder APT update + install + retries | 15 min | Update: 5 min; install: remaining budget | +| WASIX compiler asset set | 30 min | 15 min per asset | +| Android SDK setup / emulator packages (separate operations) | 30 min each | Command-line-tools origin: 4 min; SDK packages: remaining budget | +| Moon + Proto + plugins | 15 min | 5 min per asset or registry request | +| Bun/Deno; Node; npm publisher | 5 min each | Bun/Deno origin: 2 min; others: remaining budget | +| Wasmer LLVM | 30 min | Remaining budget | +| Maintainer binary; winflexbison | 3 min each | Remaining budget | +| Swift signing keys | 2 min | Remaining budget | + +`OLIPHAUNT_ACQUISITION_TIMEOUT_SECONDS` overrides an operation's total (integer +1–7200); it never raises an endpoint cap or replenishes an enclosing operation. +Defaults allow cold installation while bounding repeated failures. The APT +budget is deliberately above observed normal cold transactions (roughly a +minute), below the observed 40-minute failure tail. Tune with hosted timings, +not another retry layer. Separate scripts/actions have separate transactions; +this is not a workflow-wide download allowance. + +Shell and curl suffice for bootstrap downloads. Git, APT and SDK-manager +processes require GNU `timeout` (`coreutils`, `brew install coreutils` on macOS). +The shared helper verifies GNU identity, preferring `gtimeout` and falling back +to `/usr/bin/timeout` when Windows System32 shadows Git Bash's executable. +Android setup checks the timer before treating an SDK installation as invalid; +the macOS setup action provisions Coreutils when missing. +Commands receive TERM on expiry and KILL five seconds later if needed, including +children in their process group. Expiry reports the acquisition label and a +nonzero status (124 on expiry, or 137 after a forced kill; owners may add their +own failure status). +Interrupted curl attempts are not retried or mirrored. Existing traps clean staging and preserve prior +installations; short atomic promotion and rollback finish outside the timed +child. Source validation/extraction also consumes the source budget. Other +installers' local validation and promotion are not independently hard-timed. +The portable shared clock uses epoch seconds; per-process timers bound active +work. Runner clock corrections can shift subsequent remaining-time calculations. + +The helper's test owns clock arithmetic and process termination; source and +installer fault tests own preservation, retry/failover and validation. Moon +inputs and WASIX recipe identity include the helper. Build the WASIX Dockerfile +from the repository root; its adjacent `Dockerfile.dockerignore` limits context +to the recipe and helper. To build from another checkout, set `REPO_ROOT` and +`WASIX_TOOLCHAIN_ROOT` together so the recipe and COPY inputs refer to that tree. + +Upstream actions and general dependency resolution (Cargo, npm, Homebrew, +Chocolatey, pip and host build-tool setup) retain their existing job/setup +budgets. This helper does not replace package-manager retry or installation +semantics. A deadline stops a bad acquisition promptly; it does not turn a +failed transfer into successful qualification. + ## Moon Tasks Moon task names are intentionally narrow: @@ -148,6 +226,25 @@ runtime evidence. React Native installed-app smokes delegate runtime materialization to the Expo platform scripts and hard-fail there if native artifacts cannot be built or located. +The Linux native consumer job also owns these uncached SDK runtime suites: + +- `oliphaunt-rust:test-integration`: native smoke and SQL behavior. +- `oliphaunt-mobile-bindings:test-native`: shared broker streaming and lifecycle. +- `oliphaunt-swift:test-native`: Swift native runtime behavior. +- `oliphaunt-kotlin:test-native-bindings`: Kotlin native binding behavior. + +They consume the same-run packaged runtime, with tools and broker archives when +needed. The shared `src/native/sdks/tests/with-runtime.sh` stages each invocation +in an isolated temporary directory; Moon retains the producers for local runs +and the hosted transfer runner substitutes downloaded artifacts. Swift/Kotlin +share one generated binding dependency. These suites require executed tests and +valid runtime inputs; zero tests or unavailable artifacts cannot pass as proof. + +`oliphaunt-wasix-rust:test-integration` runs the standard and ICU seed resource +tests in the existing WASIX regression job, consuming its portable runtime/AOT +and same-run database resource artifacts. This host coverage supplements the +installed mobile apps and per-platform package checks. + React Native installed-app smoke is split by platform: ```sh @@ -159,37 +256,21 @@ PR jobs run RN static, unit, Codegen, JSI, config-plugin, and package checks. Affected PR, main, and explicit manual lanes run the installed Android/iOS app smokes selected by the CI plan. -Installed-app E2E runner choice is closed, not a recurring research task. -Decision (2026-06-08): Oliphaunt uses the pinned open-source Maestro CLI -through GitHub-hosted emulator/simulator jobs. This is not an open research loop. -Reopen that decision only when a written implementation proposal names an -installed-app E2E requirement that the pinned open-source Maestro CLI cannot -satisfy. Do not keep re-checking Maestro, Detox, Appium, EAS, Firebase Test -Lab, BrowserStack, Sauce, AWS Device Farm, or other hosted-device services while -implementing this plan. Routine maintenance verifies the pinned installer, flow -files, app artifacts, runner behavior, and CI logs for the selected Maestro -lanes; it does not revisit provider selection. - -`tools/dev/setup-maestro.sh` installs only the exact versioned release asset and -SHA-256 recorded in `tools/dev/maestro.toml`; that manifest is the -single release pin. It does not execute the vendor's network installer. Version -upgrades change the reviewed manifest metadata and must keep the staged -archive/layout/version and atomic-promotion regression tests green; incomplete -or inconsistent metadata fails before any download. +Installed-app SDK smoke runs the real app on a hosted emulator or simulator. +The app performs the SQL, extension, and resource assertions itself. CI and +manual replay share `.github/actions/run-mobile-e2e`; both require a validated +structured receipt from continuous logs captured for that launch. Explicit +failure, app death, dead capture, and a missing receipt fail within the smoke +budget. Lifecycle and crash-recovery drills retain their separate assertions. The native Node addon uses the Rust Node-API adapter. It no longer downloads Node C headers or a separate Windows import library through a custom fallback. -Prior provider research is historical context, not a standing checklist. Maestro -pin upgrades are dependency maintenance; they do not reopen the runner decision -unless they expose a concrete installed-app E2E requirement this path cannot -meet. - The default installed-app path must remain free and public-checkout reproducible. Paid hosted-device providers, SaaS-only runners, and required private runner infrastructure are not part of the default proof path. When mobile E2E breaks, inspect the selected implementation first: app artifact shape, -simulator/emulator setup, Maestro flow files, logs, and CI runner assumptions. +simulator/emulator setup, exact-launch logs, receipts, and CI runner assumptions. Debug the chosen implementation first. Do not restart provider research unless the failure proves a concrete requirement this model cannot satisfy. diff --git a/src/examples/moon.yml b/src/examples/moon.yml index 29cebf100..6ab8d17e0 100644 --- a/src/examples/moon.yml +++ b/src/examples/moon.yml @@ -99,12 +99,9 @@ tasks: - "integration-examples:react-native-android-build" inputs: - "native/react-native-expo/src/**/*" - - "native/react-native-expo/maestro/**/*" - "native/react-native-expo/package.json" - project: "oliphaunt-react-native" group: "code" - - "/tools/dev/maestro.toml" - - "/tools/dev/setup-maestro.sh" - "/target/mobile-build/react-native/android/**/*" options: cache: false @@ -163,12 +160,9 @@ tasks: - "integration-examples:react-native-ios-build" inputs: - "native/react-native-expo/src/**/*" - - "native/react-native-expo/maestro/**/*" - "native/react-native-expo/package.json" - project: "oliphaunt-react-native" group: "code" - - "/tools/dev/maestro.toml" - - "/tools/dev/setup-maestro.sh" - "/target/mobile-build/react-native/ios/**/*" options: cache: false diff --git a/src/examples/native/react-native-expo/maestro/installed-smoke.yaml b/src/examples/native/react-native-expo/maestro/installed-smoke.yaml deleted file mode 100644 index 771a20c6c..000000000 --- a/src/examples/native/react-native-expo/maestro/installed-smoke.yaml +++ /dev/null @@ -1,15 +0,0 @@ -appId: ${APP_ID} -name: Oliphaunt installed app smoke -tags: - - oliphaunt - - smoke - - installed-app ---- -- extendedWaitUntil: - visible: - id: liboliphaunt-smoke-status-passed - timeout: ${SMOKE_TIMEOUT_MS} -- assertVisible: - id: liboliphaunt-smoke-status-passed -- assertVisible: - id: liboliphaunt-smoke-result diff --git a/src/extensions/artifacts/native/moon.yml b/src/extensions/artifacts/native/moon.yml index c5e725ff0..1d73bebbf 100644 --- a/src/extensions/artifacts/native/moon.yml +++ b/src/extensions/artifacts/native/moon.yml @@ -87,20 +87,18 @@ tasks: group: sources - /src/native/runtime/sources/*.toml - /src/extensions/artifacts/native/tools/**/* + - "!/src/extensions/artifacts/native/tools/**/*.test.*" - /tools/packaging/check-linux-consumer-baseline.sh - /src/third-party/tools/source-fetch-core.mts - /src/extensions/tools/extension-upstream-licenses.mts - - /tools/packaging/linux-abi-baseline.test.sh - "/tools/packaging/*.{mjs,mts}" - /src/extensions/tools/extension-artifact-archive-policy.mts - /src/extensions/artifacts/native/tools/native-extension-asset-index-contract.mts - /src/native/runtime/tools/native-runtime-payload-policy.json - /tools/packaging/windows-vc-runtime-policy.json - /tools/release/platform-compatibility-policy.mts - - /tools/release/platform-compatibility-policy.test.mts - /tools/packaging/platform-binary-contract.mts - /tools/packaging/strip-native-binaries.sh - - /tools/packaging/platform-binary-contract.test.mts - /tools/packaging/release-notices.mts - /tools/packaging/release-directory-safety.mts - /tools/packaging/windows-vc-runtime-closure.mts diff --git a/src/extensions/artifacts/native/tools/package-release-assets.sh b/src/extensions/artifacts/native/tools/package-release-assets.sh index 9ef9cf2bf..4727a318b 100755 --- a/src/extensions/artifacts/native/tools/package-release-assets.sh +++ b/src/extensions/artifacts/native/tools/package-release-assets.sh @@ -702,9 +702,11 @@ package_ios_target() { package_android_target() { local source_runtime runtime mobile_extensions android_root android_static_target - build_mobile_host_extension_runtime mobile_extensions="$(mobile_module_extensions_csv)" - build_mobile_static_artifacts "$mobile_extensions" + if [ "${OLIPHAUNT_EXTENSION_PHASE:-all}" != android-package ]; then + build_mobile_host_extension_runtime + build_mobile_static_artifacts "$mobile_extensions" + fi source_runtime="$(host_extension_runtime_root)" require_dir "$source_runtime" "mobile host extension runtime" runtime="$(prepare_extension_release_runtime "$source_runtime")" @@ -720,7 +722,9 @@ package_android_target() { ;; *) fail "Android target packager called for $target_id" ;; esac - tools/dev/bun.sh tools/packaging/platform-binary-contract.mts --target "$target_id" --root "$android_root/out" + if [ -n "$mobile_extensions" ]; then + tools/dev/bun.sh tools/packaging/platform-binary-contract.mts --target "$target_id" --root "$android_root/out" + fi local sql_name pg_major creates_extension stem dependencies shared_preload desktop_prebuilt mobile_prebuilt mobile_static_required mobile_static_targets data_files artifact_policy runtime_artifact android_archive static_prefix while IFS=$'\t' read -r sql_name pg_major creates_extension stem dependencies shared_preload desktop_prebuilt mobile_prebuilt mobile_static_required mobile_static_targets data_files artifact_policy; do [ -n "$sql_name" ] || continue @@ -753,10 +757,22 @@ package_android_target() { done < <(catalog_rows) } -fetch_extension_source_assets +phase="${OLIPHAUNT_EXTENSION_PHASE:-all}" +case "$phase:$target_id" in + all:*) fetch_extension_source_assets ;; + android-static:android-*) fetch_extension_source_assets ;; + android-package:android-*) ;; + *) fail "invalid extension phase $phase for $target_id" ;; +esac echo "==> Reading exact extension catalog" bun "$packager" list-catalog >"$catalog_file" write_indexes +if [ "$phase" = android-static ]; then + # SQL-only selections still transfer an empty, valid native output directory. + mkdir -p "$mobile_extension_work_root/$target_id/${target_id%-v8a}/out" + build_mobile_static_artifacts "$(mobile_module_extensions_csv)" + exit 0 +fi case "$target_id" in macos-arm64|linux-x64-gnu|linux-arm64-gnu|windows-x64-msvc) diff --git a/src/extensions/artifacts/native/tools/package-release-assets.test.sh b/src/extensions/artifacts/native/tools/package-release-assets.test.sh new file mode 100644 index 000000000..b68aa39df --- /dev/null +++ b/src/extensions/artifacts/native/tools/package-release-assets.test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +export OLIPHAUNT_EXTENSION_TARGET=android-arm64-v8a +export OLIPHAUNT_EXTENSION_PRODUCTS=oliphaunt-extension-pgtap +export OLIPHAUNT_EXTENSION_HOST_RUNTIME_ROOT="$scratch/host" +export OLIPHAUNT_MOBILE_EXTENSION_WORK_ROOT="$scratch/mobile" +export OLIPHAUNT_EXTENSION_RELEASE_ASSET_DIR="$scratch/assets" +export OLIPHAUNT_EXTENSION_RELEASE_STAGE_ROOT="$scratch/stage" +producer=src/extensions/artifacts/native/tools/package-release-assets.sh +sql="$scratch/host/share/postgresql/extension" +mkdir -p "$sql" +bun - "$scratch/host/postgres" <<'JS' +import {writeFileSync} from 'node:fs'; +import {elfFixture} from './tools/packaging/testdata/release-fixture-utils.mts'; +writeFileSync(process.argv[2], elfFixture({machine: 62, requiredVersions: ['GLIBC_2.17']})); +JS +printf "default_version = '1.3.5'\n" > "$sql/pgtap.control" +printf 'select 1;\n' > "$sql/pgtap--1.3.5.sql" +# SQL-only selection has a valid empty transfer and needs neither compiler. +OLIPHAUNT_EXTENSION_PHASE=android-static bash "$producer" +[ -d "$scratch/mobile/android-arm64-v8a/android-arm64/out" ] +OLIPHAUNT_EXTENSION_PHASE=android-package bash "$producer" +[ "$(find "$scratch/assets" -name '*-pgtap-*-runtime.tar.gz' | wc -l)" = 1 ] +# A package-only invocation must fail on missing inputs, never rebuild them. +if OLIPHAUNT_EXTENSION_PHASE=android-package OLIPHAUNT_EXTENSION_PRODUCTS=oliphaunt-extension-vector \ + bash "$producer" > "$scratch/missing-static.log" 2>&1; then + echo 'accepted missing static archive' >&2; exit 1 +fi +grep -Eq 'no ELF binaries|missing Android static archive for vector' "$scratch/missing-static.log" +rm -rf "$scratch/host" +if OLIPHAUNT_EXTENSION_PHASE=android-package bash "$producer" > "$scratch/missing-host.log" 2>&1; then + echo 'accepted missing Linux support' >&2; exit 1 +fi +grep -q 'missing mobile host extension runtime' "$scratch/missing-host.log" +if OLIPHAUNT_EXTENSION_PHASE=android-package OLIPHAUNT_EXTENSION_TARGET=ios-xcframework \ + bash "$producer" > "$scratch/wrong-phase.log" 2>&1; then + echo 'accepted Android phase for iOS' >&2; exit 1 +fi +grep -q 'invalid extension phase' "$scratch/wrong-phase.log" +echo 'Android split extension producer checks passed' diff --git a/src/extensions/artifacts/wasix/moon.yml b/src/extensions/artifacts/wasix/moon.yml index c328f25c5..8b2f966bb 100644 --- a/src/extensions/artifacts/wasix/moon.yml +++ b/src/extensions/artifacts/wasix/moon.yml @@ -25,7 +25,7 @@ tasks: tags: ["artifact", "ci-liboliphaunt-wasix-runtime"] command: "bash src/extensions/artifacts/wasix/tools/build-portable.sh" deps: ["liboliphaunt-wasix:runtime-portable"] - inputs: ["tools/build-portable.sh", "/src/wasix/runtime/assets/build/**/*", "/src/wasix/runtime/tools/extension-build-scripts.mts", "/src/wasix/runtime/tools/xtask/**/*", {project: "extensions", group: "build"}] + inputs: ["tools/build-portable.sh", "/src/wasix/runtime/assets/build/**/*", "!/src/wasix/runtime/assets/build/**/*.test.*", "/src/wasix/runtime/tools/extension-build-scripts.mts", "/src/wasix/runtime/tools/xtask/**/*", {project: "extensions", group: "build"}] outputs: ["/target/extensions/wasix/assets/**/*"] options: {runFromWorkspaceRoot: true, cache: local} build-aot: diff --git a/src/extensions/evidence/matrix.toml b/src/extensions/evidence/matrix.toml index 62f5d9802..3705288bc 100644 --- a/src/extensions/evidence/matrix.toml +++ b/src/extensions/evidence/matrix.toml @@ -100,7 +100,7 @@ extension = "amcheck" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -108,7 +108,7 @@ extension = "auto_explain" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -116,7 +116,7 @@ extension = "bloom" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -124,7 +124,7 @@ extension = "btree_gin" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -132,7 +132,7 @@ extension = "btree_gist" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -140,7 +140,7 @@ extension = "citext" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -148,7 +148,7 @@ extension = "cube" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -156,7 +156,7 @@ extension = "dict_int" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -164,7 +164,7 @@ extension = "dict_xsyn" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -172,7 +172,7 @@ extension = "earthdistance" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -180,7 +180,7 @@ extension = "file_fdw" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -188,7 +188,7 @@ extension = "fuzzystrmatch" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -196,7 +196,7 @@ extension = "hstore" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -204,7 +204,7 @@ extension = "intarray" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -212,7 +212,7 @@ extension = "isn" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -220,7 +220,7 @@ extension = "lo" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -228,7 +228,7 @@ extension = "ltree" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -236,7 +236,7 @@ extension = "pageinspect" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -244,7 +244,7 @@ extension = "pg_buffercache" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -252,7 +252,7 @@ extension = "pg_freespacemap" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -260,7 +260,7 @@ extension = "pg_hashids" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -268,7 +268,7 @@ extension = "pg_ivm" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -276,7 +276,7 @@ extension = "pg_surgery" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -284,7 +284,7 @@ extension = "pg_textsearch" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -292,7 +292,7 @@ extension = "pg_trgm" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -300,7 +300,7 @@ extension = "pg_uuidv7" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -308,7 +308,7 @@ extension = "pg_visibility" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -316,7 +316,7 @@ extension = "pg_walinspect" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -324,7 +324,7 @@ extension = "pgcrypto" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -332,7 +332,7 @@ extension = "pgtap" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -340,7 +340,7 @@ extension = "postgis" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -348,7 +348,7 @@ extension = "seg" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -356,7 +356,7 @@ extension = "tablefunc" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -364,7 +364,7 @@ extension = "tcn" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -372,7 +372,7 @@ extension = "tsm_system_rows" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -380,7 +380,7 @@ extension = "tsm_system_time" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -388,7 +388,7 @@ extension = "unaccent" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -396,7 +396,7 @@ extension = "uuid_ossp" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] [[claims]] @@ -404,5 +404,5 @@ extension = "vector" postgres-major = 18 artifact-family = "wasix-runtime" platform-targets = ["portable"] -runtime-modes = ["direct", "server", "restart", "backup-restore"] +runtime-modes = ["direct","server","restart","backup-restore","materialization"] evidence-required = ["wasix-full-lifecycle-v1"] diff --git a/src/extensions/generated/docs/extension-evidence.json b/src/extensions/generated/docs/extension-evidence.json index 8720c014f..00b689fe6 100644 --- a/src/extensions/generated/docs/extension-evidence.json +++ b/src/extensions/generated/docs/extension-evidence.json @@ -16,7 +16,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -28,7 +29,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "amcheck" }, @@ -48,7 +50,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -60,7 +63,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "auto_explain" }, @@ -80,7 +84,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -92,7 +97,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "bloom" }, @@ -112,7 +118,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -124,7 +131,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "btree_gin" }, @@ -144,7 +152,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -156,7 +165,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "btree_gist" }, @@ -176,7 +186,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -188,7 +199,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "citext" }, @@ -208,7 +220,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -220,7 +233,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "cube" }, @@ -240,7 +254,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -252,7 +267,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "dict_int" }, @@ -272,7 +288,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -284,7 +301,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "dict_xsyn" }, @@ -304,7 +322,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -316,7 +335,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "earthdistance" }, @@ -336,7 +356,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -348,7 +369,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "file_fdw" }, @@ -368,7 +390,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -380,7 +403,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "fuzzystrmatch" }, @@ -400,7 +424,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -412,7 +437,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "hstore" }, @@ -432,7 +458,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -444,7 +471,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "intarray" }, @@ -464,7 +492,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -476,7 +505,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "isn" }, @@ -496,7 +526,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -508,7 +539,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "lo" }, @@ -528,7 +560,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -540,7 +573,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "ltree" }, @@ -560,7 +594,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -572,7 +607,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pageinspect" }, @@ -592,7 +628,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -604,7 +641,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_buffercache" }, @@ -624,7 +662,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -636,7 +675,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_freespacemap" }, @@ -656,7 +696,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -668,7 +709,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_hashids" }, @@ -688,7 +730,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -700,7 +743,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_ivm" }, @@ -720,7 +764,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -732,7 +777,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_surgery" }, @@ -752,7 +798,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -764,7 +811,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_textsearch" }, @@ -784,7 +832,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -796,7 +845,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_trgm" }, @@ -816,7 +866,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -828,7 +879,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_uuidv7" }, @@ -848,7 +900,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -860,7 +913,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_visibility" }, @@ -880,7 +934,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -892,7 +947,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pg_walinspect" }, @@ -912,7 +968,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -924,7 +981,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pgcrypto" }, @@ -944,7 +1002,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -956,7 +1015,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "pgtap" }, @@ -976,7 +1036,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -988,7 +1049,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "postgis" }, @@ -1008,7 +1070,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1020,7 +1083,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "seg" }, @@ -1040,7 +1104,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1052,7 +1117,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "tablefunc" }, @@ -1072,7 +1138,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1084,7 +1151,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "tcn" }, @@ -1104,7 +1172,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1116,7 +1185,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "tsm_system_rows" }, @@ -1136,7 +1206,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1148,7 +1219,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "tsm_system_time" }, @@ -1168,7 +1240,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1180,7 +1253,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "unaccent" }, @@ -1200,7 +1274,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1212,7 +1287,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "uuid-ossp" }, @@ -1232,7 +1308,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ] } ], @@ -1244,7 +1321,8 @@ "direct", "server", "restart", - "backup-restore" + "backup-restore", + "materialization" ], "sql-name": "vector" } diff --git a/src/extensions/tests/native/moon.yml b/src/extensions/tests/native/moon.yml index 2fc7cd332..108e9534c 100644 --- a/src/extensions/tests/native/moon.yml +++ b/src/extensions/tests/native/moon.yml @@ -26,6 +26,7 @@ tasks: - project: "shared-test-fixtures" group: "fixtures" - "tools/**/*.mts" + - "/.github/scripts/{release-candidate-lib,write-release-candidate,verify-release-candidate}.mts" - "/tools/release/**/*.mts" - "/src/extensions/artifacts/native/tools/**/*.mts" - "/src/extensions/artifacts/packages/tools/**/*.mts" diff --git a/src/extensions/tests/native/tools/native-extension-lifecycle-receipts.test.mts b/src/extensions/tests/native/tools/native-extension-lifecycle-receipts.test.mts index 9a6bdb596..6949b999a 100644 --- a/src/extensions/tests/native/tools/native-extension-lifecycle-receipts.test.mts +++ b/src/extensions/tests/native/tools/native-extension-lifecycle-receipts.test.mts @@ -2,11 +2,25 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; - +import { + affectedPlanBinding, + assertBindingMatches, + assertCandidateBindingShape, + nativeEvidenceBinding, +} from '../../../../../.github/scripts/release-candidate-lib.mts'; import { compareText, exactExtensionProducts, @@ -368,3 +382,138 @@ test('aggregate verification rejects candidate, shard, and PASS-record drift eve } } }); + +test('release qualification binds and revalidates same-run native lifecycle shards', () => { + const value = fixture(['cube', 'earthdistance']); + try { + const root = path.join(value.root, 'evidence'); + mkdirSync(path.join(root, 'input'), { recursive: true }); + mkdirSync(path.join(root, 'output')); + const receipt = writeShard(value, 0, { shardCount: 1 }); + const transferred = path.join(root, 'input', path.basename(receipt)); + copyFileSync(receipt, transferred); + const output = path.join(root, 'output/aggregate-receipt.json'); + verifyReceipts({ + receipts: path.join(root, 'input'), + 'candidate-sha': CANDIDATE_SHA, + 'candidate-tree': CANDIDATE_TREE, + 'expected-extensions-csv': 'cube,earthdistance', + 'expected-shard-count': '1', + repository: 'f0rr0/oliphaunt', + 'run-id': '123', + 'run-attempt': '1', + output, + }); + const planFile = path.join(value.root, 'plan.json'); + writeFileSync( + planFile, + JSON.stringify({ + jobs: ['affected', 'native-extension-lifecycle'], + projects: [], + native_extension_lifecycle_sql_names: ['cube', 'earthdistance'], + native_extension_lifecycle_shard_count: 1, + }), + ); + const affectedPlan = affectedPlanBinding(planFile, false); + const provenance = { + repository: 'f0rr0/oliphaunt', + runId: '123', + runAttempt: 2, + sha: CANDIDATE_SHA, + tree: CANDIDATE_TREE, + selection: affectedPlan.nativeExtensionLifecycle, + }; + const candidateFile = path.join(value.root, 'candidate.json'); + const env = { + ...process.env, + CI_HEAD_SHA: CANDIDATE_SHA, + CI_CHECKED_OUT_SHA: CANDIDATE_SHA, + CI_SOURCE_TREE: CANDIDATE_TREE, + CI_PLAN_PATH: planFile, + CI_QUALIFICATION_MODE: 'full-payload', + WASIX_RELEASE_REGRESSION_REQUIRED: 'false', + NATIVE_EVIDENCE_ROOT: root, + GITHUB_REPOSITORY: provenance.repository, + GITHUB_WORKFLOW: 'CI', + GITHUB_RUN_ID: '123', + GITHUB_RUN_ATTEMPT: '2', + GITHUB_EVENT_NAME: 'push', + GITHUB_REF: 'refs/heads/main', + GITHUB_WORKFLOW_REF: 'f0rr0/oliphaunt/.github/workflows/ci.yml@refs/heads/main', + CI_RUN_ID: '123', + RELEASE_HEAD_SHA: CANDIDATE_SHA, + PRODUCER_RECEIPTS_JSON: '[]', + }; + const run = (command, ...args) => { + const result = Bun.spawnSync( + ['bun', `.github/scripts/${command}-release-candidate.mts`, candidateFile, ...args], + { env }, + ); + return { code: result.exitCode, output: result.stdout.toString() + result.stderr.toString() }; + }; + const written = run('write'); + assert.equal(written.code, 0, written.output); + const verifyArgs = [ + '--plan', + planFile, + '--wasix-evidence-required', + 'false', + '--native-evidence-required', + 'true', + '--native-evidence-root', + root, + ]; + const verified = run('verify', ...verifyArgs); + assert.equal(verified.code, 0, verified.output); + const missingProduct = run( + 'verify', + ...verifyArgs, + '--products-json', + '["oliphaunt-extension-vector"]', + ); + assert.notEqual(missingProduct.code, 0); + assert.match(missingProduct.output, /missing published extension vector/); + const binding = nativeEvidenceBinding(root, provenance); + const candidate = { + schemaVersion: 2, + ...provenance, + affectedPlan, + evidenceRequirements: { + wasixReleaseRegression: false, + nativeExtensionLifecycle: true, + artifacts: ['native-extension-lifecycle-evidence'], + }, + evidence: { wasixReleaseRegression: null, nativeExtensionLifecycle: binding }, + }; + assert.doesNotThrow(() => assertCandidateBindingShape(candidate)); + assert.throws( + () => + assertCandidateBindingShape({ ...candidate, evidence: { wasixReleaseRegression: null } }), + /native evidence digest/, + ); + for (const patch of [ + { runId: '124' }, + { repository: 'other/repo' }, + { runAttempt: 0 }, + { sha: 'c'.repeat(40) }, + { tree: 'c'.repeat(40) }, + { selection: { extensions: ['cube'], shardCount: 1 } }, + { selection: { extensions: ['cube', 'earthdistance'], shardCount: 2 } }, + ]) + assert.throws(() => nativeEvidenceBinding(root, { ...provenance, ...patch })); + writeFileSync(output, `${readFileSync(output, 'utf8')}\n`); + assert.throws( + () => + assertBindingMatches(binding, nativeEvidenceBinding(root, provenance), 'native evidence'), + /does not match/, + ); + const shard = JSON.parse(readFileSync(transferred, 'utf8')); + shard.passRecords.pop(); + writeFileSync(transferred, JSON.stringify(shard)); + assert.throws(() => nativeEvidenceBinding(root, provenance), /digest mismatch/); + rmSync(transferred); + assert.throws(() => nativeEvidenceBinding(root, provenance), /exactly 1 shard/); + } finally { + rmSync(value.root, { force: true, recursive: true }); + } +}); diff --git a/src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts b/src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts index 0724efe44..7800e7766 100644 --- a/src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts +++ b/src/extensions/tests/native/tools/verify-native-extension-lifecycle-receipts.mts @@ -211,11 +211,36 @@ export function verifyReceipts(options) { `aggregate extension coverage drift: expected=${expected.join(',')}; actual=${actual.join(',')}`, ); } + let github; + if ( + options.repository !== undefined || + options['run-id'] !== undefined || + options['run-attempt'] !== undefined + ) { + const runId = Number(options['run-id']); + const runAttempt = Number(options['run-attempt']); + if ( + !/^[^/]+\/[^/]+$/.test(options.repository ?? '') || + !Number.isSafeInteger(runId) || + runId < 1 || + !Number.isSafeInteger(runAttempt) || + runAttempt < 1 + ) + fail('GitHub provenance requires repository, positive run ID and attempt'); + github = { + repository: options.repository, + workflow: 'CI', + runId, + runAttempt, + job: 'native-extension-lifecycle-aggregate', + }; + } const aggregateCore = { schema: 'oliphaunt-native-extension-lifecycle-aggregate-v1', candidateSha: options['candidate-sha'], candidateTree: options['candidate-tree'], target: 'linux-x64-gnu', + ...(github ? { github } : {}), shardCount: expectedShardCount, extensionCount: expected.length, extensions: expected, @@ -231,8 +256,11 @@ export function verifyReceipts(options) { .sort((left, right) => left.shardIndex - right.shardIndex), }; const aggregate = { ...aggregateCore, aggregateSha256: sha256(JSON.stringify(aggregateCore)) }; - writeFileSync(options.output, `${JSON.stringify(aggregate, null, 2)}\n`); - console.log(`native extension lifecycle aggregate verified: ${options.output}`); + if (options.output) { + writeFileSync(options.output, `${JSON.stringify(aggregate, null, 2)}\n`); + console.log(`native extension lifecycle aggregate verified: ${options.output}`); + } + return aggregate; } if (import.meta.main) { diff --git a/src/extensions/tools/collect-wasix-evidence.sh b/src/extensions/tools/collect-wasix-evidence.sh index 93dab05a1..7acb1d473 100755 --- a/src/extensions/tools/collect-wasix-evidence.sh +++ b/src/extensions/tools/collect-wasix-evidence.sh @@ -33,10 +33,10 @@ trap 'rm -rf "$OLIPHAUNT_EXTENSION_EVIDENCE_DIR"' EXIT # This command exercises every catalogued extension in direct, server, restart, # materialization, and physical backup/restore modes. The record command is deliberately # after it so a failing or interrupted run cannot produce passed evidence. -bash src/wasix/runtime/tools/runtime-smoke.sh regression +.github/scripts/run-planned-moon-job.sh wasix-release-regression bash src/extensions/tools/check-extension-model.sh \ --record-wasix-evidence-run "$run_id" \ - --observed-at "$observed_at" -bash src/extensions/tools/check-extension-model.sh --check --require-current-evidence + --observed-at "$observed_at" \ + --require-current-evidence echo "recorded immutable WASIX extension evidence run: $run_id" diff --git a/src/extensions/tools/extension-evidence.mts b/src/extensions/tools/extension-evidence.mts index 53fd2cfda..1cbbed4b4 100644 --- a/src/extensions/tools/extension-evidence.mts +++ b/src/extensions/tools/extension-evidence.mts @@ -83,7 +83,7 @@ export function evidenceMatrix(catalog: Row): string { 'postgres-major = 18', 'artifact-family = "wasix-runtime"', 'platform-targets = ["portable"]', - 'runtime-modes = ["direct", "server", "restart", "backup-restore"]', + `runtime-modes = ${JSON.stringify(modes)}`, `evidence-required = ["${tier}"]`, '', ]; diff --git a/src/extensions/tools/extension-model.test.mts b/src/extensions/tools/extension-model.test.mts index f9946df5a..64b618a52 100644 --- a/src/extensions/tools/extension-model.test.mts +++ b/src/extensions/tools/extension-model.test.mts @@ -47,6 +47,7 @@ function run() { direct: 'passed', server: 'passed', restart: 'passed', + materialization: 'passed', 'backup-restore': 'passed', }, }, @@ -78,6 +79,9 @@ test('only observed passing results for the current source commit qualify', () = (run: any) => { run.results[0].runtimeModeStatuses['backup-restore'] = 'failed'; }, + (run: any) => { + delete run.results[0].runtimeModeStatuses.materialization; + }, (run: any) => { run.status = 'failed'; }, diff --git a/src/native/mobile-bindings/moon.yml b/src/native/mobile-bindings/moon.yml index 63dc4a07a..f150a763c 100644 --- a/src/native/mobile-bindings/moon.yml +++ b/src/native/mobile-bindings/moon.yml @@ -22,14 +22,12 @@ tasks: options: runFromWorkspaceRoot: true test-native: - tags: ["integration", "runtime", "requires-rust"] + tags: ["ci-native-consumers", "platform-linux-x64-gnu", "integration", "runtime", "requires-rust"] script: | set -e - . src/native/runtime/tools/runtime-preflight.sh - oliphaunt_runtime_native_host_require basic - cargo test -p oliphaunt --no-default-features --features mobile-bindings --test mobile_broker --locked -- --nocapture - deps: ["liboliphaunt-native:build-runtime-desktop-target", "~:cargo-sources"] - inputs: ["@group(cargo-workspace)", "src/**/*", "/src/native/sdks/rust/src/**/*", "/src/native/sdks/rust/tests/mobile_broker.rs", "/src/native/broker/src/**/*"] + bash src/native/sdks/tests/with-runtime.sh cargo nextest run -p oliphaunt --no-default-features --features mobile-bindings --test mobile_broker --locked --no-tests=fail --test-threads=1 + deps: ["liboliphaunt-native:package-runtime-desktop-target", "~:cargo-sources"] + inputs: ["/src/native/sdks/tests/with-runtime.sh", "/src/native/runtime/tools/runtime-preflight.sh", "@group(cargo-workspace)", "src/**/*", "/src/native/sdks/rust/src/**/*", "/src/native/sdks/rust/tests/mobile_broker.rs", "/src/native/broker/src/**/*"] options: runFromWorkspaceRoot: true cache: false diff --git a/src/native/runtime/bin/build-postgres18-android-arm64.sh b/src/native/runtime/bin/build-postgres18-android-arm64.sh index ba1a7ca4c..24ecc73e5 100755 --- a/src/native/runtime/bin/build-postgres18-android-arm64.sh +++ b/src/native/runtime/bin/build-postgres18-android-arm64.sh @@ -8,6 +8,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/mobile-postgis-extensions.sh" script_path="$script_dir/$(basename "$0")" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" pg_version="18.4" pg_sha256="81a81ec695fb0c7901407defaa1d2f7973617154cf27ba74e3a7ab8e64436094" diff --git a/src/native/runtime/bin/build-postgres18-ios-device.sh b/src/native/runtime/bin/build-postgres18-ios-device.sh index 071f3056e..b8b276c5c 100755 --- a/src/native/runtime/bin/build-postgres18-ios-device.sh +++ b/src/native/runtime/bin/build-postgres18-ios-device.sh @@ -8,6 +8,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/mobile-postgis-extensions.sh" script_path="$script_dir/$(basename "$0")" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" oliphaunt_mobile_target="ios-device" pg_version="18.4" diff --git a/src/native/runtime/bin/build-postgres18-ios-simulator.sh b/src/native/runtime/bin/build-postgres18-ios-simulator.sh index 77f89a788..d844ff891 100755 --- a/src/native/runtime/bin/build-postgres18-ios-simulator.sh +++ b/src/native/runtime/bin/build-postgres18-ios-simulator.sh @@ -8,6 +8,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/mobile-postgis-extensions.sh" script_path="$script_dir/$(basename "$0")" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" oliphaunt_mobile_target="ios-simulator" pg_version="18.4" diff --git a/src/native/runtime/bin/build-postgres18-linux.sh b/src/native/runtime/bin/build-postgres18-linux.sh index 02f6f053d..c25380229 100755 --- a/src/native/runtime/bin/build-postgres18-linux.sh +++ b/src/native/runtime/bin/build-postgres18-linux.sh @@ -7,6 +7,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/postgis-dependency-cache.sh" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" pg_version="18.4" pg_sha256="81a81ec695fb0c7901407defaa1d2f7973617154cf27ba74e3a7ab8e64436094" diff --git a/src/native/runtime/bin/build-postgres18-macos.sh b/src/native/runtime/bin/build-postgres18-macos.sh index f6e255ed9..c8183d6bd 100755 --- a/src/native/runtime/bin/build-postgres18-macos.sh +++ b/src/native/runtime/bin/build-postgres18-macos.sh @@ -7,6 +7,7 @@ script_path="$script_dir/$(basename "${BASH_SOURCE[0]}")" . "$script_dir/../../../third-party/icu/tools/build.sh" . "$script_dir/postgis-dependency-cache.sh" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" macos_deployment_target="${MACOSX_DEPLOYMENT_TARGET:-11.0}" case "$macos_deployment_target" in diff --git a/src/native/runtime/bin/build-postgres18-windows.sh b/src/native/runtime/bin/build-postgres18-windows.sh index 8e619ccdf..3399fe7ef 100755 --- a/src/native/runtime/bin/build-postgres18-windows.sh +++ b/src/native/runtime/bin/build-postgres18-windows.sh @@ -5,6 +5,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$script_dir/common.sh" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" cd "$repo_root" +source tools/dev/acquisition.sh source src/third-party/postgres/fetch-source.sh source src/native/runtime/tools/liboliphaunt-extension-guard.sh diff --git a/src/native/runtime/bin/check-postgres18-ios-simulator.sh b/src/native/runtime/bin/check-postgres18-ios-simulator.sh index 78beae366..a882d7e1d 100755 --- a/src/native/runtime/bin/check-postgres18-ios-simulator.sh +++ b/src/native/runtime/bin/check-postgres18-ios-simulator.sh @@ -5,6 +5,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$script_dir/common.sh" . "$script_dir/../../../third-party/icu/tools/build.sh" repo_root="$(oliphaunt_resolve_repo_root "$script_dir")" +. "$repo_root/tools/dev/acquisition.sh" . "$repo_root/src/third-party/postgres/fetch-source.sh" pg_version="18.4" pg_sha256="81a81ec695fb0c7901407defaa1d2f7973617154cf27ba74e3a7ab8e64436094" diff --git a/src/native/runtime/moon.yml b/src/native/runtime/moon.yml index 383f604a4..44fc8b40d 100644 --- a/src/native/runtime/moon.yml +++ b/src/native/runtime/moon.yml @@ -54,6 +54,7 @@ fileGroups: - src/**/* - patches/**/* - postgres/**/* + - /tools/dev/acquisition.sh - /src/third-party/postgres/**/* - "!/src/third-party/postgres/**/*.test.*" - "!/src/third-party/postgres/testdata/**/*" diff --git a/src/native/runtime/tools/runtime-preflight.sh b/src/native/runtime/tools/runtime-preflight.sh index 866e50392..e5ec8a5a3 100644 --- a/src/native/runtime/tools/runtime-preflight.sh +++ b/src/native/runtime/tools/runtime-preflight.sh @@ -67,23 +67,14 @@ oliphaunt_runtime_native_host_postgres() { printf '%s\n' "${OLIPHAUNT_POSTGRES:-$(oliphaunt_runtime_native_host_install_dir)/bin/postgres$suffix}" } -oliphaunt_runtime_native_host_pg_config() { - case "$(uname -s)" in - MINGW* | MSYS* | CYGWIN*) suffix=.exe ;; - *) suffix= ;; - esac - printf '%s\n' "${OLIPHAUNT_PG_CONFIG:-$(oliphaunt_runtime_native_host_install_dir)/bin/pg_config$suffix}" -} - oliphaunt_runtime_native_host_export_defaults() { LIBOLIPHAUNT_PATH="$(oliphaunt_runtime_native_host_lib)" OLIPHAUNT_INSTALL_DIR="$(oliphaunt_runtime_native_host_install_dir)" OLIPHAUNT_INITDB="$(oliphaunt_runtime_native_host_initdb)" OLIPHAUNT_POSTGRES="$(oliphaunt_runtime_native_host_postgres)" - OLIPHAUNT_PG_CONFIG="$(oliphaunt_runtime_native_host_pg_config)" OLIPHAUNT_POSTGRES_TOOL_DIR="${OLIPHAUNT_POSTGRES_TOOL_DIR:-$OLIPHAUNT_INSTALL_DIR/bin}" export LIBOLIPHAUNT_PATH OLIPHAUNT_INSTALL_DIR OLIPHAUNT_INITDB - export OLIPHAUNT_POSTGRES OLIPHAUNT_PG_CONFIG OLIPHAUNT_POSTGRES_TOOL_DIR + export OLIPHAUNT_POSTGRES OLIPHAUNT_POSTGRES_TOOL_DIR } oliphaunt_runtime_native_host_require() { @@ -95,8 +86,7 @@ oliphaunt_runtime_native_host_require() { [ -f "$LIBOLIPHAUNT_PATH" ] && [ -d "$OLIPHAUNT_INSTALL_DIR" ] && [ -x "$OLIPHAUNT_INITDB" ] && - [ -x "$OLIPHAUNT_POSTGRES" ] && - [ -x "$OLIPHAUNT_PG_CONFIG" ] && return 0 + [ -x "$OLIPHAUNT_POSTGRES" ] && return 0 cat >&2 < 0 and all(int(suite.attrib.get(key, 0)) == 0 for key in ("skipped", "failures", "errors")), "native binding tests must execute and pass" +CHECK diff --git a/src/native/sdks/react-native/tools/check-package.test.mts b/src/native/sdks/react-native/tools/check-package.test.mts index 1127eb836..aba8e84c8 100644 --- a/src/native/sdks/react-native/tools/check-package.test.mts +++ b/src/native/sdks/react-native/tools/check-package.test.mts @@ -1,4 +1,8 @@ import test from 'node:test'; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { stageArtifacts } from './stage-release-artifacts.mts'; import { iosBaseLegalMetadata } from '../../swift/tools/ios-carrier-manifest.mts'; import assert from 'node:assert/strict'; import { validateReactNativePackagedCarrier } from './check-package.mts'; @@ -87,3 +91,49 @@ test('binds the React Native npm carrier bytes to selection-neutral staged evide /must match liboliphaunt-native 1\.2\.3/u, ); }); + +test('package staging accepts frozen current iOS metadata and rejects stale or corrupt metadata', () => { + const root = mkdtempSync(path.join(tmpdir(), 'rn-frozen-carrier-')); + const previous = process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER; + try { + const version = readFileSync( + new URL('../../../runtime/VERSION', import.meta.url), + 'utf8', + ).trim(); + const carrier = selectionNeutralCarrier(version); + const frozen = path.join(root, 'frozen.json'); + const work = path.join(root, 'work'); + const artifacts = path.join(root, 'artifacts'); + mkdirSync(path.join(work, 'package'), { recursive: true }); + writeFileSync(path.join(work, 'package/package.json'), '{}'); + writeFileSync(frozen, JSON.stringify(carrier)); + process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER = frozen; + stageArtifacts(artifacts, work); + const evidence = readFileSync( + path.join(artifacts, 'ios-carriers/oliphaunt-react-native-ios-carriers.json'), + 'utf8', + ); + assert.deepEqual(JSON.parse(evidence), carrier); + assert.equal( + readFileSync(path.join(work, 'package/oliphaunt-react-native-ios-carriers.json'), 'utf8'), + evidence, + ); + for (const mutate of [ + (value) => { + value.base.version = '999.0.0'; + }, + (value) => { + value.base.assets[0].sha256 = 'invalid'; + }, + ]) { + const invalid = structuredClone(carrier); + mutate(invalid); + writeFileSync(frozen, JSON.stringify(invalid)); + assert.throws(() => stageArtifacts(artifacts, work)); + } + } finally { + if (previous === undefined) delete process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER; + else process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER = previous; + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/src/native/sdks/react-native/tools/expo-android-runner.sh b/src/native/sdks/react-native/tools/expo-android-runner.sh index ff47165e7..c3b04a9a2 100755 --- a/src/native/sdks/react-native/tools/expo-android-runner.sh +++ b/src/native/sdks/react-native/tools/expo-android-runner.sh @@ -58,19 +58,10 @@ case "$build_type" in esac build_only="${OLIPHAUNT_EXPO_ANDROID_BUILD_ONLY:-0}" e2e_only="${OLIPHAUNT_EXPO_ANDROID_E2E_ONLY:-0}" -e2e_assertion_runner="${OLIPHAUNT_EXPO_ANDROID_E2E_ASSERTION_RUNNER:-${OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER:-log}}" -case "$e2e_assertion_runner" in - auto|log|maestro) - ;; - *) - echo "error: OLIPHAUNT_EXPO_ANDROID_E2E_ASSERTION_RUNNER must be auto, log, or maestro, got $e2e_assertion_runner" >&2 - exit 1 - ;; -esac + build_type_capitalized="$(printf '%s' "$build_type" | awk '{ print toupper(substr($0, 1, 1)) substr($0, 2) }')" apk="${OLIPHAUNT_EXPO_ANDROID_APK:-$example_dir/android/app/build/outputs/apk/$build_type/app-$build_type.apk}" build_artifact_dir="${OLIPHAUNT_EXPO_ANDROID_BUILD_ARTIFACT_DIR:-$root/target/mobile-build/react-native/android}" -maestro_flow="${OLIPHAUNT_EXPO_ANDROID_MAESTRO_FLOW:-$source_example_dir/maestro/installed-smoke.yaml}" app_id="${OLIPHAUNT_EXPO_ANDROID_APP_ID:-dev.oliphaunt.reactnative.example}" scheme="${OLIPHAUNT_EXPO_ANDROID_SCHEME:-reactnativeoliphauntexpo}" dev_client_scheme="${OLIPHAUNT_EXPO_ANDROID_DEV_CLIENT_SCHEME:-exp+react-native-oliphaunt-expo}" @@ -700,7 +691,12 @@ start_metro_if_needed() { fail "Expo Metro did not start on port $metro_port" } -trap cleanup EXIT +android_log_pid="" +cleanup_android_runner() { + stop_mobile_log_capture "$android_log_pid" + cleanup +} +trap cleanup_android_runner EXIT write_android_package_metrics() { local apk_bytes="$1" @@ -742,7 +738,7 @@ write_android_build_artifact_report() { } main() { - if ! { is_truthy "$e2e_only" && [ "$build_type" = "release" ] && [ "$e2e_assertion_runner" = "maestro" ]; }; then + if ! is_truthy "$e2e_only"; then need_cmd rg fi if [ "$build_type" = "debug" ]; then @@ -771,6 +767,17 @@ main() { prepare_expo_example_workspace pack_react_native_sdk_if_needed ensure_android_project + # Expo's useExpoVersionCatalog reads the installed React Native version catalog. + local expo_ndk + expo_ndk="$(bun - "$example_dir" <<'JS' +import {readFileSync} from 'node:fs'; +import {dirname, join} from 'node:path'; +const rn = require.resolve('react-native/package.json', {paths:[process.argv[2]]}); +console.log(Bun.TOML.parse(readFileSync(join(dirname(rn), 'gradle/libs.versions.toml'), 'utf8')).versions.ndkVersion ?? ''); +JS +)" + [ -n "$expo_ndk" ] || fail "React Native's version catalog must declare ndkVersion" + bash "$root/tools/dev/setup-android-sdk.sh" --sdk-root "$ANDROID_HOME" --expo-ndk-version "$expo_ndk" local runtime_resources jni_libs source_so static_registry_source source_so="$(find_android_liboliphaunt_so)" static_registry_source="$(mobile_static_registry_source_for_library "$source_so")" diff --git a/src/native/sdks/react-native/tools/expo-ios-runner.sh b/src/native/sdks/react-native/tools/expo-ios-runner.sh index 245e56123..3e381e12b 100755 --- a/src/native/sdks/react-native/tools/expo-ios-runner.sh +++ b/src/native/sdks/react-native/tools/expo-ios-runner.sh @@ -68,14 +68,7 @@ background_seconds="${OLIPHAUNT_EXPO_IOS_BACKGROUND_SECONDS:-3}" reuse_installed_app="${OLIPHAUNT_EXPO_IOS_REUSE_INSTALLED_APP:-0}" clean_simulator_install="${OLIPHAUNT_EXPO_IOS_CLEAN_INSTALL:-1}" e2e_only="${OLIPHAUNT_EXPO_IOS_E2E_ONLY:-0}" -e2e_assertion_runner="${OLIPHAUNT_EXPO_IOS_E2E_ASSERTION_RUNNER:-${OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER:-log}}" -case "$e2e_assertion_runner" in - auto | log | maestro) ;; - *) - echo "error: OLIPHAUNT_EXPO_IOS_E2E_ASSERTION_RUNNER must be auto, log, or maestro, got $e2e_assertion_runner" >&2 - exit 1 - ;; -esac + configuration="${OLIPHAUNT_EXPO_IOS_CONFIGURATION:-Debug}" sdk="${OLIPHAUNT_EXPO_IOS_SDK:-iphonesimulator}" destination="${OLIPHAUNT_EXPO_IOS_DESTINATION:-}" @@ -86,7 +79,6 @@ derived_data="$scratch_root/DerivedData" workspace="$example_dir/ios/reactnativeoliphauntexpo.xcworkspace" xcode_scheme="reactnativeoliphauntexpo" build_artifact_dir="${OLIPHAUNT_EXPO_IOS_BUILD_ARTIFACT_DIR:-$root/target/mobile-build/react-native/ios}" -maestro_flow="${OLIPHAUNT_EXPO_IOS_MAESTRO_FLOW:-$source_example_dir/maestro/installed-smoke.yaml}" expo_use_precompiled_modules="${OLIPHAUNT_EXPO_IOS_USE_PRECOMPILED_MODULES:-true}" use_ccache="${OLIPHAUNT_EXPO_IOS_USE_CCACHE:-1}" liboliphaunt_pod_mode="vendored-framework" diff --git a/src/native/sdks/react-native/tools/expo-runner-android-device.sh b/src/native/sdks/react-native/tools/expo-runner-android-device.sh index d9496df95..5edeebf23 100644 --- a/src/native/sdks/react-native/tools/expo-runner-android-device.sh +++ b/src/native/sdks/react-native/tools/expo-runner-android-device.sh @@ -2,90 +2,6 @@ # Shared Android device, logcat, lifecycle, and installed-app helpers for the # Expo Android runner. This file is sourced by expo-android-runner.sh. -should_use_maestro_e2e() { - [ "$runner" = "smoke" ] || return 1 - case "$e2e_assertion_runner" in - maestro) - maestro_binary >/dev/null || fail "missing required command: maestro; run tools/dev/setup-maestro.sh" - return 0 - ;; - auto) - maestro_binary >/dev/null - return - ;; - *) - return 1 - ;; - esac -} - -run_maestro_installed_smoke() { - local device_id="$1" - local adb="${2:-$ANDROID_HOME/platform-tools/adb}" - local reports_dir="$scratch_root/reports" - [ -f "$maestro_flow" ] || fail "missing Maestro installed-app smoke flow: $maestro_flow" - local maestro - maestro="$(maestro_binary)" || fail "missing required command: maestro; run tools/dev/setup-maestro.sh" - mkdir -p "$reports_dir" - echo "==> $maestro --device $device_id test $maestro_flow" - MAESTRO_CLI_NO_ANALYTICS=true \ - MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED=true \ - "$maestro" --device "$device_id" test \ - -e APP_ID="$app_id" \ - -e SMOKE_TIMEOUT_MS="$((timeout_seconds * 1000))" \ - "$maestro_flow" \ - >"$reports_dir/maestro.log" 2>&1 & - local maestro_pid=$! - local failure_receipt="" - while kill -0 "$maestro_pid" 2>/dev/null; do - failure_receipt="$(latest_android_failure_receipt "$adb")" - [ -z "$failure_receipt" ] || break - sleep 1 - done - if [ -z "$failure_receipt" ]; then - # Close the race where the app emits its authoritative failure receipt as - # Maestro exits after observing the failed UI state. - failure_receipt="$(latest_android_failure_receipt "$adb")" - fi - if [ -n "$failure_receipt" ]; then - { - printf '%s\n' "$failure_receipt" - printf 'maestroPid=%s\n' "$maestro_pid" - } >"$reports_dir/maestro-authoritative-failure.txt" - terminate_maestro_process "$maestro_pid" - wait "$maestro_pid" 2>/dev/null || true - printf '%s\n' "$failure_receipt" >&2 - tail -160 "$reports_dir/maestro.log" >&2 || true - return 2 - fi - local maestro_status=0 - wait "$maestro_pid" || maestro_status=$? - if [ "$maestro_status" -ne 0 ]; then - tail -160 "$reports_dir/maestro.log" >&2 || true - return 1 - fi - tail -80 "$reports_dir/maestro.log" >&2 || true -} - -latest_android_failure_receipt() { - local adb="$1" - "$adb" logcat -d -v raw ReactNativeJS:I '*:S' 2>/dev/null | - grep -F "$failure_tag" | - tail -1 || true -} - -terminate_maestro_process() { - local maestro_pid="$1" - kill -0 "$maestro_pid" 2>/dev/null || return 0 - # The checksum-pinned Maestro launcher ends with `exec "$JAVACMD" "$@"`, - # so this PID is the JVM rather than a shell wrapper that could orphan it. - kill -TERM "$maestro_pid" 2>/dev/null || true - # Maestro normally exits immediately on TERM. The KILL fallback prevents a - # wedged launcher from defeating the authoritative app-side fail-fast path. - sleep 0.2 - kill -KILL "$maestro_pid" 2>/dev/null || true -} - latest_metro_tag() { local offset="$1" local tag="$2" @@ -329,37 +245,35 @@ install_and_launch() { local url url="$(android_runner_url "$runner")" local shell_url="'$url'" - run "$adb" shell am start -a android.intent.action.VIEW -d "$shell_url" "$app_id" + local android_log_file="$scratch_root/logs/$runner-logcat.log" + local app_uid + app_uid="$("$adb" shell pm list packages -U --user current "$app_id" | + awk -v package="package:$app_id" '$1 == package {sub(/^uid:/, "", $2); sub(/\r$/, "", $2); print $2}')" + [[ "$app_uid" =~ ^[0-9]+$ ]] || fail "failed to resolve Android app UID for $app_id" + mkdir -p "$scratch_root/logs" + # The installed UID scopes every app process before launch, including early crashes. + "$adb" logcat -v raw --uid="$app_uid" ReactNativeJS:I AndroidRuntime:E '*:S' >"$android_log_file" 2>&1 & + android_log_pid=$! + run "$adb" shell am start -W -a android.intent.action.VIEW -d "$shell_url" "$app_id" if [ "$build_type" = "debug" ]; then dismiss_expo_dev_menu_onboarding "$adb" fi local logs pass - if should_use_maestro_e2e; then - [ "$lifecycle_smoke" != "1" ] || - fail "Maestro mobile E2E does not drive lifecycle transitions; use mobile-drill or set OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE=0" - local maestro_status=0 - run_maestro_installed_smoke "$device_id" "$adb" || maestro_status=$? - if [ "$maestro_status" -ne 0 ]; then - if [ "$maestro_status" -eq 2 ]; then - write_android_e2e_diagnostics "$adb" "authoritative-smoke-failure" - fail "Expo Android installed app emitted $failure_tag while Maestro was running" - fi - write_android_e2e_diagnostics "$adb" "maestro-failure" - fail "Expo Android installed-app Maestro smoke failed" - fi - logs="$("$adb" logcat -d)" - pass="$(latest_metro_tag "$metro_offset" "$success_tag")" - if [ -z "$pass" ]; then - pass="$(printf '%s\n' "$logs" | grep -F "$success_tag" | tail -1 || true)" - fi - if [ -n "$pass" ]; then - printf '\n%s\n' "$pass" - write_runner_report "$pass" - else - write_android_e2e_diagnostics "$adb" "missing-authoritative-pass-receipt" - fail "Expo Android installed-app smoke UI passed without an authoritative OLIPHAUNT_EXPO_SMOKE_PASS receipt" + if [ "$runner" = smoke ] && [ "$lifecycle_smoke" != 1 ]; then + local receipt_status=0 + mobile_app_is_alive() { "$adb" shell pidof "$app_id" >/dev/null 2>&1; } + pass="$(wait_for_mobile_receipt "$android_log_file" "$android_log_pid")" || receipt_status=$? + kill -0 "$android_log_pid" 2>/dev/null || receipt_status=3 + stop_mobile_log_capture "$android_log_pid" + android_log_pid="" + mobile_log_has_failure "$android_log_file" && receipt_status=2 + mobile_app_is_alive || receipt_status=4 + if [ "$receipt_status" != 0 ]; then + write_android_e2e_diagnostics "$adb" receipt-failure + fail "Android smoke receipt failed (status $receipt_status: timeout=1, failure=2, capture=3, app=4)" fi + write_runner_report "$pass" || fail "invalid Android smoke receipt" write_android_process_metrics "$adb" return fi diff --git a/src/native/sdks/react-native/tools/expo-runner-android-device.test.sh b/src/native/sdks/react-native/tools/expo-runner-android-device.test.sh index 45ae2dd3f..22fa35959 100644 --- a/src/native/sdks/react-native/tools/expo-runner-android-device.test.sh +++ b/src/native/sdks/react-native/tools/expo-runner-android-device.test.sh @@ -1,141 +1,123 @@ #!/usr/bin/env bash set -euo pipefail +root="$(git rev-parse --show-toplevel)" +source "$root/src/native/sdks/react-native/tools/expo-runner-common.sh" +scratch="$(mktemp -d)" +capture_pid="" +trap 'stop_mobile_log_capture "$capture_pid"; rm -rf "$scratch"' EXIT +success_tag=OLIPHAUNT_EXPO_SMOKE_PASS +failure_tag=OLIPHAUNT_EXPO_SMOKE_FAIL +timeout_seconds=0 +mobile_app_is_alive() { [ "$app_alive" = true ]; } +sleep 60 & +capture_pid=$! +app_alive=true +for scenario in pass fail crash missing dead-capture dead-app; do + printf '%s valid-receipt\n' "$success_tag" >"$scratch/log" + expected=0 + pid="$capture_pid" + app_alive=true + case "$scenario" in + fail) printf '%s explicit-failure\n' "$failure_tag" >>"$scratch/log"; expected=2 ;; + crash) printf 'FATAL EXCEPTION\n' >>"$scratch/log"; expected=2 ;; + missing) : >"$scratch/log"; expected=1 ;; + dead-capture) pid=99999999; expected=3 ;; + dead-app) app_alive=false; expected=4 ;; + esac + status=0 + wait_for_mobile_receipt "$scratch/log" "$pid" >"$scratch/out" 2>"$scratch/err" || status=$? + [ "$status" = "$expected" ] || { echo "$scenario: expected $expected, got $status" >&2; exit 1; } +done +# A fast terminal receipt remains readable while the current capture is alive. +timeout_seconds=5 +app_alive=true +: >"$scratch/log" +(sleep 0.1; printf '%s valid-receipt\n' "$success_tag" >>"$scratch/log") & +writer=$! +wait_for_mobile_receipt "$scratch/log" "$capture_pid" >"$scratch/out" +wait "$writer" +grep -Fq "$success_tag" "$scratch/out" +stop_mobile_log_capture "$capture_pid" +if kill -0 "$capture_pid" 2>/dev/null; then exit 1; fi +capture_pid="" +echo 'Mobile receipts: failure precedence, crash, missing receipt, capture/app death and cleanup passed' -root="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "must run inside the Oliphaunt git checkout" >&2 - exit 1 -} -. "$root/src/native/sdks/react-native/tools/expo-runner-android-device.sh" -test_root="$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-android-maestro-test.XXXXXX")" -trap 'rm -rf "$test_root"' EXIT - -scratch_root="$test_root/scratch" -maestro_flow="$test_root/installed-smoke.yaml" -app_id="dev.oliphaunt.test" -timeout_seconds=600 -failure_tag="OLIPHAUNT_EXPO_SMOKE_FAIL" -fake_maestro="$test_root/maestro" -fake_adb="$test_root/adb" -export FAKE_MAESTRO_STARTED="$test_root/maestro-started" -export FAKE_MAESTRO_TERMINATED="$test_root/maestro-terminated" -export FAKE_MAESTRO_PID="$test_root/maestro-pid" -export FAKE_MAESTRO_MODE=slow -export FAKE_ADB_FAILURE_FILE="$test_root/adb-failure" - -mkdir -p "$scratch_root" -printf 'appId: dev.oliphaunt.test\n---\n- assertVisible: smoke\n' >"$maestro_flow" - -cat >"$fake_maestro" <<'SH' -#!/usr/bin/env sh -printf '%s\n' "$$" >"$FAKE_MAESTRO_PID" -printf 'started\n' >"$FAKE_MAESTRO_STARTED" -case "$FAKE_MAESTRO_MODE" in - success) - printf 'simulated Maestro success\n' - exit 0 - ;; - error) - printf 'simulated Maestro failure\n' >&2 - exit 7 +# Exercise the installed-app entry point: logcat clearing must discard an old +# PASS, and only this app's receipt emitted after this launch can succeed. +source "$root/src/native/sdks/react-native/tools/expo-runner-android-device.sh" +mkdir -p "$scratch/sdk/platform-tools" "$scratch/app" +cat > "$scratch/sdk/platform-tools/adb" <<'ADB' +#!/usr/bin/env bash +set -eu +case "$*" in + devices) printf 'List of devices attached\nfixture\tdevice\n' ;; + 'install -r '*|'shell am force-stop '*|'shell pm clear '*|'shell pidof '*) ;; + 'shell pm list packages -U --user current dev.oliphaunt.fixture') + printf 'package:dev.oliphaunt.fixture.other uid:10099\r\n' + case "$ADB_SCENARIO" in + missing-uid) ;; + invalid-uid) printf 'package:dev.oliphaunt.fixture uid:invalid\r\n' ;; + *) printf 'package:dev.oliphaunt.fixture uid:10042\r\n' ;; + esac ;; - slow) - trap 'printf "terminated\n" >"$FAKE_MAESTRO_TERMINATED"; exit 143' TERM INT - count=0 - while [ "$count" -lt 50 ]; do - sleep 0.1 - count=$((count + 1)) + 'logcat -c') : > "$ADB_RECEIPT" ;; + 'logcat -v '*) + trap 'exit 0' TERM + uid_filter="" + for arg in "$@"; do + case "$arg" in --uid=*) uid_filter="${arg#--uid=}" ;; esac done - exit 0 + while [ ! -s "$ADB_RECEIPT" ]; do sleep 0.05; done + awk -v uid="$uid_filter" 'uid == "" || $1 == uid {sub(/^[0-9]+ /, ""); print}' "$ADB_RECEIPT" + while :; do sleep 0.05; done ;; - *) - exit 64 + 'shell am start -W '*) + case "$ADB_SCENARIO" in + stale) ;; + unrelated-pass) printf '10099 %s\n' "$ADB_CURRENT_RECEIPT" > "$ADB_RECEIPT" ;; + *) + { + case "$ADB_SCENARIO" in + unrelated-crash) printf '10099 FATAL EXCEPTION: main\n' ;; + app-crash) printf '10042 FATAL EXCEPTION: main\n' ;; + esac + printf '10042 %s\n' "$ADB_CURRENT_RECEIPT" + } > "$ADB_RECEIPT" + ;; + esac ;; + *) echo "unexpected adb invocation: $*" >&2; exit 9 ;; esac -SH -chmod +x "$fake_maestro" - -cat >"$fake_adb" <<'SH' -#!/usr/bin/env sh -if [ "$*" = "logcat -d -v raw ReactNativeJS:I *:S" ] && - [ -f "$FAKE_MAESTRO_STARTED" ] && [ -s "$FAKE_ADB_FAILURE_FILE" ]; then - cat "$FAKE_ADB_FAILURE_FILE" -fi -SH -chmod +x "$fake_adb" - -maestro_binary() { - printf '%s\n' "$fake_maestro" -} - -fail_test() { - echo "expo-runner-android-device.test.sh: $*" >&2 - exit 1 -} - -reset_fixture() { - rm -rf "$scratch_root/reports" - rm -f \ - "$FAKE_MAESTRO_STARTED" \ - "$FAKE_MAESTRO_TERMINATED" \ - "$FAKE_MAESTRO_PID" \ - "$FAKE_ADB_FAILURE_FILE" -} - -reset_fixture -printf '%s\n' "07-18 12:00:03.244 I ReactNativeJS: $failure_tag {\"error\":\"fixture\"}" \ - >"$FAKE_ADB_FAILURE_FILE" -start_seconds=$SECONDS -set +e -run_maestro_installed_smoke emulator-5554 "$fake_adb" \ - >"$test_root/fail-fast.stdout" 2>"$test_root/fail-fast.stderr" -status=$? -set -e -[ "$status" -eq 2 ] || fail_test "authoritative failure returned $status instead of 2" -# Loaded CI hosts can delay the one-second receipt poll and shell process -# reaping even after TERM is delivered. The assertions below independently -# prove that Maestro received TERM and is no longer running. -[ $((SECONDS - start_seconds)) -lt 10 ] || fail_test "authoritative failure did not terminate Maestro promptly" -[ -f "$FAKE_MAESTRO_TERMINATED" ] || fail_test "authoritative failure did not terminate Maestro" -maestro_pid="$(cat "$FAKE_MAESTRO_PID")" -if kill -0 "$maestro_pid" 2>/dev/null; then - fail_test "terminated Maestro process $maestro_pid is still running" -fi -grep -Fq "$failure_tag" "$scratch_root/reports/maestro-authoritative-failure.txt" || - fail_test "authoritative failure report omitted the app receipt" -grep -Fq "$failure_tag" "$test_root/fail-fast.stderr" || - fail_test "authoritative failure was not printed to stderr" - -reset_fixture -export FAKE_MAESTRO_MODE=success -run_maestro_installed_smoke emulator-5554 "$fake_adb" \ - >"$test_root/success.stdout" 2>"$test_root/success.stderr" || - fail_test "successful Maestro run was rejected" -grep -Fq "simulated Maestro success" "$scratch_root/reports/maestro.log" || - fail_test "successful Maestro output was not preserved" - -reset_fixture -export FAKE_MAESTRO_MODE=error -set +e -run_maestro_installed_smoke emulator-5554 "$fake_adb" \ - >"$test_root/error.stdout" 2>"$test_root/error.stderr" -status=$? -set -e -[ "$status" -eq 1 ] || fail_test "failed Maestro run returned $status instead of 1" -grep -Fq "simulated Maestro failure" "$scratch_root/reports/maestro.log" || - fail_test "failed Maestro output was not preserved" - -# Exercise the final logcat read after an immediate Maestro exit. This closes -# the race between the UI assertion ending and the authoritative app receipt. -reset_fixture -export FAKE_MAESTRO_MODE=success -printf '%s\n' "07-18 12:00:03.244 I ReactNativeJS: $failure_tag {\"error\":\"race\"}" \ - >"$FAKE_ADB_FAILURE_FILE" -set +e -run_maestro_installed_smoke emulator-5554 "$fake_adb" \ - >"$test_root/race.stdout" 2>"$test_root/race.stderr" -status=$? -set -e -[ "$status" -eq 2 ] || fail_test "final authoritative failure read returned $status instead of 2" - -echo "Android Maestro fail-fast tests passed" +ADB +chmod +x "$scratch/sdk/platform-tools/adb" +export ADB_RECEIPT="$scratch/adb-receipt" +export ADB_CURRENT_RECEIPT="$success_tag current-launch" +ANDROID_HOME="$scratch/sdk" +app_id=dev.oliphaunt.fixture +apk="$scratch/app.apk" +build_type=release +runner=smoke +lifecycle_smoke=0 +scratch_root="$scratch/app" +timeout_seconds=3 +wake_android_device() { :; } +android_runner_url() { printf 'fixture://smoke'; } +write_android_process_metrics() { :; } +write_android_e2e_diagnostics() { :; } +write_runner_report() { [ "$1" = "$ADB_CURRENT_RECEIPT" ]; } +for scenario in pass unrelated-crash app-crash stale unrelated-pass missing-uid invalid-uid; do + printf '10042 %s stale-launch\n' "$success_tag" > "$ADB_RECEIPT" + export ADB_SCENARIO="$scenario" + status=0 + ( + android_log_pid="" + trap 'stop_mobile_log_capture "$android_log_pid"' EXIT + install_and_launch + ) > "$scratch/$scenario-launch.log" 2>&1 || status=$? + case "$scenario" in + pass|unrelated-crash) [ "$status" = 0 ] ;; + *) [ "$status" != 0 ] ;; + esac || { cat "$scratch/$scenario-launch.log" >&2; echo "unexpected $scenario status: $status" >&2; exit 1; } +done +echo 'Android installed-app capture scopes receipts and crashes to the current app and launch' diff --git a/src/native/sdks/react-native/tools/expo-runner-common.sh b/src/native/sdks/react-native/tools/expo-runner-common.sh index cdec93561..a92faadf5 100644 --- a/src/native/sdks/react-native/tools/expo-runner-common.sh +++ b/src/native/sdks/react-native/tools/expo-runner-common.sh @@ -71,16 +71,38 @@ need_cmd() { command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1" } -maestro_binary() { - if command -v maestro >/dev/null 2>&1; then - command -v maestro - return - fi - if [ -x "$HOME/.maestro/bin/maestro" ]; then - printf '%s\n' "$HOME/.maestro/bin/maestro" - return - fi - return 1 +# The app owns SDK assertions. Read only the continuous capture started for this +# launch, and reject failures/dead processes before accepting its receipt. +mobile_log_has_failure() { + grep -Eq "$failure_tag|Fatal error|FATAL EXCEPTION|terminating with uncaught exception" "$1" +} + +wait_for_mobile_receipt() { + local log_file="$1" capture_pid="$2" + local deadline=$((SECONDS + timeout_seconds)) logs pass + while :; do + logs="$(cat "$log_file")" || return 3 + if mobile_log_has_failure "$log_file"; then tail -20 "$log_file" >&2; return 2; fi + kill -0 "$capture_pid" 2>/dev/null || return 3 + mobile_app_is_alive || return 4 + pass="$(printf '%s\n' "$logs" | grep -F "$success_tag" | tail -1 || true)" + if [ -n "$pass" ]; then printf '%s\n' "$pass"; return 0; fi + [ "$SECONDS" -lt "$deadline" ] || return 1 + sleep 1 + done +} + +stop_mobile_log_capture() { + local pid="${1:-}" + [ -n "$pid" ] || return 0 + kill -TERM "$pid" 2>/dev/null || true + local attempts=20 + while kill -0 "$pid" 2>/dev/null && [ "$attempts" -gt 0 ]; do + sleep 0.1 + attempts=$((attempts - 1)) + done + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true } stat_mtime() { diff --git a/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh b/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh index 75a31cee2..9b46344e5 100644 --- a/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh +++ b/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.sh @@ -30,99 +30,6 @@ latest_metro_runner_failure() { } | grep -E "$failure_tag|metro:bundling:failed|Unable to resolve" | tail -20 || true } -should_use_maestro_e2e() { - [ "$runner" = "smoke" ] || return 1 - [ "$sdk" = "iphonesimulator" ] || return 1 - case "$e2e_assertion_runner" in - maestro) - maestro_binary >/dev/null || fail "missing required command: maestro; run tools/dev/setup-maestro.sh" - return 0 - ;; - auto) - maestro_binary >/dev/null - return - ;; - *) - return 1 - ;; - esac -} - -run_maestro_installed_smoke() { - local device_udid="$1" - local reports_dir="$scratch_root/reports" - [ -f "$maestro_flow" ] || fail "missing Maestro installed-app smoke flow: $maestro_flow" - local maestro - maestro="$(maestro_binary)" || fail "missing required command: maestro; run tools/dev/setup-maestro.sh" - mkdir -p "$reports_dir" - echo "==> $maestro --device $device_udid test $maestro_flow" - MAESTRO_CLI_NO_ANALYTICS=true \ - MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED=true \ - "$maestro" --device "$device_udid" test \ - -e APP_ID="$app_id" \ - -e SMOKE_TIMEOUT_MS="$((timeout_seconds * 1000))" \ - "$maestro_flow" \ - >"$reports_dir/maestro.log" 2>&1 & - local maestro_pid=$! - local failure_receipt="" - local capture_failed=0 - while kill -0 "$maestro_pid" 2>/dev/null; do - failure_receipt="$(latest_ios_simulator_capture_tag "$failure_tag")" - [ -z "$failure_receipt" ] || break - if ! ios_simulator_log_capture_is_alive; then - capture_failed=1 - break - fi - sleep 1 - done - if [ -z "$failure_receipt" ]; then - # Close the race where the app emits its terminal receipt as Maestro exits - # after observing the corresponding UI state. - failure_receipt="$(latest_ios_simulator_capture_tag "$failure_tag")" - fi - if [ -n "$failure_receipt" ]; then - { - printf '%s\n' "$failure_receipt" - printf 'maestroPid=%s\n' "$maestro_pid" - printf 'simulatorLog=%s\n' "${ios_simulator_log_file:-}" - } >"$reports_dir/maestro-authoritative-failure.txt" - terminate_ios_maestro_process "$maestro_pid" - wait "$maestro_pid" 2>/dev/null || true - printf '%s\n' "$failure_receipt" >&2 - tail -160 "$reports_dir/maestro.log" >&2 || true - return 2 - fi - if [ "$capture_failed" = "1" ]; then - { - printf 'maestroPid=%s\n' "$maestro_pid" - printf 'simulatorLog=%s\n' "${ios_simulator_log_file:-}" - printf 'reason=unified-log-capture-ended-before-maestro\n' - } >"$reports_dir/maestro-log-capture-failure.txt" - terminate_ios_maestro_process "$maestro_pid" - wait "$maestro_pid" 2>/dev/null || true - tail -160 "$reports_dir/maestro.log" >&2 || true - [ -z "${ios_simulator_log_file:-}" ] || tail -160 "$ios_simulator_log_file" >&2 || true - return 3 - fi - local maestro_status=0 - wait "$maestro_pid" || maestro_status=$? - if [ "$maestro_status" -ne 0 ]; then - tail -160 "$reports_dir/maestro.log" >&2 || true - return 1 - fi - tail -80 "$reports_dir/maestro.log" >&2 || true -} - -terminate_ios_maestro_process() { - local maestro_pid="$1" - kill -0 "$maestro_pid" 2>/dev/null || return 0 - # The checksum-pinned Maestro launcher ends with `exec "$JAVACMD" "$@"`, - # so this PID is the JVM rather than a shell wrapper that could orphan it. - kill -TERM "$maestro_pid" 2>/dev/null || true - sleep 0.2 - kill -KILL "$maestro_pid" 2>/dev/null || true -} - resolve_ios_app_process_name() { local app="$1" local plist="$app/Info.plist" @@ -178,7 +85,7 @@ start_ios_simulator_log_capture() { ios_simulator_log_pid=$! # Start the stream before launching the app so a fast Release smoke cannot - # age out while Maestro starts its driver. The scoped file is also durable + # age out before the runner observes it. The scoped file is also durable # evidence and cannot contain a receipt from an already-terminated launch. sleep "${OLIPHAUNT_EXPO_IOS_LOG_CAPTURE_STARTUP_SECONDS:-1}" if ! ios_simulator_log_capture_is_alive; then @@ -216,62 +123,6 @@ stop_ios_simulator_log_capture() { return 0 } -latest_ios_simulator_capture_tag() { - local tag="$1" - [ -n "${ios_simulator_log_file:-}" ] && [ -f "$ios_simulator_log_file" ] || return 0 - grep -F "$tag" "$ios_simulator_log_file" | tail -1 || true -} - -wait_for_ios_simulator_maestro_receipt() { - local grace_seconds="${OLIPHAUNT_EXPO_IOS_RECEIPT_GRACE_SECONDS:-15}" - case "$grace_seconds" in - '' | *[!0-9]*) - echo "OLIPHAUNT_EXPO_IOS_RECEIPT_GRACE_SECONDS must be a nonnegative integer, got $grace_seconds" >&2 - return 4 - ;; - esac - local deadline=$((SECONDS + grace_seconds)) - local pass failure_receipt - while :; do - failure_receipt="$(latest_ios_simulator_capture_tag "$failure_tag")" - if [ -n "$failure_receipt" ]; then - printf '%s\n' "$failure_receipt" >&2 - return 2 - fi - pass="$(latest_ios_simulator_capture_tag "$success_tag")" - if [ -n "$pass" ]; then - printf '%s\n' "$pass" - return 0 - fi - ios_simulator_log_capture_is_alive || return 3 - [ "$SECONDS" -lt "$deadline" ] || return 1 - sleep 1 - done -} - -write_ios_maestro_diagnostics() { - local device_udid="$1" - local process_name="$2" - local reason="$3" - local reports_dir="$scratch_root/reports" - mkdir -p "$reports_dir" - { - printf 'reason=%s\n' "$reason" - printf 'deviceUdid=%s\n' "$device_udid" - printf 'processName=%s\n' "$process_name" - printf 'simulatorLog=%s\n' "${ios_simulator_log_file:-}" - } >"$reports_dir/maestro-$reason.txt" - xcrun simctl spawn "$device_udid" log show \ - --style compact \ - --last 15m \ - --predicate "process == '$process_name'" \ - >"$reports_dir/maestro-unified-log-$reason.txt" 2>&1 || true - xcrun simctl io "$device_udid" screenshot \ - "$reports_dir/maestro-screen-$reason.png" >/dev/null 2>&1 || true - [ -s "$reports_dir/maestro-screen-$reason.png" ] || rm -f "$reports_dir/maestro-screen-$reason.png" - tail -200 "$reports_dir/maestro-unified-log-$reason.txt" >&2 || true -} - write_ios_process_metrics() { local launch_pid="$1" [ -n "$launch_pid" ] || return 0 @@ -707,16 +558,10 @@ install_and_launch() { local scratch_metro_offset dev_metro_offset scratch_metro_offset="$(file_bytes "$scratch_root/metro.log")" dev_metro_offset="$(file_bytes "$metro_dev_log")" - local use_maestro_e2e=0 app_process_name="" - if should_use_maestro_e2e; then - [ "$lifecycle_smoke" != "1" ] || - fail "Maestro mobile E2E does not drive lifecycle transitions; use mobile-drill or set OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE=0" - app_process_name="$(resolve_ios_app_process_name "$app")" || - fail "failed to resolve the installed iOS app executable for unified-log capture" - start_ios_simulator_log_capture "$device_udid" "$app_process_name" || - fail "failed to start exact-launch iOS unified-log capture" - use_maestro_e2e=1 - fi + local app_process_name + app_process_name="$(resolve_ios_app_process_name "$app")" || fail "missing iOS executable" + start_ios_simulator_log_capture "$device_udid" "$app_process_name" || + fail "failed to start exact-launch iOS unified-log capture" local url url="$(ios_runner_url "$runner")" local launch_output launch_pid @@ -731,45 +576,18 @@ install_and_launch() { fi local logs pass - if [ "$use_maestro_e2e" = "1" ]; then - local maestro_status=0 - run_maestro_installed_smoke "$device_udid" || maestro_status=$? - if [ "$maestro_status" -ne 0 ]; then - stop_ios_simulator_log_capture || true - if [ "$maestro_status" = "2" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "authoritative-smoke-failure" - fail "Expo iOS installed app emitted $failure_tag while Maestro was running" - fi - if [ "$maestro_status" = "3" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "log-capture-failure" - fail "Expo iOS exact-launch unified-log capture ended while Maestro was running" - fi - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "maestro-failure" - fail "Expo iOS installed-app Maestro smoke failed" - fi - + if [ "$runner" = smoke ] && [ "$lifecycle_smoke" != 1 ]; then local receipt_status=0 - pass="$(wait_for_ios_simulator_maestro_receipt)" || receipt_status=$? - stop_ios_simulator_log_capture || true - if [ "$receipt_status" = "0" ]; then - printf '\n%s\n' "$pass" - if ! write_runner_report "$pass"; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "invalid-authoritative-pass-receipt" - fail "Expo iOS app emitted an invalid authoritative OLIPHAUNT_EXPO_SMOKE_PASS receipt" - fi - elif [ "$receipt_status" = "2" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "authoritative-smoke-failure" - fail "Expo iOS installed app emitted $failure_tag after Maestro observed the UI" - elif [ "$receipt_status" = "3" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "log-capture-failure" - fail "Expo iOS exact-launch unified-log capture ended before the app receipt was collected" - elif [ "$receipt_status" = "4" ]; then - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "invalid-receipt-grace" - fail "Expo iOS receipt grace configuration is invalid" - else - write_ios_maestro_diagnostics "$device_udid" "$app_process_name" "missing-authoritative-pass-receipt" - fail "Expo iOS installed-app smoke UI passed without an authoritative OLIPHAUNT_EXPO_SMOKE_PASS receipt" + mobile_app_is_alive() { [ -n "$launch_pid" ] && kill -0 "$launch_pid" 2>/dev/null; } + pass="$(wait_for_mobile_receipt "$ios_simulator_log_file" "$ios_simulator_log_pid")" || receipt_status=$? + stop_ios_simulator_log_capture || receipt_status=3 + mobile_log_has_failure "$ios_simulator_log_file" && receipt_status=2 + mobile_app_is_alive || receipt_status=4 + if [ "$receipt_status" != 0 ]; then + tail -200 "$ios_simulator_log_file" >&2 || true + fail "iOS smoke receipt failed (status $receipt_status: timeout=1, failure=2, capture=3, app=4)" fi + write_runner_report "$pass" || fail "invalid iOS smoke receipt" write_ios_process_metrics "$launch_pid" return fi @@ -777,7 +595,7 @@ install_and_launch() { local deadline=$((SECONDS + timeout_seconds)) local fail_line lifecycle_exercised=0 while [ "$SECONDS" -lt "$deadline" ]; do - logs="$(xcrun simctl spawn "$device_udid" log show --style compact --last 30s --predicate "process == 'reactnativeoliphauntexpo'" 2>/dev/null || true)" + logs="$(cat "$ios_simulator_log_file")" if [ "$lifecycle_smoke" = "1" ] && [ "$lifecycle_exercised" = "0" ]; then if { printf '%s\n' "$logs" diff --git a/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.test.sh b/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.test.sh index eec4544b6..56f85e1fe 100644 --- a/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.test.sh +++ b/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.test.sh @@ -60,52 +60,12 @@ for profile in standard icu; do fi done scratch_root="$test_root/scratch" -maestro_flow="$test_root/installed-smoke.yaml" -app_id="dev.oliphaunt.test" -runner="smoke" -mobile_platform="ios" -timeout_seconds=600 -success_tag="OLIPHAUNT_EXPO_SMOKE_PASS" -failure_tag="OLIPHAUNT_EXPO_SMOKE_FAIL" -ios_simulator_log_pid="" -ios_simulator_log_file="$test_root/simulator.log" +runner=smoke +mobile_platform=ios +success_tag=OLIPHAUNT_EXPO_SMOKE_PASS export CI_HEAD_SHA="$(git rev-parse HEAD)" export OLIPHAUNT_MOBILE_E2E_EXPECT_ICU=0 export OLIPHAUNT_MOBILE_E2E_EXPECT_CATALOG_PROFILE=standard -export FAKE_MAESTRO_STARTED="$test_root/maestro-started" -export FAKE_MAESTRO_TERMINATED="$test_root/maestro-terminated" - -mkdir -p "$scratch_root/reports" -printf 'appId: dev.oliphaunt.test\n---\n- assertVisible: smoke\n' >"$maestro_flow" -fake_maestro="$test_root/maestro" -cat >"$fake_maestro" <<'SH' -#!/usr/bin/env bash -trap 'printf "terminated\n" >"$FAKE_MAESTRO_TERMINATED"; exit 143' TERM INT -printf 'started\n' >"$FAKE_MAESTRO_STARTED" -while :; do sleep 0.1; done -SH -chmod +x "$fake_maestro" - -maestro_binary() { printf '%s\n' "$fake_maestro"; } -ios_simulator_log_capture_is_alive() { return 0; } -latest_ios_simulator_capture_tag() { - [ "$1" = "$failure_tag" ] || return 0 - local attempts=100 - while [ "$attempts" -gt 0 ] && [ ! -f "$FAKE_MAESTRO_STARTED" ]; do - command sleep 0.01 - attempts=$((attempts - 1)) - done - printf '%s fixture\n' "$failure_tag" -} - -set +e -run_maestro_installed_smoke simulator-1 >"$test_root/fail.stdout" 2>"$test_root/fail.stderr" -status=$? -set -e -[ "$status" -eq 2 ] -[ -f "$FAKE_MAESTRO_TERMINATED" ] -grep -Fq "$failure_tag" "$scratch_root/reports/maestro-authoritative-failure.txt" - receipt_json="$( bun "$root/src/native/sdks/react-native/tools/expo-runner-ios-installed-app.fixture.mts" receipt "$root/src/extensions/generated/sdk/extensions.json" )" diff --git a/src/native/sdks/react-native/tools/expo-runner-reporting.mts b/src/native/sdks/react-native/tools/expo-runner-reporting.mts index 16b04a416..491a9541d 100644 --- a/src/native/sdks/react-native/tools/expo-runner-reporting.mts +++ b/src/native/sdks/react-native/tools/expo-runner-reporting.mts @@ -175,27 +175,6 @@ switch (command) { ); break; } - case 'maestro-report': { - const report = { - runner: 'maestro', - platform: process.env.OLIPHAUNT_MAESTRO_PLATFORM, - appId: process.env.OLIPHAUNT_MAESTRO_APP_ID, - flow: process.env.OLIPHAUNT_MAESTRO_FLOW, - passedAt: new Date().toISOString(), - }; - process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); - break; - } - case 'maestro-pass': { - const report = { - runner: 'maestro', - platform: process.env.OLIPHAUNT_MAESTRO_PLATFORM, - appId: process.env.OLIPHAUNT_MAESTRO_APP_ID, - flow: process.env.OLIPHAUNT_MAESTRO_FLOW, - }; - process.stdout.write(`OLIPHAUNT_EXPO_MAESTRO_PASS ${JSON.stringify(report)}\n`); - break; - } case 'package-sizes': { const [report, artifactSizeKey, artifactBytes, rnPackageBytes] = args; const payload = { diff --git a/src/native/sdks/react-native/tools/expo-runner-reporting.sh b/src/native/sdks/react-native/tools/expo-runner-reporting.sh index 198f26f42..655808f0d 100644 --- a/src/native/sdks/react-native/tools/expo-runner-reporting.sh +++ b/src/native/sdks/react-native/tools/expo-runner-reporting.sh @@ -2,7 +2,7 @@ # Shared report helpers for React Native Expo mobile runners. Platform runners # own platform metrics and artifact copying; this file only normalizes runner -# pass/report JSON emitted from Metro logs or Maestro installed-app flows. +# pass/report JSON emitted by the app through Metro or platform logs. configure_mobile_catalog_profile_probe() { local profile="$1" @@ -307,19 +307,6 @@ verify_mobile_e2e_smoke_receipt() { echo "$platform mobile E2E extension receipt postcondition: $receipt" >&2 } -write_maestro_runner_report() { - local platform="$1" - local reports_dir="$scratch_root/reports" - mkdir -p "$reports_dir" - OLIPHAUNT_MAESTRO_PLATFORM="$platform" \ - OLIPHAUNT_MAESTRO_APP_ID="$app_id" \ - OLIPHAUNT_MAESTRO_FLOW="$maestro_flow" \ - bun "$root/src/native/sdks/react-native/tools/expo-runner-reporting.mts" maestro-report >"$reports_dir/$runner-report.json" - OLIPHAUNT_MAESTRO_PLATFORM="$platform" \ - OLIPHAUNT_MAESTRO_APP_ID="$app_id" \ - OLIPHAUNT_MAESTRO_FLOW="$maestro_flow" \ - bun "$root/src/native/sdks/react-native/tools/expo-runner-reporting.mts" maestro-pass >"$reports_dir/$runner-pass.log" -} write_mobile_package_size_report() { local artifact_size_key="$1" diff --git a/src/native/sdks/react-native/tools/mobile-e2e.sh b/src/native/sdks/react-native/tools/mobile-e2e.sh index f2a6d5727..90fc49ffe 100755 --- a/src/native/sdks/react-native/tools/mobile-e2e.sh +++ b/src/native/sdks/react-native/tools/mobile-e2e.sh @@ -36,7 +36,6 @@ case "$platform" in export OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE="${OLIPHAUNT_EXPO_ANDROID_BUILD_TYPE:-release}" export OLIPHAUNT_EXPO_ANDROID_E2E_ONLY=1 export OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE="${OLIPHAUNT_EXPO_ANDROID_LIFECYCLE_SMOKE:-0}" - export OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER="${OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER:-maestro}" export OLIPHAUNT_EXPO_ANDROID_SCRATCH="$mobile_scratch" if [ "$mobile_runner" = "smoke" ]; then command -v unzip >/dev/null 2>&1 || { @@ -70,7 +69,6 @@ case "$platform" in export OLIPHAUNT_EXPO_IOS_CONFIGURATION="${OLIPHAUNT_EXPO_IOS_CONFIGURATION:-Release}" export OLIPHAUNT_EXPO_IOS_E2E_ONLY=1 export OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE="${OLIPHAUNT_EXPO_IOS_LIFECYCLE_SMOKE:-0}" - export OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER="${OLIPHAUNT_MOBILE_E2E_ASSERTION_RUNNER:-maestro}" export OLIPHAUNT_EXPO_IOS_SCRATCH="$mobile_scratch" if [ "$mobile_runner" = "smoke" ]; then export_mobile_e2e_icu_expectation_from_ios_app "$app" diff --git a/src/native/sdks/react-native/tools/stage-release-artifacts.mts b/src/native/sdks/react-native/tools/stage-release-artifacts.mts index fc072cd6c..3e25a28e4 100644 --- a/src/native/sdks/react-native/tools/stage-release-artifacts.mts +++ b/src/native/sdks/react-native/tools/stage-release-artifacts.mts @@ -11,13 +11,15 @@ export function stageArtifacts(artifactRoot, workRoot) { const releasePackageDir = path.join(workRoot, 'package'); requireDir(releasePackageDir); const assetDir = process.env.OLIPHAUNT_REACT_NATIVE_IOS_RELEASE_ASSET_DIR; - if (!assetDir) { + const baseCarrierManifest = process.env.OLIPHAUNT_REACT_NATIVE_IOS_BASE_CARRIER || undefined; + if (!assetDir && !baseCarrierManifest) { fail( 'oliphaunt-react-native package artifacts require OLIPHAUNT_REACT_NATIVE_IOS_RELEASE_ASSET_DIR', ); } const carrier = buildIosCarrierManifest({ baseAssetDir: assetDir, + baseCarrierManifest, extensionManifests: [], }); writeFileSync( diff --git a/src/native/sdks/rust/moon.yml b/src/native/sdks/rust/moon.yml index 7077e31dc..70acd31e1 100644 --- a/src/native/sdks/rust/moon.yml +++ b/src/native/sdks/rust/moon.yml @@ -94,26 +94,30 @@ tasks: cache: true runFromWorkspaceRoot: true test-integration: - tags: ["regression", "runtime"] + tags: ["ci-native-consumers", "platform-linux-x64-gnu", "regression", "runtime", "requires-rust"] script: | set -e - . src/native/runtime/tools/runtime-preflight.sh - oliphaunt_runtime_native_host_require basic - cargo test -p oliphaunt --locked --test native_smoke --test native_sql_regression -- --test-threads=1 + bash src/native/sdks/tests/with-runtime.sh --tools --broker cargo nextest run -p oliphaunt --locked --test native_smoke --test native_sql_regression --no-tests=fail --test-threads=1 env: CARGO_TARGET_DIR: "target" deps: - target: "~:cargo-sources" cacheStrategy: hash - - "liboliphaunt-native:build-runtime-desktop-target" + - "liboliphaunt-native:package-runtime-desktop-target" + - "postgres-tools-native:package-assets" + - "oliphaunt-broker:build-release-assets" inputs: + - "/src/native/sdks/tests/with-runtime.sh" - "@group(cargo-workspace)" - project: "shared-test-fixtures" group: "fixtures" - - "@group(code)" + - "@group(sources)" + - "tests/native_smoke.rs" + - "tests/native_sql_regression.rs" + - "tests/support/**/*" - "/src/native/runtime/tools/runtime-preflight.sh" options: - cache: local + cache: false runFromWorkspaceRoot: true test-extensions: tags: ["regression", "runtime", "extensions"] @@ -273,8 +277,13 @@ tasks: packaging-unit: tags: ["quality", "unit", "requires-rust"] - command: "bash src/native/sdks/rust/tools/prepare-rust-release-source.test.sh" + script: | + set -e + bash src/native/sdks/rust/tools/prepare-rust-release-source.test.sh + bash src/native/sdks/tests/with-runtime.test.sh inputs: + - "/src/native/sdks/tests/with-runtime*.sh" + - "/src/native/runtime/tools/runtime-preflight.sh" - "**/*" - "/src/query/rust/src/lib.rs" - "/src/native/rust-bindings/**/*" diff --git a/src/native/sdks/swift/moon.yml b/src/native/sdks/swift/moon.yml index ca19832b3..ddbc36c2d 100644 --- a/src/native/sdks/swift/moon.yml +++ b/src/native/sdks/swift/moon.yml @@ -106,20 +106,20 @@ tasks: - "@group(code)" - "/src/native/runtime/include/oliphaunt.h" test-native: - tags: ["runtime", "smoke", "requires-swift"] + tags: ["runtime", "smoke", "requires-swift", "ci-native-consumers", "platform-linux-x64-gnu"] script: | - set -e - . src/native/runtime/tools/runtime-preflight.sh - oliphaunt_runtime_native_host_require basic - env OLIPHAUNT_SWIFT_REQUIRE_NATIVE=1 \ - LIBOLIPHAUNT_PATH="$(oliphaunt_runtime_native_host_lib)" \ - OLIPHAUNT_INSTALL_DIR="$(oliphaunt_runtime_native_host_install_dir)" \ - swift test --package-path src/native/sdks/swift --filter NativeRuntimeTests + set -euo pipefail + log="$(mktemp)" + trap 'rm -f "$log"' EXIT + bash src/native/sdks/tests/with-runtime.sh env OLIPHAUNT_SWIFT_REQUIRE_NATIVE=1 \ + swift test --package-path src/native/sdks/swift --filter NativeRuntimeTests 2>&1 | tee "$log" + grep -Eq 'Test run with [1-9][0-9]* tests? .*passed' "$log" deps: - "oliphaunt-swift:prepare-bindings" - {target: "oliphaunt-mobile-bindings:cargo-sources", cacheStrategy: hash} - - "liboliphaunt-native:build-runtime-desktop-target" + - "liboliphaunt-native:package-runtime-desktop-target" inputs: + - "/src/native/sdks/tests/with-runtime.sh" - project: "shared-test-fixtures" group: "fixtures" - project: "cluster-seed-contract" @@ -127,12 +127,15 @@ tasks: - "/src/native/runtime/include/oliphaunt.h" - project: "liboliphaunt-native" group: "runtime" - - "@group(code)" + - "Sources/**/*" + - "Tests/**/*" + - "Package.swift" + - ".swift-version" - "!/src/native/sdks/swift/.build" - "!/src/native/sdks/swift/.build/**" - "/src/native/runtime/tools/runtime-preflight.sh" options: - cache: local + cache: false runFromWorkspaceRoot: true package-source: tags: ["package"] diff --git a/src/native/sdks/swift/tools/prepare-bindings.sh b/src/native/sdks/swift/tools/prepare-bindings.sh index 6de4f05b4..065701701 100644 --- a/src/native/sdks/swift/tools/prepare-bindings.sh +++ b/src/native/sdks/swift/tools/prepare-bindings.sh @@ -2,7 +2,6 @@ set -euo pipefail root="$(git rev-parse --show-toplevel)" cd "$root" -bash src/native/mobile-bindings/tools/generate.sh stage="$root/src/native/sdks/swift/.build/native-bindings" mkdir -p "$stage/swift" "$stage/ffi" cp target/mobile-bindings/generated/OliphauntNativeBindings.swift "$stage/swift/" diff --git a/src/native/sdks/swift/tools/swift.sh b/src/native/sdks/swift/tools/swift.sh index 4ecd00548..f9caaaffa 100644 --- a/src/native/sdks/swift/tools/swift.sh +++ b/src/native/sdks/swift/tools/swift.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail root="$(git rev-parse --show-toplevel)" +bash "$root/src/native/mobile-bindings/tools/generate.sh" bash "$root/src/native/sdks/swift/tools/prepare-bindings.sh" exec swift "$@" --package-path "$root/src/native/sdks/swift" diff --git a/src/native/sdks/tests/README.md b/src/native/sdks/tests/README.md index 8066ca3bf..1f97d4ed3 100644 --- a/src/native/sdks/tests/README.md +++ b/src/native/sdks/tests/README.md @@ -29,7 +29,7 @@ runtime resources plus a seed. Do not mix a released runtime with changed C APIs The Swift builder expects these repository-relative artifacts: -- `target/mobile-bindings/generated/` (run the Swift `prepare-bindings.sh`). +- `target/mobile-bindings/generated/` (run `moon run oliphaunt-swift:prepare-bindings`). - `target/aarch64-apple-ios/debug/liboliphaunt_mobile_bindings.a` and `target/aarch64-apple-ios-sim/debug/liboliphaunt_mobile_bindings.a`. - `target/liboliphaunt-ios-simulator/out/liboliphaunt.dylib`. diff --git a/src/native/sdks/tests/with-runtime.sh b/src/native/sdks/tests/with-runtime.sh new file mode 100644 index 000000000..872162be9 --- /dev/null +++ b/src/native/sdks/tests/with-runtime.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" +cd "$root" +. src/native/runtime/tools/runtime-preflight.sh +target="$(oliphaunt_runtime_native_host_target_id)" +version="$(bun tools/release/product-version.mts version liboliphaunt-native)" +assets="${OLIPHAUNT_LIBOLIPHAUNT_RELEASE_ASSETS:-$root/target/liboliphaunt/desktop-release-assets/$target}" +stage="$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-sdk-runtime.XXXXXX")" +trap 'rm -rf "$stage"' EXIT +extract() { + mkdir -p "$2" + case "$target" in + windows-*) unzip -q "$1.zip" -d "$2" ;; + *) tar -xzf "$1.tar.gz" -C "$2" ;; + esac +} +extract "$assets/liboliphaunt-$version-$target" "$stage" +suffix= +case "$target" in + windows-*) library=bin/oliphaunt.dll; suffix=.exe ;; + macos-*) library=lib/liboliphaunt.dylib ;; + *) library=lib/liboliphaunt.so ;; +esac +export LIBOLIPHAUNT_PATH="$stage/$library" +export OLIPHAUNT_INSTALL_DIR="$stage/runtime" +export OLIPHAUNT_INITDB="$stage/runtime/bin/initdb$suffix" +export OLIPHAUNT_POSTGRES="$stage/runtime/bin/postgres$suffix" +export OLIPHAUNT_EMBEDDED_MODULE_DIR="$stage/lib/modules" +export LD_LIBRARY_PATH="$stage/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +while [ "${1:-}" = --tools ] || [ "${1:-}" = --broker ]; do + option="$1" + shift + if [ "$option" = --tools ]; then + tools_version="$(bun tools/release/product-version.mts version postgres-tools-native)" + tools_assets="${OLIPHAUNT_POSTGRES_TOOLS_RELEASE_ASSETS:-$root/target/postgres-tools/native/release-assets}" + extract "$tools_assets/oliphaunt-tools-$tools_version-$target" "$stage/tools" + export OLIPHAUNT_TOOLS_DIR="$stage/tools/runtime" + else + broker_version="$(bun tools/release/product-version.mts version oliphaunt-broker)" + broker_assets="${OLIPHAUNT_BROKER_RELEASE_ASSETS:-$root/target/oliphaunt-broker/release-assets}" + extract "$broker_assets/oliphaunt-broker-$broker_version-$target" "$stage/broker" + export OLIPHAUNT_BROKER="$stage/broker/bin/oliphaunt-broker$suffix" + test -x "$OLIPHAUNT_BROKER" + fi +done +[ "$#" -gt 0 ] || { echo "usage: with-runtime.sh [--tools] [--broker] COMMAND [ARGS...]" >&2; exit 2; } +oliphaunt_runtime_native_host_require basic +"$@" diff --git a/src/native/sdks/tests/with-runtime.test.sh b/src/native/sdks/tests/with-runtime.test.sh new file mode 100644 index 000000000..162fee94e --- /dev/null +++ b/src/native/sdks/tests/with-runtime.test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" "$scratch/assets" "$scratch/payload/lib" "$scratch/payload/runtime/bin" +cat > "$scratch/bin/bun" <<'BUN' +#!/usr/bin/env bash +printf '1.0.0\n' +BUN +chmod +x "$scratch/bin/bun" +. src/native/runtime/tools/runtime-preflight.sh +target="$(oliphaunt_runtime_native_host_target_id)" +case "$target" in + linux-*) library=liboliphaunt.so ;; + macos-*) library=liboliphaunt.dylib ;; + *) echo 'archive fixture runs on Unix hosts'; exit 0 ;; +esac +touch "$scratch/payload/lib/$library" +for tool in initdb postgres; do + printf '#!/bin/sh\nexit 0\n' > "$scratch/payload/runtime/bin/$tool" + chmod +x "$scratch/payload/runtime/bin/$tool" +done +archive="$scratch/assets/liboliphaunt-1.0.0-$target.tar.gz" +tar -czf "$archive" -C "$scratch/payload" . +export PATH="$scratch/bin:$PATH" +export OLIPHAUNT_LIBOLIPHAUNT_RELEASE_ASSETS="$scratch/assets" +export OLIPHAUNT_INITDB=/must/not/use/a/previous/runtime +"$BASH" src/native/sdks/tests/with-runtime.sh bash -c ' + test -f "$LIBOLIPHAUNT_PATH" + test -x "$OLIPHAUNT_INITDB" + test ! -e "$OLIPHAUNT_INSTALL_DIR/bin/pg_config" + printf "%s\n" "$OLIPHAUNT_INSTALL_DIR" > "$1" +' -- "$scratch/staged" +test ! -e "$(cat "$scratch/staged")" +tar -czf "$scratch/assets/oliphaunt-tools-1.0.0-$target.tar.gz" -C "$scratch/payload" . +mkdir -p "$scratch/broker/bin" +cp "$scratch/payload/runtime/bin/postgres" "$scratch/broker/bin/oliphaunt-broker" +tar -czf "$scratch/assets/oliphaunt-broker-1.0.0-$target.tar.gz" -C "$scratch/broker" . +export OLIPHAUNT_POSTGRES_TOOLS_RELEASE_ASSETS="$scratch/assets" +export OLIPHAUNT_BROKER_RELEASE_ASSETS="$scratch/assets" +"$BASH" src/native/sdks/tests/with-runtime.sh --tools --broker bash -c ' + test -x "$OLIPHAUNT_TOOLS_DIR/bin/postgres" + test -x "$OLIPHAUNT_BROKER" +' +status=0 +"$BASH" src/native/sdks/tests/with-runtime.sh bash -c 'exit 43' || status=$? +test "$status" = 43 +rm "$scratch/payload/runtime/bin/initdb" +tar -czf "$archive" -C "$scratch/payload" . +if "$BASH" src/native/sdks/tests/with-runtime.sh true > "$scratch/missing.log" 2>&1; then + echo 'incomplete runtime accepted' >&2; exit 1 +fi +grep -q 'missing native Oliphaunt runtime artifacts' "$scratch/missing.log" +echo 'SDK runtime staging: shipped payload, isolated paths, cleanup and failure propagation passed' diff --git a/src/third-party/postgres/fetch-source.sh b/src/third-party/postgres/fetch-source.sh index 34dabe795..24eb62506 100644 --- a/src/third-party/postgres/fetch-source.sh +++ b/src/third-party/postgres/fetch-source.sh @@ -2,7 +2,7 @@ # Shared, fail-closed transport for the pinned PostgreSQL source archive. # Keep this file POSIX-compatible: native and WASIX build scripts source it on -# both macOS and Linux. +# both macOS and Linux. Callers source tools/dev/acquisition.sh first. oliphaunt_postgresql_sha256_file() ( oliphaunt_sha_path="${1:?oliphaunt_postgresql_sha256_file requires a path}" @@ -17,6 +17,7 @@ oliphaunt_postgresql_sha256_file() ( ) oliphaunt_fetch_postgresql_source_archive() ( + oliphaunt_acquisition_start 'PostgreSQL source archive' 180 || return $? if [ "$#" -ne 4 ]; then echo "usage: oliphaunt_fetch_postgresql_source_archive DESTINATION VERSION SHA256 PRIMARY_URL" >&2 return 2 @@ -91,17 +92,12 @@ oliphaunt_fetch_postgresql_source_archive() ( for oliphaunt_url in "$oliphaunt_primary_url" "$oliphaunt_fallback_url"; do rm -f "$oliphaunt_partial" oliphaunt_curl_status=0 - if curl \ + if oliphaunt_acquisition_curl 90 5 3 curl \ --location \ --fail \ --silent \ --show-error \ - --retry 4 \ - --retry-all-errors \ - --retry-delay 3 \ - --retry-max-time 90 \ --connect-timeout 20 \ - --max-time 60 \ --max-filesize 67108864 \ --proto '=https' \ --proto-redir '=https' \ @@ -117,6 +113,7 @@ oliphaunt_fetch_postgresql_source_archive() ( echo "discarding PostgreSQL $oliphaunt_version source from $oliphaunt_url with checksum $oliphaunt_actual_sha instead of $oliphaunt_expected_sha" >&2 else oliphaunt_curl_status=$? + case "$oliphaunt_curl_status" in 126|127|129|130|137|143) return "$oliphaunt_curl_status" ;; esac echo "PostgreSQL $oliphaunt_version source download from $oliphaunt_url failed after bounded retries (curl exit $oliphaunt_curl_status)" >&2 fi done diff --git a/src/third-party/postgres/fetch-source.test.sh b/src/third-party/postgres/fetch-source.test.sh index c72e761f1..91d17eafc 100644 --- a/src/third-party/postgres/fetch-source.test.sh +++ b/src/third-party/postgres/fetch-source.test.sh @@ -2,6 +2,7 @@ set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$script_dir/../../../tools/dev/acquisition.sh" . "$script_dir/fetch-source.sh" fail() { @@ -32,12 +33,9 @@ assert_transport_flags() { for expected in \ '--location' \ '--fail' \ - '--retry 4' \ - '--retry-all-errors' \ - '--retry-delay 3' \ - '--retry-max-time 90' \ + '--retry 0' \ '--connect-timeout 20' \ - '--max-time 60' \ + '--max-time' \ '--max-filesize 67108864' \ '--proto =https' \ '--proto-redir =https' \ @@ -59,6 +57,8 @@ fake_bin="$work_root/fake-bin" mkdir -p "$fake_bin" cp "$script_dir/testdata/curl" "$fake_bin/curl" chmod 0755 "$fake_bin/curl" +printf '#!/bin/sh\nexit 0\n' > "$fake_bin/sleep" +chmod +x "$fake_bin/sleep" fake_path="$fake_bin:$PATH" primary_url="https://primary.invalid/postgresql-18.4.tar.bz2" fallback_url="https://fossies.org/linux/misc/postgresql-18.4.tar.bz2" @@ -86,7 +86,7 @@ OLIPHAUNT_FETCH_TEST_FINAL="$fallback_destination" \ PATH="$fake_path" \ oliphaunt_fetch_postgresql_source_archive "$fallback_destination" 18.4 "$fixture_sha" "$primary_url" assert_verified_file "$fallback_destination" "$fixture_sha" -[[ "$(wc -l < "$fallback_log" | tr -d ' ')" == 2 ]] || fail "transport did not try exactly the primary and fallback URLs" +[[ "$(wc -l < "$fallback_log" | tr -d ' ')" == 6 ]] || fail "transport did not try five primary attempts then the fallback URL" grep -F -- "$primary_url" "$fallback_log" >/dev/null || fail "primary URL was not attempted" grep -F -- "$fallback_url" "$fallback_log" >/dev/null || fail "fallback URL was not attempted" assert_transport_flags "$fallback_log" @@ -115,7 +115,7 @@ if OLIPHAUNT_FETCH_TEST_LOG="$failed_log" \ fail "all-failed transport unexpectedly succeeded" fi [[ ! -e "$failed_destination" ]] || fail "failed transport promoted an unverified final destination" -[[ "$(wc -l < "$failed_log" | tr -d ' ')" == 2 ]] || fail "failed transport exceeded or skipped the two bounded URL attempts" +[[ "$(wc -l < "$failed_log" | tr -d ' ')" == 10 ]] || fail "failed transport exceeded or skipped the bounded URL attempts" assert_no_partials "$work_root/failed" interrupted_destination="$work_root/interrupted/postgresql-18.4.tar.bz2" diff --git a/src/third-party/tools/fetch-sources.sh b/src/third-party/tools/fetch-sources.sh index 1c59728cd..3e1ac4184 100644 --- a/src/third-party/tools/fetch-sources.sh +++ b/src/third-party/tools/fetch-sources.sh @@ -4,6 +4,7 @@ source_tools=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) source_core="$source_tools/source-fetch-core.mts" # shellcheck source=tools/dev/curl-platform-flags.sh source "$source_tools/../../../tools/dev/curl-platform-flags.sh" +source "$source_tools/../../../tools/dev/acquisition.sh" source_git() { local seconds=$1 directory=$2 status=0 @@ -11,7 +12,7 @@ source_git() { # Bound both streams, including diagnostics, without holding them in memory. # pipefail rejects a producer killed by SIGPIPE; the byte count also catches # a producer that completed its last write before head closed the pipe. - "$source_timeout" --kill-after=5 "$seconds" git -C "$directory" \ + oliphaunt_acquisition_run "$seconds" git -C "$directory" \ -c core.fsmonitor=false -c submodule.recurse=false \ -c core.autocrlf=false -c core.eol=lf "$@" \ 2> >(head -c 16777217 > "$source_stage/git-error") | @@ -31,7 +32,7 @@ source_snapshot() { if [[ -d "$checkout" && ! -L "$checkout" && -d "$checkout/.git" && ! -L "$checkout/.git" ]]; then source_git 60 "$checkout" rev-parse --show-toplevel > "$snapshot/worktree" source_git 60 "$checkout" rev-parse --absolute-git-dir > "$snapshot/git-directory" - bun "$source_core" git-identity "$pin" "$checkout" "$snapshot" + oliphaunt_acquisition_run 900 bun "$source_core" git-identity "$pin" "$checkout" "$snapshot" source_git 60 "$checkout" status --porcelain=v1 --untracked-files=all > "$snapshot/status" # Missing pin fields make a clean checkout stale; repository errors above # remain fatal. The data validator compares the complete snapshot. @@ -46,15 +47,16 @@ source_snapshot() { fetch_source() ( set -euo pipefail local pin=$1 checkout_root=$2 archive_root=$3 mode=$4 - local name kind url mirror branch commit archive_name canonical checkout readiness fetched - local source_lock='' lock_deadline + local name kind url mirror branch commit archive_name canonical checkout readiness fetched status + local source_lock='' + oliphaunt_acquisition_start "source $(basename "$pin")" 900 mkdir -p "$checkout_root" "$archive_root" source_stage=$(mktemp -d "$checkout_root/.source-stage-XXXXXX") trap 'rm -rf "$source_stage"; if [[ -n "$source_lock" ]]; then rmdir "$source_lock"; fi' EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM - bun "$source_core" fields "$pin" > "$source_stage/fields" + oliphaunt_acquisition_run 900 bun "$source_core" fields "$pin" > "$source_stage/fields" { IFS= read -r -d '' name; IFS= read -r -d '' kind; IFS= read -r -d '' url IFS= read -r -d '' mirror; IFS= read -r -d '' branch; IFS= read -r -d '' commit IFS= read -r -d '' archive_name; IFS= read -r -d '' canonical @@ -62,11 +64,8 @@ fetch_source() ( checkout="$checkout_root/$name" # Scopes overlap (notably ICU data). Serialize inspection through promotion, # so a waiter reuses the complete checkout instead of replacing it concurrently. - lock_deadline=$((SECONDS + 3600)) until mkdir "$checkout.lock" 2>/dev/null; do - if (( SECONDS >= lock_deadline )); then - echo "timed out waiting for source checkout lock: $checkout.lock" >&2; exit 1 - fi + oliphaunt_acquisition_remaining 900 >/dev/null || exit $? sleep 0.1 done source_lock="$checkout.lock" @@ -77,11 +76,8 @@ fetch_source() ( : > "$source_stage/empty.gitconfig" export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL="$source_stage/empty.gitconfig" export GIT_TERMINAL_PROMPT=0 GCM_INTERACTIVE=Never - source_timeout=$(command -v timeout || command -v gtimeout) || { - echo 'source fetching requires GNU timeout (brew install coreutils on macOS)' >&2; exit 1; - } source_snapshot "$checkout" "$source_stage/durable" - readiness=$(bun "$source_core" inspect "$pin" "$checkout" "$source_stage/durable") + readiness=$(oliphaunt_acquisition_run 900 bun "$source_core" inspect "$pin" "$checkout" "$source_stage/durable") if [[ "$readiness" == ready ]]; then exit 0; fi if [[ "$mode" == verify ]]; then echo "source checkout $checkout is missing or stale" >&2; exit 1; fi @@ -98,10 +94,14 @@ fetch_source() ( if source_git 300 "$candidate" \ -c protocol.allow=never -c protocol.https.allow=always -c credential.helper= \ -c http.followRedirects=false -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=120 \ - fetch --no-tags --depth=1 "$transport" "$commit"; then success=true; break; fi + fetch --no-tags --depth=1 "$transport" "$commit"; then success=true; break + else + status=$? + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac + fi echo "fetch $name from $transport failed on attempt $attempt/5" >&2 if (( attempt < 5 && attempt % ${#transports[@]} == 0 )); then - sleep "$((attempt * 5 / ${#transports[@]}))" + oliphaunt_acquisition_sleep "$((attempt * 5 / ${#transports[@]}))" || exit $? fi done "$success" || exit 1 @@ -111,10 +111,10 @@ fetch_source() ( fi source_git 60 "$candidate" checkout --quiet -B "$branch" "$commit" source_snapshot "$candidate" "$source_stage/candidate" - bun "$source_core" git-candidate "$pin" "$candidate" "$source_stage/candidate" + oliphaunt_acquisition_run 900 bun "$source_core" git-candidate "$pin" "$candidate" "$source_stage/candidate" else local archive="$archive_root/$archive_name" download="$source_stage/$archive_name" - if [[ "$(bun "$source_core" archive-valid "$pin" "$archive")" != valid ]]; then + if [[ "$(oliphaunt_acquisition_run 900 bun "$source_core" archive-valid "$pin" "$archive")" != valid ]]; then local urls=() endpoint success=false if [[ -n "$canonical" ]]; then urls+=("$canonical"); fi urls+=("$url") @@ -122,26 +122,31 @@ fetch_source() ( local tls_flag tls_flag=$(oliphaunt_curl_platform_tls_flag) for endpoint in "${urls[@]}"; do - if "$source_timeout" --kill-after=5 620 curl --disable --fail --location --silent --show-error \ - --retry 2 --retry-all-errors --retry-connrefused --retry-delay 5 --retry-max-time 600 \ - --connect-timeout 20 --max-time 600 --speed-limit 1024 --speed-time 120 \ + if oliphaunt_acquisition_curl 300 3 5 curl --fail --location --silent --show-error \ + --connect-timeout 20 --speed-limit 1024 --speed-time 120 \ --max-filesize 1073741824 --max-redirs 5 --proto-default https \ --proto '=https' --proto-redir '=https' --tlsv1.2 ${tls_flag:+"$tls_flag"} \ - --remove-on-error --url "$endpoint" --output "$download"; then success=true; break; fi + --remove-on-error --url "$endpoint" --output "$download"; then success=true; break + else + status=$? + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac + fi echo "download $name from $endpoint failed" >&2 rm -f "$download" done "$success" || exit 1 # An invalid candidate never replaces even a corrupt existing cache. - [[ "$(bun "$source_core" archive-valid "$pin" "$download")" == valid ]] || exit 1 + [[ "$(oliphaunt_acquisition_run 900 bun "$source_core" archive-valid "$pin" "$download")" == valid ]] || exit 1 + oliphaunt_acquisition_remaining 900 >/dev/null || exit $? bun "$source_core" promote "$download" "$archive" fi - bun "$source_core" unpack "$pin" "$archive" "$candidate" + oliphaunt_acquisition_run 900 bun "$source_core" unpack "$pin" "$archive" "$candidate" fi # Reinspect immediately before replacing an existing checkout, including a # source that changed kind. The promotion helper restores a prior tree on error. source_snapshot "$checkout" "$source_stage/durable" - bun "$source_core" inspect "$pin" "$checkout" "$source_stage/durable" > /dev/null + oliphaunt_acquisition_run 900 bun "$source_core" inspect "$pin" "$checkout" "$source_stage/durable" > /dev/null + oliphaunt_acquisition_remaining 900 >/dev/null || exit $? bun "$source_core" promote "$candidate" "$checkout" ) @@ -153,6 +158,7 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then bun "$source_tools/fetch-sources.mts" plan "$source_work" "$@" mode=$(cat "$source_work/mode") if [[ "$mode" != skip ]]; then + oliphaunt_acquisition_start 'source scope' 1800 while IFS= read -r -d '' pin; do fetch_source "$pin" "$PWD/target/oliphaunt-sources/checkouts" "$PWD/target/oliphaunt-sources/archives" "$mode" # Bash 3.2 does not propagate a failed subshell function through this loop. diff --git a/src/third-party/tools/moon.yml b/src/third-party/tools/moon.yml index 8c4c818cd..003065c9f 100644 --- a/src/third-party/tools/moon.yml +++ b/src/third-party/tools/moon.yml @@ -23,6 +23,7 @@ tasks: - "**/*" - "!**/*.test.*" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - "@group(release-archive-contract)" options: cache: false @@ -34,6 +35,7 @@ tasks: script: "set -e\nbash src/third-party/tools/fetch-sources.sh production-all --validate-only\nbash src/third-party/tools/source-fetch-core.test.sh\nbash tools/dev/bun.sh test ./src/third-party/tools/source-fetch-scopes.test.mts\nbash src/third-party/postgres/fetch-source.test.sh\n" inputs: - "/tools/dev/bun.sh" + - /tools/dev/acquisition.sh - /src/third-party/postgres/**/* - "/src/third-party/{icu,openssl}/source.toml" - /src/native/runtime/sources/*.toml @@ -64,6 +66,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - /src/extensions/tools/extension-upstream-licenses.mts - "@group(release-archive-contract)" options: @@ -83,6 +86,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - "@group(release-archive-contract)" options: cache: false @@ -103,6 +107,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - "@group(release-archive-contract)" options: cache: false @@ -121,6 +126,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh options: cache: false internal: true @@ -138,6 +144,7 @@ tasks: - "!**/*.test.*" - "!verify-source-tree.mts" - /tools/dev/curl-platform-flags.sh + - /tools/dev/acquisition.sh - /src/extensions/tools/extension-upstream-licenses.mts - "@group(release-archive-contract)" options: diff --git a/src/third-party/tools/source-fetch-core.mts b/src/third-party/tools/source-fetch-core.mts index f0323f0dc..e1f7e894d 100644 --- a/src/third-party/tools/source-fetch-core.mts +++ b/src/third-party/tools/source-fetch-core.mts @@ -297,10 +297,11 @@ export function validateSource(source) { if ( !parsedUrl.pathname.endsWith('.tar.gz') && !parsedUrl.pathname.endsWith('.tgz') && + !parsedUrl.pathname.endsWith('.crate') && !parsedUrl.pathname.endsWith('.zip') ) { throw new Error( - `archive source '${source.name}' URL must identify a .tar.gz, .tgz, or .zip file`, + `archive source '${source.name}' URL must identify a .tar.gz, .tgz, .crate, or .zip file`, ); } if (source.stripPrefix === '.' && !parsedUrl.pathname.endsWith('.zip')) { diff --git a/src/third-party/tools/source-fetch-core.test.mts b/src/third-party/tools/source-fetch-core.test.mts index d5682199b..2fd90be12 100644 --- a/src/third-party/tools/source-fetch-core.test.mts +++ b/src/third-party/tools/source-fetch-core.test.mts @@ -271,6 +271,12 @@ test('source pins reject unsafe URLs, branches, names, and unpinned archives', ( stripPrefix: 'pkg', }; assert.doesNotThrow(() => validateSource(archive)); + assert.doesNotThrow(() => + validateSource({ + ...archive, + url: 'https://static.crates.io/crates/wasmer/wasmer-6.1.0.crate', + }), + ); assert.throws(() => validateSource({ ...archive, mirrorUrl: archive.url })); for (const url of [ 'https://example.invalid/libiconv/libiconv-1.19.tar.gz', diff --git a/src/third-party/tools/source-fetch-core.test.sh b/src/third-party/tools/source-fetch-core.test.sh index b8d378e95..081dd1fa2 100644 --- a/src/third-party/tools/source-fetch-core.test.sh +++ b/src/third-party/tools/source-fetch-core.test.sh @@ -7,6 +7,7 @@ trap 'rm -rf "$scratch"' EXIT test_data="$tools/source-fetch-core.test.mts" archive_tool="$tools/source-archive.mts" export FETCH_TEST_GIT="$(command -v git)" +export FETCH_TEST_DATE="$(command -v date)" export FETCH_TEST_SLEEP="$(command -v sleep)" base_path="$PATH" for scope in icu native-runtime wasix-runtime wasix-postmaster-runtime production-all; do @@ -22,7 +23,7 @@ done bash "$root_dir/tools/dev/bun.sh" test "$test_data" mkdir "$scratch/fixtures" "$scratch/bin" bun "$test_data" prepare "$scratch/fixtures" -for name in git curl sleep; do +for name in git curl sleep date; do cp "$tools/source-fetch-transport.test.sh" "$scratch/bin/$name" chmod +x "$scratch/bin/$name" done @@ -110,7 +111,7 @@ fail_command 'archive sha256: expected' fetch "$root" "$fixtures/valid.json" : > "$root/requests" export FETCH_TEST_ARCHIVE="$fixtures/valid.tar.gz" RUNNER_OS=Windows fetch "$root" "$fixtures/valid.json" -printf '%s\n' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' > "$scratch/expected" +printf '%s\n' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' > "$scratch/expected" cmp "$scratch/expected" "$root/requests" cmp "$cache" "$fixtures/valid.tar.gz" cmp "$scratch/trusted" "$checkout/file.txt" @@ -138,6 +139,10 @@ FETCH_TEST_FAULT=gnu FETCH_TEST_ARCHIVE="$fixtures/updated.tar.gz" \ : > "$root/requests" FETCH_TEST_FAULT=gnu FETCH_TEST_ARCHIVE="$fixtures/valid.tar.gz" fetch "$root" "$root/pin.json" printf '%s\n' 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' \ + 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' \ + 'https://ftp.gnu.org/gnu/libiconv/libiconv-1.19.tar.gz' \ + 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' \ + 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' \ 'https://ftpmirror.gnu.org/libiconv/libiconv-1.19.tar.gz' \ 'https://mirror.example.invalid/libiconv.tar.gz' > "$scratch/expected" cmp "$scratch/expected" "$root/requests" @@ -255,3 +260,35 @@ ln -s "$root/upstream/.git" "$checkout/.git" bun "$test_data" git-pin "$root" "$commit" fail_command 'unsupported non-directory \.git metadata' fetch "$root" "$root/pin.json" echo 'Source fetch: verified archives, fallback/retries, exact Git pins, LF repair, symlink containment and local-edit preservation passed' + +# Retries and mirrors share the pin's deadline; expiry preserves existing data +# and releases the checkout lock, including when a transport returns failure. +root="$scratch/archive-deadline" +mkdir -p "$root/archives" +printf '1000000\n' > "$root/clock" +cache="$root/archives/libiconv-$sha.tar.gz" +printf 'prior cache' > "$cache" +FETCH_TEST_EXPIRE=1 FETCH_TEST_CLOCK="$root/clock" fail_command 'deadline' fetch "$root" "$fixtures/valid.json" +[[ "$(wc -l < "$root/requests")" -eq 1 ]] +[[ "$(cat "$cache")" == 'prior cache' ]] +[[ -z "$(ls -A "$root/checkouts")" ]] +root="$scratch/git-deadline" +init_repo "$root/upstream" 'new bytes' upstream +commit="$(git -C "$root/upstream" rev-parse HEAD)" +bun "$test_data" git-pin "$root" "$commit" +init_repo "$root/checkouts/source" prior +prior="$(git -C "$root/checkouts/source" rev-parse HEAD)" +printf '1000000\n' > "$root/clock" +FETCH_TEST_EXPIRE=1 FETCH_TEST_CLOCK="$root/clock" fail_command 'deadline' fetch "$root" "$root/pin.json" +[[ "$(wc -l < "$root/requests")" -eq 1 ]] +[[ "$(git -C "$root/checkouts/source" rev-parse HEAD)" == "$prior" ]] +[[ "$(ls -A "$root/checkouts")" == source ]] +echo 'Source acquisition expiry preserves prior data, stops retries and releases locks' + +# A waiting caller must neither replenish its budget nor remove another owner's lock. +root="$scratch/lock-deadline" +mkdir -p "$root/checkouts/libiconv.lock" +printf '1000000\n' > "$root/clock" +FETCH_TEST_EXPIRE=1 FETCH_TEST_CLOCK="$root/clock" fail_command 'deadline' fetch "$root" "$fixtures/valid.json" +[[ ! -s "$root/requests" ]] +[[ "$(ls -A "$root/checkouts")" == libiconv.lock ]] diff --git a/src/third-party/tools/source-fetch-transport.test.sh b/src/third-party/tools/source-fetch-transport.test.sh index 8355d8bbc..3458f86de 100644 --- a/src/third-party/tools/source-fetch-transport.test.sh +++ b/src/third-party/tools/source-fetch-transport.test.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash set -euo pipefail case "${0##*/}" in + date) + if [[ -n ${FETCH_TEST_CLOCK:-} ]]; then cat "$FETCH_TEST_CLOCK"; else exec "$FETCH_TEST_DATE" "$@"; fi + ;; sleep) printf '%s\n' "$1" >> "$FETCH_TEST_ROOT/sleeps" if [[ "$1" == 0.1 ]]; then touch "$FETCH_TEST_ROOT/lock-waiting" + if [[ ${FETCH_TEST_EXPIRE:-} == 1 ]]; then printf "9999999\n" > "$FETCH_TEST_CLOCK"; exit 0; fi exec "$FETCH_TEST_SLEEP" "$1" fi ;; @@ -14,6 +18,7 @@ case "${0##*/}" in count=${#args[@]} url=${args[$((count-2))]} printf '%s\n' "$url" >> "$FETCH_TEST_ROOT/requests" + if [[ ${FETCH_TEST_EXPIRE:-} == 1 ]]; then printf "9999999\n" > "$FETCH_TEST_CLOCK"; exit 28; fi if [[ ${FETCH_TEST_FAULT:-} == all || (${FETCH_TEST_FAULT:-} == primary && "$url" != https://mirror.example.invalid/source.git) ]]; then echo "transport fault: $url" >&2; exit 1 fi @@ -27,7 +32,7 @@ case "${0##*/}" in curl) [[ "$1" == --disable ]] case " $* " in *' --insecure '*|*' -k '*) exit 90 ;; esac - for required in '--proto =https' '--proto-redir =https' '--max-filesize 1073741824' '--max-time 600' '--tlsv1.2' '--retry 2'; do + for required in '--proto =https' '--proto-redir =https' '--max-filesize 1073741824' '--retry 0' '--tlsv1.2'; do [[ " $* " == *" $required "* ]] || exit 91 done if [[ ${RUNNER_OS:-} == Windows ]]; then [[ " $* " == *' --ssl-revoke-best-effort '* ]]; fi @@ -37,6 +42,7 @@ case "${0##*/}" in shift done printf '%s\n' "$url" >> "$FETCH_TEST_ROOT/requests" + if [[ ${FETCH_TEST_EXPIRE:-} == 1 ]]; then printf "9999999\n" > "$FETCH_TEST_CLOCK"; exit 28; fi if [[ ${FETCH_TEST_BARRIER:-} == 1 ]]; then touch "$FETCH_TEST_ROOT/downloading" while [[ ! -e "$FETCH_TEST_ROOT/release-download" ]]; do "$FETCH_TEST_SLEEP" 0.1; done diff --git a/src/wasix/browser-host/build-provenance.mts b/src/wasix/browser-host/build-provenance.mts index dd34ae0d9..ee54b2b9e 100644 --- a/src/wasix/browser-host/build-provenance.mts +++ b/src/wasix/browser-host/build-provenance.mts @@ -8,7 +8,7 @@ const repositoryRoot = resolve(hostDirectory, '../../..'); const sourceManifestPath = 'src/wasix/browser-host/source.toml'; const buildScriptPath = 'src/wasix/browser-host/build-sdk.sh'; const provenanceScriptPath = 'src/wasix/browser-host/build-provenance.mts'; -const safePatchName = /^\d{4}-wasmer-(?:(?:js|wasix)-)?[a-z0-9-]+\.patch$/u; +const safePatchName = /^\d{4}-(?:wasmer-(?:(?:js|wasix)-)?|virtual-(?:fs|mio)-)[a-z0-9-]+\.patch$/u; export async function loadHostBuildContract() { const source = await readFile(resolve(repositoryRoot, sourceManifestPath), 'utf8'); @@ -27,6 +27,11 @@ export async function loadHostBuildContract() { ...patchSeries.map((patch) => `src/wasix/browser-host/patches/${patch}`), buildScriptPath, provenanceScriptPath, + 'tools/dev/curl-platform-flags.sh', + 'tools/dev/acquisition.sh', + 'src/third-party/tools/fetch-sources.sh', + 'src/third-party/tools/source-fetch-core.mts', + 'src/third-party/tools/source-archive.mts', ]); const digests = []; for (const input of inputs) { diff --git a/src/wasix/browser-host/build-sdk.sh b/src/wasix/browser-host/build-sdk.sh index 0db80d0c0..f83c13bce 100755 --- a/src/wasix/browser-host/build-sdk.sh +++ b/src/wasix/browser-host/build-sdk.sh @@ -27,17 +27,13 @@ toml_value() { ' "$source_manifest" } -wasmer_js_url="$(toml_value wasmer-js url)" wasmer_js_version="$(toml_value wasmer-js version)" -wasmer_js_commit="$(toml_value wasmer-js commit)" -wasmer_wasix_url="$(toml_value wasmer-wasix url)" wasmer_wasix_version="$(toml_value wasmer-wasix version)" -wasmer_wasix_sha256="$(toml_value wasmer-wasix sha256)" -wasmer_url="$(toml_value wasmer url)" wasmer_version="$(toml_value wasmer version)" -wasmer_sha256="$(toml_value wasmer sha256)" +virtual_fs_version="$(toml_value virtual-fs version)" +virtual_mio_version="$(toml_value virtual-mio version)" -for value in "$wasmer_js_url" "$wasmer_js_version" "$wasmer_js_commit" "$wasmer_wasix_url" "$wasmer_wasix_version" "$wasmer_wasix_sha256" "$wasmer_url" "$wasmer_version" "$wasmer_sha256"; do +for value in "$wasmer_js_version" "$wasmer_wasix_version" "$wasmer_version" "$virtual_fs_version" "$virtual_mio_version"; do if [[ -z "$value" ]]; then echo "wasix-ts host build: malformed $source_manifest" >&2 exit 1 @@ -48,17 +44,13 @@ if ! command -v bun >/dev/null 2>&1; then echo "wasix-ts host build: required command not found: bun" >&2 exit 1 fi -mapfile -t patch_series < <(bun "$provenance_script" --patch-series) +patch_series="$(bun "$provenance_script" --patch-series)" input_hash="$(bun "$provenance_script" --inputs-sha256)" patch_command="patch" -sha256sum_command="sha256sum" if command -v gpatch >/dev/null 2>&1; then patch_command="gpatch" fi -if command -v gsha256sum >/dev/null 2>&1; then - sha256sum_command="gsha256sum" -fi if [[ -f "$target_dir/.oliphaunt-input-sha256" ]] \ && [[ "$(<"$target_dir/.oliphaunt-input-sha256")" == "$input_hash" ]] \ @@ -71,7 +63,7 @@ fi mkdir -p "$target_parent" -for command_name in awk bun curl git node npm "$patch_command" "$sha256sum_command" tar wasm-pack; do +for command_name in awk bun curl git node npm "$patch_command" wasm-pack; do if ! command -v "$command_name" >/dev/null 2>&1; then echo "wasix-ts host build: required command not found: $command_name" >&2 exit 1 @@ -89,38 +81,38 @@ cleanup() { trap cleanup EXIT wasmer_js_dir="$build_root/wasmer-js" -wasmer_wasix_archive="$build_root/wasmer-wasix.crate" wasmer_wasix_dir="$build_root/wasmer-wasix-$wasmer_wasix_version" -wasmer_archive="$build_root/wasmer.crate" wasmer_dir="$build_root/wasmer-$wasmer_version" - -git init --quiet "$wasmer_js_dir" -git -C "$wasmer_js_dir" remote add origin "$wasmer_js_url" -git -C "$wasmer_js_dir" fetch --quiet --depth 1 origin "$wasmer_js_commit" -git -C "$wasmer_js_dir" checkout --quiet --detach FETCH_HEAD -if [[ "$(git -C "$wasmer_js_dir" rev-parse HEAD)" != "$wasmer_js_commit" ]]; then - echo "wasix-ts host build: Wasmer JS checkout did not resolve the pinned commit" >&2 - exit 1 -fi +virtual_fs_dir="$build_root/virtual-fs-$virtual_fs_version" +virtual_mio_dir="$build_root/virtual-mio-$virtual_mio_version" + +# Use the same bounded transports, exact pins and safe extraction as the other +# producers. Only temporary source trees are patched; verified archives persist. +source "$repo_root/src/third-party/tools/fetch-sources.sh" +bun - "$source_manifest" "$build_root" <<'JS' +import {readFileSync, writeFileSync} from 'node:fs'; +const pins = Bun.TOML.parse(readFileSync(process.argv[2], 'utf8')); +for (const name of ['wasmer-js', 'wasmer-wasix', 'wasmer', 'virtual-fs', 'virtual-mio']) { + const pin = pins[name]; + const source = name === 'wasmer-js' + ? {name, kind:'git', url:pin.url, branch:'oliphaunt-pinned', commit:pin.commit} + : {name:`${name}-${pin.version}`, kind:'archive', url:pin.url, branch:pin.version, + commit:pin.sha256, sha256:pin.sha256, stripPrefix:`${name}-${pin.version}`}; + writeFileSync(`${process.argv[3]}/${name}.json`, JSON.stringify(source)); +} +JS +oliphaunt_acquisition_start 'browser host sources' 1800 +for source_name in wasmer-js wasmer-wasix wasmer virtual-fs virtual-mio; do + fetch_source "$build_root/$source_name.json" "$build_root" "$target_parent/archives" fetch +done actual_wasmer_js_version="$(bun "$repo_root/tools/dev/node-info.mts" package-version "$wasmer_js_dir/package.json")" if [[ "$actual_wasmer_js_version" != "$wasmer_js_version" ]]; then echo "wasix-ts host build: pinned Wasmer JS version is $actual_wasmer_js_version, expected $wasmer_js_version" >&2 exit 1 fi -curl --fail --location --silent --show-error \ - --user-agent "oliphaunt-wasix-ts-source-build/0.0.0" \ - "$wasmer_wasix_url" --output "$wasmer_wasix_archive" -echo "$wasmer_wasix_sha256 $wasmer_wasix_archive" | "$sha256sum_command" --check --status -tar -xzf "$wasmer_wasix_archive" -C "$build_root" - -curl --fail --location --silent --show-error \ - --user-agent "oliphaunt-wasix-ts-source-build/0.0.0" \ - "$wasmer_url" --output "$wasmer_archive" -echo "$wasmer_sha256 $wasmer_archive" | "$sha256sum_command" --check --status -tar -xzf "$wasmer_archive" -C "$build_root" - -for patch_name in "${patch_series[@]}"; do +while IFS= read -r patch_name; do + [[ -n "$patch_name" ]] || continue patch_file="$host_dir/patches/$patch_name" case "$patch_name" in ????-wasmer-js-*.patch) @@ -132,13 +124,19 @@ for patch_name in "${patch_series[@]}"; do ????-wasmer-*.patch) patch_dir="$wasmer_dir" ;; + ????-virtual-fs-*.patch) + patch_dir="$virtual_fs_dir" + ;; + ????-virtual-mio-*.patch) + patch_dir="$virtual_mio_dir" + ;; *) echo "wasix-ts host build: patch target is not declared by its canonical name: $patch_name" >&2 exit 1 ;; esac "$patch_command" --batch --forward -d "$patch_dir" -p1 < "$patch_file" -done +done <<< "$patch_series" # The pinned source commit's npm lock predates its package metadata. Patch only # the missing root metadata and dependencies, then install the integrity-pinned diff --git a/src/wasix/browser-host/moon.yml b/src/wasix/browser-host/moon.yml index 619315ff2..95d5ed198 100644 --- a/src/wasix/browser-host/moon.yml +++ b/src/wasix/browser-host/moon.yml @@ -14,6 +14,9 @@ tasks: inputs: - "**/*" - "/tools/dev/node-info.mts" + - "/tools/dev/curl-platform-flags.sh" + - "/tools/dev/acquisition.sh" + - "/src/third-party/tools/{fetch-sources.sh,source-fetch-core.mts,source-archive.mts}" - "@group(cargo-workspace)" outputs: - "/target/oliphaunt-wasix-ts/host/wasmer-sdk/**/*" diff --git a/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch b/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch index cdae8598d..1416e7e80 100644 --- a/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch +++ b/src/wasix/browser-host/patches/0001-wasmer-js-run-configured-wasix-process.patch @@ -11,7 +11,7 @@ index b21b6eb..4a26370 100644 [package.metadata.wasm-pack.profile.release.wasm-bindgen] debug-js-glue = false -@@ -135,9 +135,11 @@ demangle-name-section = false +@@ -135,9 +135,13 @@ demangle-name-section = false dwarf-debug-info = false [package.metadata.wasm-pack.profile.release] @@ -21,6 +21,8 @@ index b21b6eb..4a26370 100644 [patch.crates-io] +wasmer = { path = "../wasmer-6.1.0" } +wasmer-wasix = { path = "../wasmer-wasix-0.601.0" } ++virtual-fs = { path = "../virtual-fs-0.601.0" } ++virtual-mio = { path = "../virtual-mio-0.601.0" } #webc = {git = "https://github.com/wasmerio/pirita", branch = "in-memory-manifest"} #virtual-net = { git = "https://github.com/wasmerio/wasmer", branch = "master" } #virtual-fs = { git = "https://github.com/wasmerio/wasmer", branch = "master" } diff --git a/src/wasix/browser-host/patches/0036-virtual-fs-poll-contended-pipe-reads.patch b/src/wasix/browser-host/patches/0036-virtual-fs-poll-contended-pipe-reads.patch new file mode 100644 index 000000000..7b277fd51 --- /dev/null +++ b/src/wasix/browser-host/patches/0036-virtual-fs-poll-contended-pipe-reads.patch @@ -0,0 +1,109 @@ +diff --git a/src/pipe.rs b/src/pipe.rs +--- a/src/pipe.rs ++++ b/src/pipe.rs +@@ -46,6 +46,23 @@ + } + + impl PipeRx { ++ fn poll_receiver<'a>( ++ rx: &'a Mutex, ++ cx: &mut Context<'_>, ++ ) -> Poll> { ++ match rx.try_lock() { ++ Ok(guard) => Poll::Ready(guard), ++ Err(std::sync::TryLockError::WouldBlock) => { ++ // A writer can briefly hold this lock while notifying its interest ++ // handler. Polling must not block the browser's main thread. The ++ // synchronous lock has no waker registration, so poll again. ++ cx.waker().wake_by_ref(); ++ Poll::Pending ++ } ++ Err(error) => panic!("{error}"), ++ } ++ } ++ + // Tries to read from the internal buffer if data is available. + fn try_read_from_buffer( + rx: &mut MutexGuard<'_, PipeReceiver>, +@@ -108,7 +125,7 @@ + ))); + }; + +- let mut rx = rx.lock().unwrap(); ++ let mut rx = std::task::ready!(Self::poll_receiver(rx, cx)); + loop { + { + if let Some(inner_buf) = rx.buffer.as_mut() { +@@ -460,7 +477,7 @@ + ))); + }; + +- let mut rx = rx.lock().unwrap(); ++ let mut rx = std::task::ready!(Self::poll_receiver(rx, cx)); + loop { + if Self::try_read_from_buffer(&mut rx, buf.remaining(), |read_buf| { + buf.put_slice(read_buf); +@@ -601,3 +618,63 @@ + /// Shared version of BidiPipe for situations where you need + /// to emulate the old behaviour of `Pipe` (both send and recv on one channel). + pub type WasiBidirectionalSharedPipePair = ArcFile; ++ ++#[cfg(test)] ++mod contention_tests { ++ use super::*; ++ use std::sync::atomic::{AtomicUsize, Ordering}; ++ use std::task::{Wake, Waker}; ++ ++ #[derive(Default)] ++ struct WakeCount(AtomicUsize); ++ ++ impl Wake for WakeCount { ++ fn wake(self: Arc) { ++ self.0.fetch_add(1, Ordering::SeqCst); ++ } ++ } ++ ++ #[test] ++ fn contended_pipe_reads_yield_and_preserve_data() { ++ let (mut tx, mut rx) = Pipe::new().split(); ++ std::io::Write::write_all(&mut tx, b"abc").unwrap(); ++ let shared = rx.rx.as_ref().unwrap().clone(); ++ let wake_count = Arc::new(WakeCount::default()); ++ let waker = Waker::from(wake_count.clone()); ++ let mut cx = Context::from_waker(&waker); ++ let mut bytes = [0; 2]; ++ let mut buf = tokio::io::ReadBuf::new(&mut bytes); ++ ++ // Deterministic contention: neither poll method may wait for this guard. ++ let guard = shared.lock().unwrap(); ++ assert!(Pin::new(&mut rx).poll_read_ready(&mut cx).is_pending()); ++ assert!(Pin::new(&mut rx).poll_read(&mut cx, &mut buf).is_pending()); ++ assert_eq!(wake_count.0.load(Ordering::SeqCst), 2); ++ assert!(buf.filled().is_empty()); ++ drop(guard); ++ ++ assert!(matches!( ++ Pin::new(&mut rx).poll_read_ready(&mut cx), ++ Poll::Ready(Ok(3)) ++ )); ++ assert!(matches!( ++ Pin::new(&mut rx).poll_read(&mut cx, &mut buf), ++ Poll::Ready(Ok(())) ++ )); ++ assert_eq!(buf.filled(), b"ab"); ++ buf.clear(); ++ assert!(matches!( ++ Pin::new(&mut rx).poll_read(&mut cx, &mut buf), ++ Poll::Ready(Ok(())) ++ )); ++ assert_eq!(buf.filled(), b"c"); ++ buf.clear(); ++ assert!(Pin::new(&mut rx).poll_read(&mut cx, &mut buf).is_pending()); ++ tx.close(); ++ assert!(matches!( ++ Pin::new(&mut rx).poll_read(&mut cx, &mut buf), ++ Poll::Ready(Ok(())) ++ )); ++ assert!(buf.filled().is_empty()); ++ } ++} diff --git a/src/wasix/browser-host/patches/0037-virtual-mio-retain-inline-wakes.patch b/src/wasix/browser-host/patches/0037-virtual-mio-retain-inline-wakes.patch new file mode 100644 index 000000000..f9133889b --- /dev/null +++ b/src/wasix/browser-host/patches/0037-virtual-mio-retain-inline-wakes.patch @@ -0,0 +1,88 @@ +diff --git a/src/waker.rs b/src/waker.rs +--- a/src/waker.rs ++++ b/src/waker.rs +@@ -1,31 +1,29 @@ + #![allow(dead_code, unused)] + use std::{ +- sync::{Arc, Condvar, Mutex}, ++ sync::{atomic::{AtomicBool, Ordering}, Arc}, ++ thread::{self, Thread}, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, + }; + + use futures::Future; + + pub struct InlineWaker { +- lock: Mutex<()>, +- condvar: Condvar, ++ thread: Thread, ++ notified: AtomicBool, + } + impl InlineWaker { + pub fn new() -> Arc { + Arc::new(Self { +- lock: Mutex::new(()), +- condvar: Condvar::new(), ++ thread: thread::current(), ++ notified: AtomicBool::new(false), + }) + } + + fn wake_now(&self) { +- // Note: This guard should be there to prevent race conditions however in the +- // browser it causes a lock up - some strange browser issue. What I suspect +- // is that the Condvar::wait call is not releasing the mutex lock +- #[cfg(not(feature = "js"))] +- let _guard = self.lock.lock().unwrap(); +- +- self.condvar.notify_all(); ++ // Retain a wake delivered during poll, before the worker parks. ++ // No synchronous mutex may block the browser thread delivering a wake. ++ self.notified.store(true, Ordering::Release); ++ self.thread.unpark(); + } + + pub fn as_waker(self: &Arc) -> Waker { +@@ -43,10 +41,11 @@ + // We loop waiting for the waker to be woken, then we poll again + let mut task = Box::pin(task); + loop { +- let lock = inline_waker.lock.lock().unwrap(); + match task.as_mut().poll(&mut cx) { + Poll::Pending => { +- inline_waker.condvar.wait(lock).ok(); ++ while !inline_waker.notified.swap(false, Ordering::Acquire) { ++ thread::park(); ++ } + } + Poll::Ready(ret) => { + return ret; +@@ -75,3 +74,28 @@ + |s| drop(Arc::from_raw(s as *const InlineWaker)), // decrease refcount + ) + }; ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use std::{future::poll_fn, sync::mpsc, time::Duration}; ++ ++ #[test] ++ fn wake_during_poll_is_retained_until_the_next_poll() { ++ let (tx, rx) = mpsc::channel(); ++ thread::spawn(move || { ++ let mut polled = false; ++ InlineWaker::block_on(poll_fn(|cx| { ++ if polled { ++ Poll::Ready(()) ++ } else { ++ polled = true; ++ cx.waker().wake_by_ref(); ++ Poll::Pending ++ } ++ })); ++ tx.send(()).unwrap(); ++ }); ++ rx.recv_timeout(Duration::from_secs(2)).unwrap(); ++ } ++} diff --git a/src/wasix/browser-host/source.toml b/src/wasix/browser-host/source.toml index f2786467f..abb31cc86 100644 --- a/src/wasix/browser-host/source.toml +++ b/src/wasix/browser-host/source.toml @@ -21,6 +21,18 @@ version = "6.1.0" url = "https://static.crates.io/crates/wasmer/wasmer-6.1.0.crate" sha256 = "2d85671948f8886a1cc946141c0b688a5617603c103699a5fceeebeb4e75b0b6" +[virtual-fs] +name = "virtual-fs" +version = "0.601.0" +url = "https://static.crates.io/crates/virtual-fs/virtual-fs-0.601.0.crate" +sha256 = "8defc513ce70576bdbb4305ee67c0cba647af18c0995de8f1fa1271b724c9859" + +[virtual-mio] +name = "virtual-mio" +version = "0.601.0" +url = "https://static.crates.io/crates/virtual-mio/virtual-mio-0.601.0.crate" +sha256 = "a14806a3f25b70315764760c376d9f68937f5467013891bc7497a33d24105097" + [patches] series = [ "0001-wasmer-js-run-configured-wasix-process.patch", @@ -50,4 +62,6 @@ series = [ # before its dependent first-poll fast path. "0035-wasmer-wasix-preserve-shared-seek-offset.patch", "0033-wasmer-wasix-poll-ready-asyncify-work.patch", + "0036-virtual-fs-poll-contended-pipe-reads.patch", + "0037-virtual-mio-retain-inline-wakes.patch", ] diff --git a/src/wasix/postgres-tools/moon.yml b/src/wasix/postgres-tools/moon.yml index 8299ee666..2d906a6e5 100644 --- a/src/wasix/postgres-tools/moon.yml +++ b/src/wasix/postgres-tools/moon.yml @@ -66,13 +66,13 @@ tasks: test-consumer: tags: ["integration", "consumer", "ci-wasix-ts-sdk-package"] command: "bash tools/ci/with-projects.sh --exec bash src/wasix/postgres-tools/ts/tests/consumer.sh" - deps: ["oliphaunt-wasix-tools-ts:package", "oliphaunt-wasix-ts:package", "oliphaunt-wasix-napi:build-release-assets", "postgres-tools-wasix:package-portable", "postgres-tools-wasix:package-aot"] + deps: ["oliphaunt-query-ts:build", "oliphaunt-wasix-tools-ts:package", {target: "oliphaunt-wasix-ts:package", cacheStrategy: ignored}, "oliphaunt-wasix-ts:build", "wasix-browser-host:build", "oliphaunt-wasix-napi:build-release-assets", "postgres-tools-wasix:package-portable", "postgres-tools-wasix:package-aot"] inputs: ["ts/tests/**/*", "/src/wasix/sdks/ts/tools/integration/packed-node-fixture.mts", "/src/wasix/sdks/ts/tools/pgwire-client.mts", "/src/test-fixtures/postgres/**/*", "tools/*.mts"] options: {cache: false, runFromWorkspaceRoot: true} test-browser: tags: ["integration", "browser", "ci-wasix-ts-sdk-package"] command: "bash tools/ci/with-projects.sh --exec bash src/wasix/sdks/ts/tools/integration/smoke-browser.sh --package-only --tools" - deps: ["oliphaunt-wasix-tools-ts:package", "oliphaunt-wasix-ts:package", "postgres-tools-wasix:package-portable"] + deps: ["oliphaunt-query-ts:build", "oliphaunt-wasix-tools-ts:package", {target: "oliphaunt-wasix-ts:package", cacheStrategy: ignored}, "oliphaunt-wasix-ts:build", "wasix-browser-host:build", "postgres-tools-wasix:package-portable"] inputs: ["ts/tests/**/*", "/src/wasix/sdks/ts/tools/integration/{packed-node-fixture.mts,smoke-browser.*}", "/src/test-fixtures/postgres/**/*", "tools/*.mts"] options: {cache: false, runFromWorkspaceRoot: true} format-check: diff --git a/src/wasix/postgres-tools/ts/tests/smoke-host.sh b/src/wasix/postgres-tools/ts/tests/smoke-host.sh index 147ee146c..55515d68b 100644 --- a/src/wasix/postgres-tools/ts/tests/smoke-host.sh +++ b/src/wasix/postgres-tools/ts/tests/smoke-host.sh @@ -14,7 +14,6 @@ esac deadline="$(command -v gtimeout || command -v timeout)" scratch="$(mktemp -d)" trap 'rm -rf "$scratch"' EXIT -bun run --cwd "$root/src/query/ts" build bun pm --cwd "$root/src/query/ts" pack --filename "$scratch/query.tgz" --quiet bun "$root/src/wasix/sdks/ts/tools/integration/packed-node-fixture.mts" "$scratch" --pgtap --tools cd "$scratch/consumer" diff --git a/src/wasix/postmaster/bin/package-release-assets.sh b/src/wasix/postmaster/bin/package-release-assets.sh index de2b57484..97b418906 100755 --- a/src/wasix/postmaster/bin/package-release-assets.sh +++ b/src/wasix/postmaster/bin/package-release-assets.sh @@ -89,7 +89,7 @@ cp -p "$project_root/bin/run-release-carrier.sh" \ chmod 0555 "$package_root/bin/oliphaunt-wasix-postmaster" cp -p "$repo_root/LICENSE" "$package_root/LICENSE" cp -p "$repo_root/THIRD_PARTY_NOTICES.md" "$package_root/THIRD_PARTY_NOTICES.md" -cp -p "$project_root/README.md" "$package_root/README.md" +cp -p "$repo_root/target/oliphaunt-wasix-postmaster/package-docs/README.md" "$package_root/README.md" bun "$repo_root/tools/packaging/archive-directory.mts" \ --keep-parent "$package_root" "$asset_dir/$asset_name" diff --git a/src/wasix/postmaster/bin/prepare-baseline.sh b/src/wasix/postmaster/bin/prepare-baseline.sh index 286db4319..3234df788 100755 --- a/src/wasix/postmaster/bin/prepare-baseline.sh +++ b/src/wasix/postmaster/bin/prepare-baseline.sh @@ -3,6 +3,7 @@ set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib/common.sh" +source "$REPO_ROOT/tools/dev/acquisition.sh" source "$REPO_ROOT/src/third-party/postgres/fetch-source.sh" print_path=0 diff --git a/src/wasix/postmaster/lib/common.sh b/src/wasix/postmaster/lib/common.sh index f8479fbc3..0d988051d 100644 --- a/src/wasix/postmaster/lib/common.sh +++ b/src/wasix/postmaster/lib/common.sh @@ -873,11 +873,14 @@ fresh_require_canonical_directory() { } fresh_wasix_builder_recipe_sha256() { + local identity_path local file_sha256 local identity_mode local path local recipe_paths=( + "$REPO_ROOT/tools/dev/acquisition.sh" "$WASIX_TOOLCHAIN_ROOT/docker/Dockerfile" + "$WASIX_TOOLCHAIN_ROOT/docker/Dockerfile.dockerignore" "$WASIX_TOOLCHAIN_ROOT/docker/isrg-root-x1.pem" "$WASIX_TOOLCHAIN_ROOT/docker/install-pinned-apt-packages.sh" "$WASIX_TOOLCHAIN_ROOT/docker/install-pinned-wasixcc.sh" @@ -903,7 +906,11 @@ fresh_wasix_builder_recipe_sha256() { else identity_mode=data fi - printf '%s\0%s\0%s\0' "${path#"$WASIX_TOOLCHAIN_ROOT"/}" "$file_sha256" "$identity_mode" + case "$path" in + "$WASIX_TOOLCHAIN_ROOT"/*) identity_path="src/wasix/runtime/assets/build/${path#"$WASIX_TOOLCHAIN_ROOT"/}" ;; + *) identity_path="${path#"$REPO_ROOT"/}" ;; + esac + printf '%s\0%s\0%s\0' "$identity_path" "$file_sha256" "$identity_mode" done } | fresh_sha256_stream } @@ -1705,11 +1712,15 @@ fresh_ensure_docker_image() { if [ "$actual_recipe" = "$expected_recipe" ]; then return fi + [ "$WASIX_TOOLCHAIN_ROOT" = "$REPO_ROOT/src/wasix/runtime/assets/build" ] || { + echo 'WASIX builder context requires WASIX_TOOLCHAIN_ROOT under REPO_ROOT; set both for another checkout' >&2 + return 2 + } "$docker_bin" build \ --label "$label=$expected_recipe" \ -f "$context/Dockerfile" \ -t "$image" \ - "$context" || return + "$REPO_ROOT" || return actual_recipe="$("$docker_bin" image inspect \ --format "{{ index .Config.Labels \"$label\" }}" "$image" 2>/dev/null || true)" [ "$actual_recipe" = "$expected_recipe" ] || { diff --git a/src/wasix/postmaster/lib/common.test.sh b/src/wasix/postmaster/lib/common.test.sh index 9ad944065..2a71aa5ae 100644 --- a/src/wasix/postmaster/lib/common.test.sh +++ b/src/wasix/postmaster/lib/common.test.sh @@ -174,15 +174,14 @@ portable_file_mode() { } builder_recipe_inputs=( docker/Dockerfile + docker/Dockerfile.dockerignore docker/isrg-root-x1.pem docker/install-pinned-apt-packages.sh docker/install-pinned-wasixcc.sh docker/pinned-wasixcc-assets.tsv ) -# The Dockerfile is itself the fifth recipe input. Every other recipe input -# must be a direct COPY source, and the Dockerfile must not consume an input -# that is absent from the recipe identity. +# Dockerfile, context policy and every COPY source enter the recipe identity. dockerfile_copy_sources="$( awk ' toupper($1) != "COPY" { next } @@ -205,11 +204,14 @@ dockerfile_copy_sources="$( exit 1 } expected_builder_context_inputs="$( - printf '%s\n' "${builder_recipe_inputs[@]#docker/}" | LC_ALL=C sort + { + printf 'src/wasix/runtime/assets/build/%s\n' "${builder_recipe_inputs[@]}" + printf 'tools/dev/acquisition.sh\n' + } | LC_ALL=C sort )" actual_builder_context_inputs="$( { - printf 'Dockerfile\n' + printf 'src/wasix/runtime/assets/build/docker/%s\n' Dockerfile Dockerfile.dockerignore printf '%s\n' "$dockerfile_copy_sources" } | LC_ALL=C sort )" @@ -220,6 +222,15 @@ actual_builder_context_inputs="$( exit 1 } +original_repo_root="$REPO_ROOT" +builder_repo="$test_root/builder-repo" +mkdir -p "$builder_repo/tools/dev" +cp "$REPO_ROOT/tools/dev/acquisition.sh" "$builder_repo/tools/dev/acquisition.sh" +REPO_ROOT="$builder_repo" +[ "$(fresh_wasix_builder_recipe_sha256)" = "$live_builder_recipe" ] +printf '\n# acquisition drift probe\n' >> "$builder_repo/tools/dev/acquisition.sh" +[ "$(fresh_wasix_builder_recipe_sha256)" != "$live_builder_recipe" ] +REPO_ROOT="$original_repo_root" builder_fixture="$test_root/builder-fixture" mkdir -p "$builder_fixture" cp -a "$original_toolchain_root/." "$builder_fixture/" @@ -624,7 +635,7 @@ run_fake_docker_case() { if [ "$expected_build_count" -eq 1 ]; then expected_build="$(printf 'build\t--label\t%s=%s\t-f\t%s/Dockerfile\t-t\t%s\t%s' \ - "$label" "$live_builder_recipe" "$context" "$image" "$context")" + "$label" "$live_builder_recipe" "$context" "$image" "$REPO_ROOT")" actual_build="$(awk -F '\t' '$1 == "build"' "$log")" [ "$actual_build" = "$expected_build" ] || { printf 'fake Docker case %s used unexpected build arguments\n' "$name" >&2 @@ -634,6 +645,8 @@ run_fake_docker_case() { fi } +# An external recipe must not build with COPY inputs from a different tree. +WASIX_TOOLCHAIN_ROOT="$builder_fixture" run_fake_docker_case external-context error expected 2 0 run_fake_docker_case matching-label expected expected 0 0 run_fake_docker_case absent-image error expected 0 1 run_fake_docker_case missing-label empty expected 0 1 diff --git a/src/wasix/postmaster/moon.yml b/src/wasix/postmaster/moon.yml index 0377aa763..72e8bab3f 100644 --- a/src/wasix/postmaster/moon.yml +++ b/src/wasix/postmaster/moon.yml @@ -38,6 +38,11 @@ owners: "**/*": - "@oliphaunt/wasix" fileGroups: + production: + - "**/*" + - "!**/*.md" + - "!**/*.test.*" + - "!testdata/**/*" guest-sealing: - bin/apply-wasix-core-overlay.sh - "bin/seal-wasix-{core-exports,linear-memory}.sh" @@ -93,6 +98,7 @@ tasks: inputs: - "/tools/dev/bun.sh" - "**/*" + - "!**/*.md" - /tools/packaging/strict-json.mts options: cache: true @@ -145,7 +151,9 @@ tasks: - liboliphaunt-wasix-postmaster:prepare-runtime inputs: - "@group(cargo-workspace)" + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - "@group(runtime-source)" - /src/wasix/postmaster/lib/common.sh @@ -173,7 +181,9 @@ tasks: - liboliphaunt-wasix-postmaster:prepare-runtime inputs: - "@group(cargo-workspace)" + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - "@group(runtime-source)" - wasmer/tests.sh @@ -191,7 +201,9 @@ tasks: - liboliphaunt-wasix-postmaster:prepare-postgres - liboliphaunt-wasix-postmaster:runtime-build inputs: + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - /src/wasix/postmaster/bin/build-wasix-core.sh - /src/wasix/postmaster/lib/common.sh @@ -211,7 +223,9 @@ tasks: inputs: - "@group(guest-sealing)" - /src/third-party/postgres/source.toml + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - /src/third-party/postgres/**/* - "!/src/third-party/postgres/**/*.test.*" @@ -237,7 +251,9 @@ tasks: deps: - liboliphaunt-wasix-postmaster:runtime-build inputs: + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - "@group(runtime-source)" - /src/wasix/postmaster/lib/common.sh @@ -276,6 +292,8 @@ tasks: deps: - liboliphaunt-wasix-postmaster:runtime-capabilities - liboliphaunt-wasix-postmaster:postgres-build + inputs: + - "@group(production)" options: cache: false internal: true @@ -290,6 +308,8 @@ tasks: command: bash src/wasix/postmaster/bin/stress-wasix-initdb.sh --iterations 20 deps: - liboliphaunt-wasix-postmaster:regression + inputs: + - "@group(production)" options: cache: false internal: true @@ -303,6 +323,8 @@ tasks: command: bash src/wasix/postmaster/bin/smoke-wasix-concurrent-connections.sh deps: - liboliphaunt-wasix-postmaster:initdb-smoke + inputs: + - "@group(production)" options: cache: false runFromWorkspaceRoot: true @@ -317,7 +339,9 @@ tasks: deps: - liboliphaunt-wasix-postmaster:smoke inputs: + - /tools/dev/acquisition.sh - /src/wasix/runtime/assets/build/docker/**/* + - "!/src/wasix/runtime/assets/build/docker/**/*.test.*" - /src/wasix/runtime/assets/build/docker_wasix_env.sh - /src/wasix/postmaster/bin/run-wasix-regress-subset.sh - /src/wasix/postmaster/bin/wasix-make.sh @@ -337,7 +361,7 @@ tasks: inputs: - "@group(legal-files)" - "@group(cargo-workspace)" - - "**/*" + - "@group(production)" - /src/wasix/postmaster/sources/*.toml - "/tools/packaging/*.{mjs,mts}" outputs: @@ -347,6 +371,17 @@ tasks: internal: true runFromWorkspaceRoot: true runInCI: true + package-docs: + tags: ["quality", "static"] + script: | + set -eu + mkdir -p target/oliphaunt-wasix-postmaster/package-docs + cp src/wasix/postmaster/README.md target/oliphaunt-wasix-postmaster/package-docs/README.md + inputs: ["README.md"] + outputs: ["/target/oliphaunt-wasix-postmaster/package-docs/README.md"] + options: + cache: true + runFromWorkspaceRoot: true release-assets: tags: - release @@ -356,9 +391,14 @@ tasks: command: bash src/wasix/postmaster/bin/package-release-assets.sh deps: - liboliphaunt-wasix-postmaster:carrier + # Documentation has its own cheap producer. A release always assembles it, + # but prose changes alone do not requalify unchanged runtime binaries. + - target: liboliphaunt-wasix-postmaster:package-docs + cacheStrategy: ignored inputs: - "@group(legal-files)" - - "**/*" + - "@group(production)" + - /target/oliphaunt-wasix-postmaster/package-docs/README.md - /src/wasix/postmaster/sources/*.toml - "/tools/packaging/*.{mjs,mts}" - /target/oliphaunt-wasix-postmaster/carriers/**/* @@ -378,6 +418,8 @@ tasks: deps: - liboliphaunt-wasix-postmaster:carrier - liboliphaunt-wasix-postmaster:initdb-stress + inputs: + - "@group(production)" options: cache: false internal: true @@ -394,7 +436,7 @@ tasks: - liboliphaunt-wasix-postmaster:carrier - liboliphaunt-wasix-postmaster:backend-wave-stress inputs: - - "**/*" + - "@group(production)" - /target/oliphaunt-wasix-postmaster/carriers/**/* options: cache: false @@ -410,7 +452,7 @@ tasks: - liboliphaunt-wasix-postmaster:postgres-build - liboliphaunt-wasix-postmaster:runtime-capabilities inputs: - - "**/*" + - "@group(production)" - /src/wasix/postmaster/sources/*.toml - "/tools/packaging/*.{mjs,mts}" - /target/oliphaunt-wasix-postmaster/install/wasix-core-release-o3.current diff --git a/src/wasix/postmaster/wasmer/tests.sh b/src/wasix/postmaster/wasmer/tests.sh index 97c9c7e91..1f363834e 100644 --- a/src/wasix/postmaster/wasmer/tests.sh +++ b/src/wasix/postmaster/wasmer/tests.sh @@ -25,13 +25,7 @@ run_tests \ remap_shared_file_fixed_accepts_a_partial_final_file_page \ remap_private_file_fixed_shares_clean_bytes_but_isolates_writes \ immutable_function_tables_are_shared_by_two_instances \ - shared_function_tables_outlive_artifact_owner_and_peer_instance -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$WASMER_ROOT/lib/vm/Cargo.toml" \ - -- \ - --exact \ + shared_function_tables_outlive_artifact_owner_and_peer_instance \ instance::allocator::tests::cached_offsets_produce_the_same_allocator_layout \ trap::traphandlers::tests::tls_stack_reuses_mapping_without_global_queue if [ "$(uname -s)-$(uname -m)" = Linux-x86_64 ]; then @@ -103,15 +97,7 @@ run_tests \ --lib \ --features wasmer/cranelift \ -- \ - --exact \ - state::tests::live_shared_mapping_registry_blocks_backing_file_shrink -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$WASMER_ROOT/lib/wasix/Cargo.toml" \ - --lib \ - --features wasmer/cranelift \ - -- \ + state::tests::live_shared_mapping_registry_blocks_backing_file_shrink \ utils::store::tests if [ "$(uname -s)" = Linux ]; then run_tests \ @@ -130,15 +116,7 @@ run_tests \ --features sys-minimal,wasmer/cranelift,ctrlc \ -- \ --test-threads=1 \ - os::task::task_join_handle::tests -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$WASMER_ROOT/lib/wasix/Cargo.toml" \ - --lib \ - --no-default-features \ - --features sys-minimal,wasmer/cranelift,ctrlc \ - -- \ + os::task::task_join_handle::tests \ runners::wasi:: run_tests \ --locked \ @@ -161,15 +139,6 @@ run_tests \ issues::llvm_rotates_and_atomic_fence_emit_expected_ir \ wast::spec::data_drop0::llvm::llvm \ wast::spec::memory_init::llvm::llvm -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$FRESH_ROOT/executor/Cargo.toml" \ - --lib \ - --no-default-features \ - --features "$FRESH_POSTMASTER_EXECUTOR_FEATURES" \ - -- \ - sealed::tests::runtime_policy_identity_ run_tests \ --locked \ --target-dir "$WASMER_TARGET_DIR" \ @@ -212,14 +181,7 @@ run_tests \ --manifest-path "$WASMER_ROOT/lib/api/Cargo.toml" \ --test module \ -- \ - serialized_artifact_inspector -run_tests \ - --locked \ - --target-dir "$WASMER_TARGET_DIR" \ - --manifest-path "$WASMER_ROOT/lib/api/Cargo.toml" \ - --test module \ - -- \ - --exact \ + serialized_artifact_inspector \ detached_module_executes_from_strict_relocated_regular_file_code_memory \ detached_mmapped_module_executes_without_retaining_serializable_state run_tests \ diff --git a/src/wasix/runtime/assets/build/docker/Dockerfile b/src/wasix/runtime/assets/build/docker/Dockerfile index 067871df4..df1aadeda 100644 --- a/src/wasix/runtime/assets/build/docker/Dockerfile +++ b/src/wasix/runtime/assets/build/docker/Dockerfile @@ -12,8 +12,9 @@ ARG OLIPHAUNT_UBUNTU_SNAPSHOT_TLS_ROOT_SHA256=22b557a27055b33606b6559f37703928d3 LABEL dev.oliphaunt.ubuntu-apt-snapshot="${OLIPHAUNT_UBUNTU_APT_SNAPSHOT}" \ dev.oliphaunt.ubuntu-snapshot-tls-root-sha256="${OLIPHAUNT_UBUNTU_SNAPSHOT_TLS_ROOT_SHA256}" -COPY --chmod=0444 isrg-root-x1.pem /usr/local/share/oliphaunt/isrg-root-x1.pem -COPY --chmod=0555 install-pinned-apt-packages.sh /usr/local/libexec/oliphaunt/install-pinned-apt-packages.sh +COPY --chmod=0444 src/wasix/runtime/assets/build/docker/isrg-root-x1.pem /usr/local/share/oliphaunt/isrg-root-x1.pem +COPY --chmod=0555 tools/dev/acquisition.sh /usr/local/libexec/oliphaunt/acquisition.sh +COPY --chmod=0555 src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh /usr/local/libexec/oliphaunt/install-pinned-apt-packages.sh RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ @@ -61,8 +62,8 @@ ENV PATH=/opt/wasixcc-home/.wasixcc/bin:$PATH ARG OLIPHAUNT_WASIXCC_ASSET_MANIFEST_SHA256=9b0ee1aabcfecda1be72c94a9f14a16c9d8a2fc020f3dc471394d5335766c519 LABEL dev.oliphaunt.wasixcc.asset-manifest-sha256="${OLIPHAUNT_WASIXCC_ASSET_MANIFEST_SHA256}" -COPY --chmod=0444 pinned-wasixcc-assets.tsv /usr/local/share/oliphaunt-wasixcc/pinned-wasixcc-assets.tsv -COPY --chmod=0555 install-pinned-wasixcc.sh /usr/local/libexec/oliphaunt/install-pinned-wasixcc.sh +COPY --chmod=0444 src/wasix/runtime/assets/build/docker/pinned-wasixcc-assets.tsv /usr/local/share/oliphaunt-wasixcc/pinned-wasixcc-assets.tsv +COPY --chmod=0555 src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh /usr/local/libexec/oliphaunt/install-pinned-wasixcc.sh RUN printf '%s %s\n' \ "$OLIPHAUNT_WASIXCC_ASSET_MANIFEST_SHA256" \ diff --git a/src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore b/src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore new file mode 100644 index 000000000..2a32c884d --- /dev/null +++ b/src/wasix/runtime/assets/build/docker/Dockerfile.dockerignore @@ -0,0 +1,12 @@ +** +!tools +!tools/dev +!tools/dev/acquisition.sh +!src +!src/wasix +!src/wasix/runtime +!src/wasix/runtime/assets +!src/wasix/runtime/assets/build +!src/wasix/runtime/assets/build/docker +!src/wasix/runtime/assets/build/docker/* +**/*.test.* diff --git a/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh b/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh index 7746945e0..a9cf9f6ff 100755 --- a/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh +++ b/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.sh @@ -2,6 +2,10 @@ set -eu export LC_ALL=C +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +acquisition_helper="$script_dir/acquisition.sh" +[ -f "$acquisition_helper" ] || acquisition_helper="$script_dir/../../../../../../tools/dev/acquisition.sh" +. "$acquisition_helper" fail() { echo "install-pinned-apt-packages: $*" >&2 @@ -52,7 +56,6 @@ for package in "$@"; do done apt_get="${OLIPHAUNT_APT_GET:-apt-get}" -sleep_command="${OLIPHAUNT_SLEEP:-sleep}" sources_file="${OLIPHAUNT_APT_SOURCES_FILE:-/etc/apt/sources.list.d/ubuntu.sources}" lists_dir="${OLIPHAUNT_APT_LISTS_DIR:-/var/lib/apt/lists}" ca_bundle="${OLIPHAUNT_CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" @@ -78,7 +81,7 @@ for required_path in "$sources_file" "$lists_dir" "$ca_bundle"; do done [ "$lists_dir" != "/" ] || fail "refusing to use / as the APT lists directory" command -v "$apt_get" >/dev/null 2>&1 || fail "missing APT command: $apt_get" -command -v "$sleep_command" >/dev/null 2>&1 || fail "missing sleep command: $sleep_command" +oliphaunt_acquisition_start 'WASIX builder APT update/install' 900 mkdir -p "$(dirname "$sources_file")" cat >"$sources_file" <&2 - "$sleep_command" "$delay" + oliphaunt_acquisition_sleep "$delay" || exit $? attempt=$((attempt + 1)) done fail "$label transaction failed after $max_attempts attempts" diff --git a/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh b/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh index 88617887e..49b9e84eb 100755 --- a/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh +++ b/src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh @@ -44,6 +44,9 @@ if [ "$operation" = update ]; then update="$(( $(cat "$update_file" 2>/dev/null || echo 0) + 1 ))" printf '%s\n' "$update" >"$update_file" case "${OLIPHAUNT_FAKE_APT_MODE:-success}" in + expired-update) + printf '9999999\n' > "$OLIPHAUNT_FAKE_APT_CLOCK" + ;; transient-update) [ "$update" -gt 1 ] || exit 100 ;; @@ -69,6 +72,12 @@ printf 'sleep=%s\n' "$*" >>"${OLIPHAUNT_FAKE_APT_LOG:?}" SLEEP chmod 0755 "$fake_bin/sleep" +cat >"$fake_bin/date" <<'DATE' +#!/usr/bin/env bash +cat "$OLIPHAUNT_FAKE_APT_CLOCK" +DATE +chmod +x "$fake_bin/date" + run_case() { local name="$1" local mode="$2" @@ -76,18 +85,19 @@ run_case() { local case_root="$work_root/$name" mkdir -p "$case_root" : >"$case_root/apt.log" + printf '1000000\n' > "$case_root/clock" if [ "$seed_ca" = true ]; then mkdir -p "$case_root/certs" printf '%s\n' fixture-pinned-snapshot-root >"$case_root/certs/ca-certificates.crt" fi PATH="$fake_bin:$PATH" \ OLIPHAUNT_APT_GET="$fake_bin/apt-get" \ - OLIPHAUNT_SLEEP="$fake_bin/sleep" \ OLIPHAUNT_APT_SOURCES_FILE="$case_root/ubuntu.sources" \ OLIPHAUNT_APT_LISTS_DIR="$case_root/lists" \ OLIPHAUNT_CA_BUNDLE="$case_root/certs/ca-certificates.crt" \ OLIPHAUNT_APT_MAX_ATTEMPTS=3 \ OLIPHAUNT_APT_RETRY_DELAY_SECONDS=1 \ + OLIPHAUNT_FAKE_APT_CLOCK="$case_root/clock" \ OLIPHAUNT_FAKE_APT_MODE="$mode" \ OLIPHAUNT_FAKE_APT_STATE="$case_root/state" \ OLIPHAUNT_FAKE_APT_LOG="$case_root/apt.log" \ @@ -156,6 +166,11 @@ if grep -q 'operation=install' "$permanent_log"; then fi [ "$(grep -c '^sleep=' "$permanent_log")" -eq 2 ] || fail "permanent update failure slept an unexpected number of times" +# A successful update cannot reset the budget before install or the next retry. +expect_failure deadline expired-update "deadline" +[ "$(cat "$work_root/deadline/state/calls")" = 1 ] || fail "deadline admitted another APT command" +[ ! -e "$work_root/deadline/state/installs" ] || fail "expired update reached installation" + expect_failure missing-ca success "pinned snapshot TLS root is missing" false missing_ca_root="$work_root/missing-ca" [ ! -s "$missing_ca_root/apt.log" ] || fail "missing TLS root reached APT" @@ -171,7 +186,6 @@ fi set +e invalid_output="$( OLIPHAUNT_APT_GET="$fake_bin/apt-get" \ - OLIPHAUNT_SLEEP="$fake_bin/sleep" \ "$installer" --snapshot latest -- bash 2>&1 )" invalid_status=$? diff --git a/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh b/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh index 82942e036..42cdf23f0 100755 --- a/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh +++ b/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.sh @@ -2,6 +2,10 @@ set -euo pipefail export LC_ALL=C +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +acquisition_helper="$script_dir/acquisition.sh" +[ -f "$acquisition_helper" ] || acquisition_helper="$script_dir/../../../../../../tools/dev/acquisition.sh" +. "$acquisition_helper" fail() { echo "install-pinned-wasixcc: $*" >&2 @@ -71,6 +75,7 @@ trap cleanup EXIT trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM +oliphaunt_acquisition_start 'WASIX compiler assets' 1800 validate_archive_members() { local archive="$1" @@ -258,17 +263,12 @@ while IFS=$'\t' read -r kind asset_name expected_bytes expected_sha256 url extra partial="$download_root/$asset_name.partial" archive="$download_root/$asset_name" rm -f "$partial" "$archive" - curl \ + oliphaunt_acquisition_curl 900 9 5 curl \ --fail \ --location \ --silent \ --show-error \ - --retry 8 \ - --retry-all-errors \ - --retry-delay 5 \ - --retry-max-time 900 \ --connect-timeout 20 \ - --max-time 900 \ --speed-limit 1024 \ --speed-time 120 \ --max-filesize "$expected_bytes" \ diff --git a/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh b/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh index b602f013c..def5bb69b 100755 --- a/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh +++ b/src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh @@ -12,6 +12,10 @@ fail() { } work_root="$(mktemp -d)" +mkdir -p "$work_root/no-delay" +printf '#!/bin/sh\n[ "$1" = 5 ] && exit 0\nexec "%s" "$@"\n' "$(command -v sleep)" > "$work_root/no-delay/sleep" +chmod +x "$work_root/no-delay/sleep" +export PATH="$work_root/no-delay:$PATH" trap 'rm -rf "$work_root"' EXIT fixtures="$work_root/fixtures" fake_bin="$work_root/fake-bin" @@ -249,10 +253,9 @@ expected_success_compiler_version="$(printf '%s\n' \ (cd "$success_root" && sha256sum --check --strict .oliphaunt-toolchain-assets.sha256 >/dev/null) || fail "installed identity manifest does not verify" for required_flag in \ - '--retry 8' \ - '--retry-all-errors' \ + '--retry 0' \ '--connect-timeout 20' \ - '--max-time 900' \ + '--max-time' \ '--proto =https' \ '--proto-redir =https' \ '--remove-on-error'; do diff --git a/src/wasix/runtime/assets/build/docker_contrib_extensions.sh b/src/wasix/runtime/assets/build/docker_contrib_extensions.sh index 078df839e..4a26058de 100755 --- a/src/wasix/runtime/assets/build/docker_contrib_extensions.sh +++ b/src/wasix/runtime/assets/build/docker_contrib_extensions.sh @@ -41,7 +41,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_initdb.sh b/src/wasix/runtime/assets/build/docker_initdb.sh index c279dd208..39e4aee0c 100755 --- a/src/wasix/runtime/assets/build/docker_initdb.sh +++ b/src/wasix/runtime/assets/build/docker_initdb.sh @@ -40,7 +40,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_oliphaunt.sh b/src/wasix/runtime/assets/build/docker_oliphaunt.sh index 6726aa1cd..82c21a6aa 100755 --- a/src/wasix/runtime/assets/build/docker_oliphaunt.sh +++ b/src/wasix/runtime/assets/build/docker_oliphaunt.sh @@ -41,7 +41,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_pgdump.sh b/src/wasix/runtime/assets/build/docker_pgdump.sh index b888a6f0e..5f77e7609 100755 --- a/src/wasix/runtime/assets/build/docker_pgdump.sh +++ b/src/wasix/runtime/assets/build/docker_pgdump.sh @@ -40,7 +40,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_pgxs_extensions.sh b/src/wasix/runtime/assets/build/docker_pgxs_extensions.sh index 1bbcfb188..110799144 100755 --- a/src/wasix/runtime/assets/build/docker_pgxs_extensions.sh +++ b/src/wasix/runtime/assets/build/docker_pgxs_extensions.sh @@ -41,7 +41,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_psql.sh b/src/wasix/runtime/assets/build/docker_psql.sh index 131f027ab..b634585c8 100755 --- a/src/wasix/runtime/assets/build/docker_psql.sh +++ b/src/wasix/runtime/assets/build/docker_psql.sh @@ -40,7 +40,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/docker_runtime_support.sh b/src/wasix/runtime/assets/build/docker_runtime_support.sh index 4ead94638..2115f3585 100755 --- a/src/wasix/runtime/assets/build/docker_runtime_support.sh +++ b/src/wasix/runtime/assets/build/docker_runtime_support.sh @@ -40,7 +40,7 @@ elif [ "${FORCE_IMAGE_BUILD:-0}" = "1" ] || ! "$DOCKER" image inspect "$IMAGE" > "$DOCKER" build \ -t "$IMAGE" \ -f "$ROOT/docker/Dockerfile" \ - "$ROOT/docker" + "$REPO_ROOT" else echo "reusing Docker image $IMAGE" fi diff --git a/src/wasix/runtime/assets/build/prepare_postgres_source.sh b/src/wasix/runtime/assets/build/prepare_postgres_source.sh index 9e725f2fc..f215338da 100755 --- a/src/wasix/runtime/assets/build/prepare_postgres_source.sh +++ b/src/wasix/runtime/assets/build/prepare_postgres_source.sh @@ -4,6 +4,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$SCRIPT_DIR/wasix_third_party.sh" REPO_ROOT="$(oliphaunt_wasix_repo_root "$SCRIPT_DIR")" +. "$REPO_ROOT/tools/dev/acquisition.sh" . "$REPO_ROOT/src/third-party/postgres/fetch-source.sh" SOURCE_TOML="$REPO_ROOT/src/third-party/postgres/source.toml" PATCH_DIR="$REPO_ROOT" diff --git a/src/wasix/runtime/assets/build/wasix_third_party.sh b/src/wasix/runtime/assets/build/wasix_third_party.sh index b581d73d8..2b7b0c9e9 100755 --- a/src/wasix/runtime/assets/build/wasix_third_party.sh +++ b/src/wasix/runtime/assets/build/wasix_third_party.sh @@ -69,7 +69,7 @@ oliphaunt_wasix_run_extension_build_in_docker_if_needed() { "$docker" build \ -t "$image" \ -f "$root/docker/Dockerfile" \ - "$root/docker" + "$(cd "$root/../../../../.." && pwd)" else echo "reusing Docker image $image" fi diff --git a/src/wasix/runtime/moon.yml b/src/wasix/runtime/moon.yml index 1235f5395..445fe19dc 100644 --- a/src/wasix/runtime/moon.yml +++ b/src/wasix/runtime/moon.yml @@ -109,6 +109,7 @@ tasks: - unit script: "set -e\nbash src/wasix/runtime/tools/check-shim-abi.sh\nbash src/wasix/runtime/assets/build/docker/install-pinned-apt-packages.test.sh\nbash src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh\n" inputs: + - /tools/dev/acquisition.sh - assets/build/docker/**/* - assets/build/wasix_shim/**/* - tools/check-shim-abi.sh @@ -134,6 +135,23 @@ tasks: deps: - source-inputs:source-fetch-wasix-runtime inputs: + - "$ASSET_PROFILE" + - "$IMAGE" + - "$OLIPHAUNT_WASM_SOURCE_LANE" + - "$CONTAINER_ROOT" + - "$CONTAINER_GENERATED_ROOT" + - "$OLIPHAUNT_WASM_WASIX_COPT" + - "$OLIPHAUNT_WASM_WASIX_LOPT" + - "$OLIPHAUNT_WASM_WASIX_CONFIGURE_WASM_OPT" + - "$OLIPHAUNT_WASM_WASIX_BUILD_WASM_OPT" + - "$OLIPHAUNT_WASM_WASM_OPT_FLAGS" + - "$OLIPHAUNT_WASM_WASM_OPT_SUPPRESS_DEFAULT" + - "$OLIPHAUNT_WASM_WASM_OPT_PRESERVE_UNOPTIMIZED" + - "$OLIPHAUNT_WASM_WASIX_COMPILER_FLAGS" + - "$OLIPHAUNT_WASM_WASIX_LINKER_FLAGS" + - "$OLIPHAUNT_WASM_PG18_DISABLE_SPINLOCKS" + - "$CONTAINER_BUILD_DIR" + - "$CONTAINER_PGSRC" - "@group(legal-files)" - "@group(cargo-workspace)" - /src/wasix/sdks/rust/THIRD_PARTY_NOTICES.md @@ -149,7 +167,9 @@ tasks: group: build - project: extension-runtime-contract group: contract + - /tools/dev/acquisition.sh - assets/build/**/* + - "!assets/build/**/*.test.*" - postgres/**/* - /src/third-party/postgres/**/* - "!/src/third-party/postgres/**/*.test.*" diff --git a/src/wasix/runtime/tools/build-compiler-output.sh b/src/wasix/runtime/tools/build-compiler-output.sh index f0d0e3d9f..952cd9451 100755 --- a/src/wasix/runtime/tools/build-compiler-output.sh +++ b/src/wasix/runtime/tools/build-compiler-output.sh @@ -34,7 +34,24 @@ export OLIPHAUNT_WASM_BUILD_PROFILE="$asset_profile" bash src/third-party/tools/fetch-sources.sh wasix-runtime --verify-only bash src/wasix/runtime/assets/build/prepare_postgres_source.sh >/dev/null build=src/wasix/runtime/assets/build -if [ "${OLIPHAUNT_SKIP_BUILD:-0}" != "1" ]; then +# Moon owns the complete source/toolchain input hash. Keep the absolute-path +# Make tree in the existing compilation cache, and verify its recorded bytes +# before bypassing compilation. Raw script calls keep normal incremental builds. +compiler_tree=target/oliphaunt-wasix/wasix-build/work/docker-oliphaunt +receipt=target/oliphaunt-wasix/wasix-build/build/compiler-input-hash +checksums=target/oliphaunt-wasix/wasix-build/build/compiler-output-sha256 +reuse=0 +if [ "${MOON_TARGET:-}" = liboliphaunt-wasix:compiler-output ] && + [[ "${MOON_TASK_HASH:-}" =~ ^[0-9a-f]{64}$ ]] && + [ "${FORCE_RECONFIGURE:-0}" != 1 ] && [ "${FORCE_IMAGE_BUILD:-0}" != 1 ] && + [ -s "$receipt" ] && [ -s "$checksums" ] && + [ "$(cat "$receipt")" = "$MOON_TASK_HASH" ] && + sha256sum --check --status "$checksums"; then + reuse=1 + echo 'Reusing verified WASIX compiler output for the current Moon input hash' +fi +if [ "${OLIPHAUNT_SKIP_BUILD:-0}" != "1" ] && [ "$reuse" = 0 ]; then + rm -f "$receipt" "$checksums" for script in docker_oliphaunt docker_runtime_support docker_initdb; do bash "$build/$script.sh" done @@ -42,3 +59,11 @@ fi awk -v profile="$asset_profile" '$0 == "profile=" profile {found=1} END {exit !found}' \ target/oliphaunt-wasix/wasix-build/work/docker-oliphaunt/.oliphaunt-wasix-build-profile cargo run -p xtask -- assets stage-runtime + +if [ "${MOON_TARGET:-}" = liboliphaunt-wasix:compiler-output ] && + [[ "${MOON_TASK_HASH:-}" =~ ^[0-9a-f]{64}$ ]] && [ "$reuse" = 0 ] && + [ "${OLIPHAUNT_SKIP_BUILD:-0}" != 1 ]; then + mkdir -p "$(dirname "$receipt")" + find "$compiler_tree" -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum > "$checksums" + printf '%s\n' "$MOON_TASK_HASH" > "$receipt" +fi diff --git a/src/wasix/runtime/tools/build-runtime-portable.test.sh b/src/wasix/runtime/tools/build-runtime-portable.test.sh index fdaa88647..40465e068 100755 --- a/src/wasix/runtime/tools/build-runtime-portable.test.sh +++ b/src/wasix/runtime/tools/build-runtime-portable.test.sh @@ -52,4 +52,24 @@ bash "$fixture/$owner/tools/build-runtime-portable.sh" --package-only echo profile=debug > "$receipt" if OLIPHAUNT_SKIP_BUILD=1 bash "$fixture/$owner/tools/build-runtime-portable.sh"; then exit 1; fi [ "$(tail -1 "$BUILD_LOG")" = prepare_postgres_source ] +# Only a complete matching input/output receipt skips the compiler. Corrupt +# bytes and a changed input hash must both return to the build path. +echo profile=release > "$receipt" +export MOON_TARGET=liboliphaunt-wasix:compiler-output +export MOON_TASK_HASH="$(printf '%064d' 1)" +: > "$BUILD_LOG" +bash "$fixture/$owner/tools/build-compiler-output.sh" +: > "$BUILD_LOG" +bash "$fixture/$owner/tools/build-compiler-output.sh" +! grep -q '^docker_' "$BUILD_LOG" +echo corrupted >> "$receipt" +: > "$BUILD_LOG" +# The fixture build scripts do not repair the profile marker; its release line +# remains present, so the assertion still allows the fresh compiler path. +bash "$fixture/$owner/tools/build-compiler-output.sh" +grep -q '^docker_oliphaunt$' "$BUILD_LOG" +export MOON_TASK_HASH="$(printf '%064d' 2)" +: > "$BUILD_LOG" +bash "$fixture/$owner/tools/build-compiler-output.sh" +grep -q '^docker_oliphaunt$' "$BUILD_LOG" echo 'WASIX build order, core-only selection, failure propagation, and stale-profile refusal passed.' diff --git a/src/wasix/sdks/rust/moon.yml b/src/wasix/sdks/rust/moon.yml index 72f258d86..a806e4c9e 100644 --- a/src/wasix/sdks/rust/moon.yml +++ b/src/wasix/sdks/rust/moon.yml @@ -46,13 +46,13 @@ tasks: - project: "database-resources" group: "cargo-carrier-sources" test-regression: - tags: ["regression", "runtime", "requires-rust"] + tags: ["regression", "runtime", "requires-rust", "ci-wasix-release-regression"] command: "bash src/wasix/runtime/tools/runtime-smoke.sh regression" - deps: [{target: "cargo-sources", cacheStrategy: hash}, "liboliphaunt-wasix:runtime-aot", "extension-artifacts-wasix:build-aot", "postgres-tools-wasix:build-aot"] + deps: [{target: "cargo-sources", cacheStrategy: hash}, "liboliphaunt-wasix:runtime-aot", "extension-artifacts-wasix:build-target", "extension-artifacts-wasix:build-aot", "postgres-tools-wasix:build-aot"] inputs: ["@group(code)", "@group(cargo-workspace)", "/src/wasix/runtime/tools/{runtime-smoke,runtime-preflight,cargo-test-filter}.sh", "/src/wasix/runtime/tools/wasix-extension-features.mts", "/src/extensions/artifacts/packages/tools/**/*"] - options: {runFromWorkspaceRoot: true, cache: local} + options: {runFromWorkspaceRoot: true, cache: false} test-integration: - tags: ["runtime", "integration", "requires-rust"] + tags: ["runtime", "integration", "requires-rust", "ci-wasix-release-regression", "platform-linux-x64-gnu"] command: "bash src/wasix/sdks/rust/tools/test-resources.sh" deps: - target: cargo-sources @@ -62,6 +62,7 @@ tasks: - "database-resources:build-wasix-icu" - "database-resources:package-icu" inputs: + - "/src/wasix/runtime/tools/runtime-preflight.sh" - "@group(code)" - "@group(cargo-workspace)" options: diff --git a/src/wasix/sdks/rust/tools/test-resources.sh b/src/wasix/sdks/rust/tools/test-resources.sh index 923ed5dbe..796dc9b12 100644 --- a/src/wasix/sdks/rust/tools/test-resources.sh +++ b/src/wasix/sdks/rust/tools/test-resources.sh @@ -2,6 +2,11 @@ set -euo pipefail root="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" cd "$root" +. src/wasix/runtime/tools/runtime-preflight.sh +oliphaunt_runtime_wasm_require +export OLIPHAUNT_ARTIFACT_CRATE_REQUIRE_PAYLOAD=1 +export OLIPHAUNT_WASIX_GENERATED_ASSETS_DIR="$root/target/oliphaunt-wasix/assets" +export OLIPHAUNT_WASM_GENERATED_AOT_DIR="$root/target/oliphaunt-wasix/aot" version="$(cat src/database-resources/VERSION)" export OLIPHAUNT_TEST_STANDARD_SEED="$root/target/database-resources/release-assets/database-resources-$version-seed-wasix-standard.tar.zst" export OLIPHAUNT_TEST_ICU_SEED="$root/target/database-resources/release-assets/database-resources-$version-seed-wasix-icu.tar.zst" @@ -9,4 +14,9 @@ OLIPHAUNT_TEST_ICU_ROOT="$(mktemp -d)" export OLIPHAUNT_TEST_ICU_ROOT trap 'rm -rf "$OLIPHAUNT_TEST_ICU_ROOT"' EXIT tar -xzf "$root/target/database-resources/release-assets/database-resources-$version-icu-data.tar.gz" -C "$OLIPHAUNT_TEST_ICU_ROOT" -cargo test -p oliphaunt-wasix --locked --test resources -- --ignored --test-threads=1 +cargo test -p oliphaunt-wasix --locked --test resources --color never -- --ignored --test-threads=1 \ + 2>&1 | tee "$OLIPHAUNT_TEST_ICU_ROOT/tests.log" +grep -Eq '^test result: ok\. [1-9][0-9]* passed;' "$OLIPHAUNT_TEST_ICU_ROOT/tests.log" || { + echo 'WASIX resource tests must execute and pass' >&2 + exit 1 +} diff --git a/src/wasix/sdks/ts/moon.yml b/src/wasix/sdks/ts/moon.yml index 679ca0493..298d4986e 100644 --- a/src/wasix/sdks/ts/moon.yml +++ b/src/wasix/sdks/ts/moon.yml @@ -78,7 +78,7 @@ tasks: - package - release - artifact-package - - ci-wasix-ts-sdk-package + - ci-wasix-ts-package script: bun run --cwd src/wasix/sdks/ts package deps: - wasix-browser-host:build @@ -155,9 +155,13 @@ tasks: - consumer - ci-wasix-ts-sdk-package deps: + - oliphaunt-query-ts:build - extension-artifacts-wasix:compiler-output - liboliphaunt-wasix:runtime-portable - - oliphaunt-wasix-ts:package + - target: oliphaunt-wasix-ts:package + cacheStrategy: ignored + - oliphaunt-wasix-ts:build + - wasix-browser-host:build - oliphaunt-wasix-napi:build-release-assets inputs: &inputs @@ -166,7 +170,7 @@ tasks: - /src/examples/wasix/browser/**/* - project: shared-test-fixtures group: fixtures - - "**/*" + - "@group(code)" - /tools/dev/bun.sh - /tools/dev/deno.sh - /src/wasix/runtime/tools/wasix-*.mts @@ -182,11 +186,15 @@ tasks: - browser - ci-wasix-ts-sdk-package deps: + - oliphaunt-query-ts:build - extension-artifacts-wasix:compiler-output - database-resources:package-wasix - database-resources:package-icu - liboliphaunt-wasix:runtime-portable - - oliphaunt-wasix-ts:package + - target: oliphaunt-wasix-ts:package + cacheStrategy: ignored + - oliphaunt-wasix-ts:build + - wasix-browser-host:build inputs: *inputs options: diff --git a/src/wasix/sdks/ts/tools/integration/smoke-browser.sh b/src/wasix/sdks/ts/tools/integration/smoke-browser.sh index cc4268b73..1f68be834 100644 --- a/src/wasix/sdks/ts/tools/integration/smoke-browser.sh +++ b/src/wasix/sdks/ts/tools/integration/smoke-browser.sh @@ -37,7 +37,6 @@ cleanup() { exit "$status" } trap cleanup EXIT -bun run --cwd "$root/src/query/ts" build bun pm --cwd "$root/src/query/ts" pack --filename "$scratch/query.tgz" --quiet bun "$tool" --prepare "$scratch" "$@" configuration="$scratch/browser.json" diff --git a/src/wasix/sdks/ts/tools/integration/smoke-node.sh b/src/wasix/sdks/ts/tools/integration/smoke-node.sh index cb034245e..47f87377a 100644 --- a/src/wasix/sdks/ts/tools/integration/smoke-node.sh +++ b/src/wasix/sdks/ts/tools/integration/smoke-node.sh @@ -13,7 +13,6 @@ done deadline="$(command -v gtimeout || command -v timeout)" scratch="$(mktemp -d)" trap 'rm -rf "$scratch"' EXIT -bun run --cwd "$root/src/query/ts" build bun pm --cwd "$root/src/query/ts" pack --filename "$scratch/query.tgz" --quiet bun "$root/src/wasix/sdks/ts/tools/integration/smoke-node.mts" "$scratch" "${args[@]}" cd "$scratch/consumer" diff --git a/tools/ci/affected.mts b/tools/ci/affected.mts index 10870e5e2..cfe704614 100644 --- a/tools/ci/affected.mts +++ b/tools/ci/affected.mts @@ -21,7 +21,11 @@ export function triggeringTaskNames(value = {}) { return Object.entries(value) .filter(([, detail]) => { if (detail === null || Array.isArray(detail) || typeof detail !== 'object') return false; - return detail.other === true || (Array.isArray(detail.files) && detail.files.length > 0); + return ( + detail.other === true || + (Array.isArray(detail.files) && detail.files.length > 0) || + (Array.isArray(detail.env) && detail.env.length > 0) + ); }) .map(([task]) => task) .sort(); diff --git a/tools/ci/capture-ci-test-observations.sh b/tools/ci/capture-ci-test-observations.sh index 6cb9ae224..71ae53b29 100644 --- a/tools/ci/capture-ci-test-observations.sh +++ b/tools/ci/capture-ci-test-observations.sh @@ -6,12 +6,18 @@ directory="${1:?expected observation directory}" bash tools/dev/bun.sh tools/ci/ci-plan-test-observations.mts "$directory" unset MOON_BASE MOON_HEAD export MOON_CACHE=off +# Each request writes its own file. Four workers bound memory and preserve all +# graph/affected proofs while avoiding serial workspace startup for every case. +# NUL records preserve paths and the empty target of an affected query on BSD/GNU xargs. while IFS=$'\t' read -r kind id target; do + printf '%s\0%s\0%s\0' "$kind" "$id" "$target" +done < "$directory/requests.tsv" | xargs -0 -n 3 -P 4 bash -eu -c ' + directory=$1 moon_bin=$2 kind=$3 id=$4 target=$5 case "$kind" in affected) - "${MOON_BIN:-moon}" query affected stdin --upstream none --downstream deep \ + "$moon_bin" query affected stdin --upstream none --downstream deep \ < "$directory/affected-$id.input" > "$directory/affected-$id.json" ;; - task) "${MOON_BIN:-moon}" task-graph "$target" --json > "$directory/task-$id.json" ;; + task) "$moon_bin" task-graph "$target" --json > "$directory/task-$id.json" ;; *) echo "unexpected test observation kind: $kind" >&2; exit 1 ;; esac -done < "$directory/requests.tsv" +' ci-observation "$directory" "${MOON_BIN:-moon}" diff --git a/tools/ci/capture-ci-test-observations.test.sh b/tools/ci/capture-ci-test-observations.test.sh new file mode 100644 index 000000000..ce3d3bdb4 --- /dev/null +++ b/tools/ci/capture-ci-test-observations.test.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(git rev-parse --show-toplevel)" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +cat > "$work/moon" <<'MOON' +#!/bin/sh +# Fail graph queries after successful affected queries have already completed. +if [ "$1" = task-graph ]; then exit 7; fi +printf '{}\n' +MOON +chmod +x "$work/moon" +if MOON_BIN="$work/moon" bash "$root/tools/ci/capture-ci-test-observations.sh" "$work/observations with spaces"; then + echo 'parallel observation capture swallowed a failed query' >&2 + exit 1 +fi +[ -n "$(find "$work/observations with spaces" -name 'affected-*.json' -print -quit)" ] +echo 'Parallel Moon observation failures propagate after completed queries' diff --git a/tools/ci/check-workflows.sh b/tools/ci/check-workflows.sh index 267eb4247..3e8559a17 100755 --- a/tools/ci/check-workflows.sh +++ b/tools/ci/check-workflows.sh @@ -28,6 +28,7 @@ require zizmor run actionlint -ignore 'unexpected key "queue" for "concurrency" section' run zizmor --config .github/zizmor.yml --min-severity medium --persona auditor .github/workflows .github/actions run bash tools/dev/bun.sh test ./tools/ci/workflow-security.test.mts +run bash tools/dev/bun.sh test ./tools/ci/workflow-caches.test.mts run tools/dev/bun.sh tools/ci/workflow-security.mts run bash .github/scripts/check-ci-gate.test.sh run bash tools/dev/bun.sh test ./.github/scripts/resolve-mobile-e2e.test.mts @@ -42,6 +43,7 @@ export OLIPHAUNT_CI_TEST_OBSERVATIONS="$observations" run bash tools/dev/bun.sh test ./.github/scripts/moon-task-capabilities.test.mts run bash .github/scripts/write-affected-moon-target-matrices.test.sh run bash .github/scripts/resolve-planned-moon-execution.test.sh +run bash tools/ci/capture-ci-test-observations.test.sh run bash tools/ci/with-projects.sh --exec bash tools/ci/capture-ci-test-observations.sh "$observations" run bash tools/ci/ci-release-scope.test.sh run bash tools/ci/with-projects.sh test \ diff --git a/tools/ci/ci-plan-node-products.test.mts b/tools/ci/ci-plan-node-products.test.mts index 73cec77f9..c52a2ae4b 100644 --- a/tools/ci/ci-plan-node-products.test.mts +++ b/tools/ci/ci-plan-node-products.test.mts @@ -1,22 +1,24 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; +import { loadExtensionTargetProfiles } from '../../src/extensions/contracts/extension-target-profiles.mts'; +import { publishedConsumerDependencies } from '../../src/native/sdks/ts/tools/published-consumer.mts'; +import { + contribCarrierDescriptor, + extensionProductForSqlName, +} from '../release/release-artifact-targets.mts'; import { buildPlan, loadGraph, normalizeFiles } from '../release/release-graph.mts'; import { affectedNames, triggeringProjectNames, triggeringTaskNames } from './affected.mts'; import { dependencyPlatformTargets, + extensionArtifactsNativeMatrixForPlan, jobTargetsForJobs, planForReleaseProducts, planJobsForAffected, + renderPlanForFullRun, requiredTasksForAffected, } from './ci_plan.mts'; import { combinedNativeWasix, paths, taskRoots } from './ci-plan-test-inputs.mts'; import { affectedObservation, taskObservation } from './ci-plan-test-observations.mts'; -import { loadExtensionTargetProfiles } from '../../src/extensions/contracts/extension-target-profiles.mts'; -import { - contribCarrierDescriptor, - extensionProductForSqlName, -} from '../release/release-artifact-targets.mts'; -import { publishedConsumerDependencies } from '../../src/native/sdks/ts/tools/published-consumer.mts'; const GRAPH = loadGraph('ci-plan-node-products.test.mts'); const NATIVE_TS_CONSUMER_JOBS = [ @@ -120,6 +122,22 @@ function effects(paths) { }; } +test('environment input changes retain their producer and runtime qualification', () => { + const roots = new Set( + triggeringTaskNames({ + 'liboliphaunt-wasix:compiler-output': { env: ['ASSET_PROFILE'], other: false }, + 'liboliphaunt-wasix:runtime-portable': { + upstream: ['liboliphaunt-wasix:compiler-output'], + other: false, + }, + }), + ); + assert.deepEqual([...roots], ['liboliphaunt-wasix:compiler-output']); + const required = requiredTasksForAffected(roots); + assert(required.has('liboliphaunt-wasix:runtime-portable')); + assert(planJobsForAffected(roots).has('wasix-release-regression')); +}); + test('shared Windows DLL policy changes select native and WASIX packaging consumers', () => { const result = effects(paths.windowsVcRuntimePolicy); for (const target of [ @@ -161,8 +179,10 @@ test('an empty Moon selection requires no product tasks or releases', () => { assert.deepEqual(buildPlan(GRAPH, [], 'ci-plan-node-products.test.mts').releaseProducts, []); }); -test('WASIX qualification selects all same-run release regression inputs', () => { +test('WASIX qualification and Linux diagnostics select all same-run regression inputs', () => { const required = [ + 'oliphaunt-wasix-rust:test-regression', + 'oliphaunt-wasix-rust:test-integration', 'liboliphaunt-wasix:runtime-portable', 'liboliphaunt-wasix:runtime-aot', 'extension-artifacts-wasix:compiler-output', @@ -170,23 +190,37 @@ test('WASIX qualification selects all same-run release regression inputs', () => 'extension-artifacts-wasix:build-aot', 'postgres-tools-wasix:compiler-output', 'postgres-tools-wasix:build-aot', + 'database-resources:build-wasix-standard', + 'database-resources:build-wasix-icu', + 'database-resources:package-icu', ]; const roots = new Set(['liboliphaunt-wasix:release-assets']); const jobs = planJobsForAffected(roots); const affected = jobTargetsForJobs(jobs, requiredTasksForAffected(roots)); const release = planForReleaseProducts(['liboliphaunt-wasix'], 'a'.repeat(40)); - for (const targets of [affected, release.job_targets]) { + const full = renderPlanForFullRun(); + const linux = renderPlanForFullRun({ wasmTarget: 'linux-x64-gnu' }); + for (const plan of [release, full, linux]) { + assert(plan.jobs.includes('wasix-release-regression')); + assert(!plan.builder_jobs.includes('wasix-release-regression')); + assert(plan.jobs.includes('js-sdk-package')); + assert(plan.extension_artifacts_wasix_matrix.include.length > 0); + assert(plan.liboliphaunt_wasix_aot_runtime_matrix_linux.include.length > 0); + } + for (const targets of [affected, release.job_targets, full.job_targets, linux.job_targets]) { const selected = new Set(Object.values(targets).flat()); for (const target of required) assert(selected.has(target), `missing evidence input ${target}`); } assert(jobs.has('extension-artifacts-wasix')); + assert( + !renderPlanForFullRun({ wasmTarget: 'macos-arm64' }).jobs.includes('wasix-release-regression'), + ); }); test('shared Rust query changes stage the native and WASIX consumer artifacts', () => { const result = effects(paths.sdksRustQuerySrcLibRs); - for (const consumer of ['oliphaunt-wasix-ts:package', 'oliphaunt-wasix-ts:test-consumer']) { - assert(result.jobTargets['wasix-ts-sdk-package'].includes(consumer), consumer); - } + assert(result.jobTargets['wasix-ts-package'].includes('oliphaunt-wasix-ts:package')); + assert(result.jobTargets['wasix-ts-sdk-package'].includes('oliphaunt-wasix-ts:test-consumer')); for (const producer of [ 'oliphaunt-rust:package', 'oliphaunt-query:package', @@ -467,6 +501,8 @@ test('combined JavaScript SDK and WASIX N-API changes release only changed produ 'node-direct', 'wasix-napi', 'wasix-napi-release-assets', + 'wasix-release-regression', + 'wasix-ts-package', 'wasix-ts-sdk-package', ]); assert.deepEqual(result.releaseProducts, ['oliphaunt-js', 'oliphaunt-wasix-napi']); @@ -540,6 +576,8 @@ test('WASIX N-API source selects only its real WASIX artifact inputs', () => { 'liboliphaunt-wasix-runtime', 'wasix-napi', 'wasix-napi-release-assets', + 'wasix-release-regression', + 'wasix-ts-package', 'wasix-ts-sdk-package', ]); assert.deepEqual(result.releaseProducts, ['oliphaunt-wasix-napi']); @@ -887,3 +925,96 @@ test('mixed SDK and query releases retain the Linux native consumer alongside mo ), ); }); + +test('frozen iOS metadata only replaces the unchanged package input in affected runs', () => { + const old = process.env.OLIPHAUNT_REUSE_IOS_CARRIER; + try { + assert( + !Object.hasOwn( + affectedObservation(paths.sdksKotlinToolsStageReleaseArtifactsMts).tasks, + 'liboliphaunt-native:finalize-runtime-ios-abi', + ), + ); + assert( + Object.hasOwn( + affectedObservation(paths.runtimesLiboliphauntNativeSrcLiboliphauntProcessC).tasks, + 'liboliphaunt-native:finalize-runtime-ios-abi', + ), + ); + process.env.OLIPHAUNT_REUSE_IOS_CARRIER = 'true'; + const roots = new Set(['integration-examples:react-native-android-build']); + const warm = requiredTasksForAffected(roots); + assert(!warm.has('liboliphaunt-native:finalize-runtime-ios-abi')); + assert( + requiredTasksForAffected(roots, undefined, false).has( + 'liboliphaunt-native:finalize-runtime-ios-abi', + ), + ); + roots.add('liboliphaunt-native:finalize-runtime-ios-abi'); + assert(requiredTasksForAffected(roots).has('liboliphaunt-native:finalize-runtime-ios-abi')); + const release = planForReleaseProducts(['oliphaunt-react-native'], 'a'.repeat(40)); + assert(release.jobs.includes('liboliphaunt-native-ios-abi')); + assert.equal(release.reuse_ios_carrier, false); + } finally { + if (old === undefined) delete process.env.OLIPHAUNT_REUSE_IOS_CARRIER; + else process.env.OLIPHAUNT_REUSE_IOS_CARRIER = old; + } +}); + +test('Android extension packaging selects matching dependency-complete Linux support', () => { + for (const target of ['android-arm64-v8a', 'android-x86_64']) { + const matrix = extensionArtifactsNativeMatrixForPlan( + new Set(['extension-artifacts-native']), + new Set([target]), + new Set([extensionProductForSqlName('earthdistance')]), + 'all', + ); + assert.deepEqual( + matrix.include.map((row) => row.target), + [target, 'linux-x64-gnu'], + ); + const android = matrix.include.find((row) => row.target === target); + const linux = matrix.include.find((row) => row.target === 'linux-x64-gnu'); + for (const product of android.extensions_csv.split(',')) { + assert(linux.extensions_csv.split(',').includes(product), product); + } + assert(linux.sql_names_csv.split(',').includes('cube')); + } +}); + +test('WASIX package README and compiler unit fixtures do not select runtime compilers', () => { + for (const file of [paths.wasixSdkReadme, paths.wasixDockerTest, paths.nativeExtensionFixture]) { + const observation = affectedObservation([file]); + const roots = new Set(triggeringTaskNames(observation.tasks)); + const required = requiredTasksForAffected(roots); + assert(!required.has('liboliphaunt-wasix:compiler-output'), file); + assert(!required.has('extension-artifacts-native:build-target'), file); + assert(!required.has('liboliphaunt-wasix-postmaster:postgres-build'), file); + } + const jobs = planJobsForAffected( + new Set(triggeringTaskNames(affectedObservation([paths.wasixSdkReadme]).tasks)), + ); + assert(jobs.has('wasix-ts-package')); + assert(!jobs.has('wasix-ts-sdk-package')); +}); + +test('SDK runtime test edits select their final hosted execution roots', () => { + for (const [file, job, target] of [ + [paths.nativeRustRuntimeTests, 'native-consumers', 'oliphaunt-rust:test-integration'], + [paths.mobileBrokerRuntimeTests, 'native-consumers', 'oliphaunt-mobile-bindings:test-native'], + [paths.nativeSwiftRuntimeTests, 'native-consumers', 'oliphaunt-swift:test-native'], + [paths.nativeKotlinRuntimeTests, 'native-consumers', 'oliphaunt-kotlin:test-native-bindings'], + [ + paths.wasixResourceRuntimeTests, + 'wasix-release-regression', + 'oliphaunt-wasix-rust:test-integration', + ], + ]) { + const selected = effects(file); + assert(selected.jobs.includes(job), `${file} must schedule ${job}`); + const roots = new Set(selected.directTasks); + const executable = requiredTasksForAffected(roots); + assert(executable.has(target), `${target} is missing from executable closure`); + assert(jobTargetsForJobs(new Set(selected.jobs), executable)[job].includes(target)); + } +}); diff --git a/tools/ci/ci-plan-test-inputs.mts b/tools/ci/ci-plan-test-inputs.mts index 7000c063e..fa2dca846 100644 --- a/tools/ci/ci-plan-test-inputs.mts +++ b/tools/ci/ci-plan-test-inputs.mts @@ -1,5 +1,15 @@ // Sample changes exercised by CI planning tests; never used for production affectedness. export const paths = { + postmasterReadme: 'src/wasix/postmaster/README.md', + nativeRustRuntimeTests: 'src/native/sdks/rust/tests/native_sql_regression.rs', + nativeSwiftRuntimeTests: 'src/native/sdks/swift/Tests/OliphauntTests/NativeRuntimeTests.swift', + nativeKotlinRuntimeTests: + 'src/native/sdks/kotlin/oliphaunt/src/androidUnitTest/kotlin/dev/oliphaunt/NativeBindingsTest.kt', + mobileBrokerRuntimeTests: 'src/native/sdks/rust/tests/mobile_broker.rs', + wasixResourceRuntimeTests: 'src/wasix/sdks/rust/tests/resources.rs', + wasixSdkReadme: 'src/wasix/sdks/ts/README.md', + wasixDockerTest: 'src/wasix/runtime/assets/build/docker/install-pinned-wasixcc.test.sh', + nativeExtensionFixture: 'src/extensions/artifacts/native/tools/create-artifact.test.mts', windowsVcRuntimePolicy: 'tools/packaging/windows-vc-runtime-policy.json', wasixRuntimeCarrierSource: 'src/wasix/runtime/crates/assets/src/lib.rs', wasixToolsCarrierSource: 'src/wasix/postgres-tools/crates/tools/src/lib.rs', @@ -77,7 +87,6 @@ export const paths = { runtimesLiboliphauntWasixPostmasterWasmerBinVerifyPostmasterConcurrencyContractTestMts: 'src/wasix/postmaster/wasmer/bin/verify-postmaster-concurrency-contract.test.mts', srcSourcesThirdPartyNativeREADMEMd: 'src/sources/third-party/native/README.md', - toolsDevMaestroToml: 'tools/dev/maestro.toml', postgresToolsWasixCratesToolsSrcLibRs: 'src/wasix/postgres-tools/crates/tools/src/lib.rs', runtimesLiboliphauntWasixToolsXtaskSrcMainRs: 'src/wasix/runtime/tools/xtask/src/main.rs', runtimesLiboliphauntWasixAssetsBuildDockerInstallPinnedWasixccSh: diff --git a/tools/ci/ci-plan-wasix-postmaster-release.test.mts b/tools/ci/ci-plan-wasix-postmaster-release.test.mts index 533e6d777..b24228464 100644 --- a/tools/ci/ci-plan-wasix-postmaster-release.test.mts +++ b/tools/ci/ci-plan-wasix-postmaster-release.test.mts @@ -1,5 +1,3 @@ -import { affectedObservation, taskObservation } from './ci-plan-test-observations.mts'; -import { paths, taskRoots } from './ci-plan-test-inputs.mts'; import assert from 'node:assert/strict'; import { test } from 'node:test'; import { buildPlan, loadGraph, normalizeFiles } from '../release/release-graph.mts'; @@ -10,6 +8,8 @@ import { renderPlanWithSelection, selectedExtensionProductsForPlan, } from './ci_plan.mts'; +import { paths, taskRoots } from './ci-plan-test-inputs.mts'; +import { affectedObservation, taskObservation } from './ci-plan-test-observations.mts'; const taskGraph = taskObservation; @@ -170,7 +170,6 @@ test('source prose, transport tests, and unrelated toolchains do not rebuild run 'liboliphaunt-wasix-postmaster:test', ], [paths.srcSourcesThirdPartyNativeREADMEMd, null], - [paths.toolsDevMaestroToml, 'ci-workflows:check'], ]; for (const [relativePath, expectedTask] of cases) { const effects = directEffects(relativePath); @@ -244,3 +243,13 @@ test('native lifecycle supervisor changes select its exact hosted proof', () => paths.runtimesLiboliphauntWasixPostmasterLibProcessSupervisionSh, ); }); + +test('shipped Postmaster documentation has a cheap producer without losing release ownership', () => { + const effects = directEffects(paths.postmasterReadme); + assert(effects.directTasks.includes('liboliphaunt-wasix-postmaster:package-docs')); + assert(!effects.jobs.includes('wasix-postmaster')); + const release = buildPlan(loadGraph(), normalizeFiles([paths.postmasterReadme])); + assert(release.releaseProducts.includes('liboliphaunt-wasix-postmaster')); + const closure = taskGraph(taskRoots.liboliphauntWasixPostmasterReleaseAssets); + assert(closure.some(({ target }) => target === 'liboliphaunt-wasix-postmaster:package-docs')); +}); diff --git a/tools/ci/ci-plan.sh b/tools/ci/ci-plan.sh index b30d16a0a..c94d9781d 100644 --- a/tools/ci/ci-plan.sh +++ b/tools/ci/ci-plan.sh @@ -30,6 +30,25 @@ if [[ $# == 0 && (${CI_GENERATED_RELEASE_PR:-false} == true || (${GITHUB_EVENT_N export CI_RELEASE_PRODUCTS_JSON fi fi +# A frozen carrier replaces only the React Native package's unchanged Apple +# metadata input. Missing/stale state falls back to the normal producers. +unset OLIPHAUNT_REUSE_IOS_CARRIER +carrier_cache=target/ci/ios-carrier +if [[ $# == 0 && ${GITHUB_EVENT_NAME:-} != workflow_dispatch && ${CI_RELEASE_PRODUCTS_JSON:-[]} == '[]' && -s "$carrier_cache/source-sha" && -s "$carrier_cache/manifest.json" ]]; then + producer_sha="$(cat "$carrier_cache/source-sha")" + if [[ "$producer_sha" =~ ^[0-9a-f]{40}$ ]] && git cat-file -e "$producer_sha^{commit}" 2>/dev/null; then + git diff --name-only "$producer_sha" HEAD > "$plan_dir/carrier-changes" + if [[ -s "$plan_dir/carrier-changes" ]]; then + (unset MOON_BASE MOON_HEAD; "$moon_bin" query affected stdin --upstream none --downstream deep < "$plan_dir/carrier-changes") > "$plan_dir/carrier-affected.json" + else + printf '{"tasks":{}}\n' > "$plan_dir/carrier-affected.json" + fi + if bun -e 'const data=await Bun.file(process.argv[1]).json(); process.exit(Object.hasOwn(data.tasks ?? {}, "liboliphaunt-native:finalize-runtime-ios-abi") ? 1 : 0)' "$plan_dir/carrier-affected.json" && + bun src/native/sdks/swift/tools/ios-carrier-manifest.mts --base-carrier "$carrier_cache/manifest.json" --output "$plan_dir/carrier.json"; then + export OLIPHAUNT_REUSE_IOS_CARRIER=true + fi + fi +fi "$moon_bin" task-graph --json >"$OLIPHAUNT_MOON_TASK_GRAPH_FILE" if [[ $# == 0 && ${GITHUB_EVENT_NAME:-} != workflow_dispatch && (-z ${CI_RELEASE_PRODUCTS_JSON:-} || ${CI_RELEASE_PRODUCTS_JSON:-} == '[]') ]]; then : "${MOON_BASE:?MOON_BASE is required for affected CI planning}" diff --git a/tools/ci/ci-release-scope.test.sh b/tools/ci/ci-release-scope.test.sh index 972dfe7d3..ed75a22b0 100644 --- a/tools/ci/ci-release-scope.test.sh +++ b/tools/ci/ci-release-scope.test.sh @@ -25,7 +25,18 @@ set -eu case "$1" in --version) echo "moon $FIXTURE_MOON_VERSION" ;; task-graph) cat "$FIXTURE_TASK_GRAPH" ;; - query) [[ "$2" == projects ]]; cat "$FIXTURE_PROJECTS" ;; + query) + if [[ "$2" == projects ]]; then cat "$FIXTURE_PROJECTS"; + elif [[ ${FIXTURE_AFFECTED:-false} == true && "$2" == affected ]]; then + if [[ ${3:-} == stdin ]]; then + [[ -n "$(cat)" ]] || { echo 'empty stdin must not be queried as a Git ref' >&2; exit 82; } + if [[ ${FIXTURE_IOS_CHANGED:-false} == true ]]; then + printf '{"tasks":{"liboliphaunt-native:finalize-runtime-ios-abi":{}}}\n' + else printf '{"tasks":{}}\n'; fi + else + printf '{"projects":{"integration-examples":{"other":true}},"tasks":{"integration-examples:react-native-android-build":{"other":true}}}\n' + fi + else echo 'unexpected affected query' >&2; exit 82; fi ;; *) echo 'unexpected affected query' >&2; exit 82 ;; esac SH @@ -56,3 +67,44 @@ if bash "$root/tools/ci/ci-plan.sh" > "$scratch/invalid.out" 2> "$scratch/invali echo 'unknown release owner was accepted' >&2; exit 1 fi bash "$root/tools/dev/bun.sh" "$root/tools/ci/ci-release-scope.test.mts" assert "$scratch" "$head" + +# A missing or unusable optimization cache must keep the normal producer path. +# Same-SHA reuse has an empty diff, which Moon otherwise interprets as a ref. +git commit --allow-empty -qm 'fix: Android app' +export MOON_BASE="$head" GITHUB_REF=refs/heads/fixture GITHUB_EVENT_NAME=pull_request +MOON_HEAD="$(git rev-parse HEAD)"; export MOON_HEAD +export FIXTURE_AFFECTED=true +ln -s "$root/src" "$repo/src" +mkdir -p target/ci/ios-carrier +bun - "$root" "$scratch/carrier.json" <<'JS' +import {writeFileSync, readFileSync} from 'node:fs'; +const root = process.argv[2]; +const {iosBaseLegalMetadata} = await import(`${root}/src/native/sdks/swift/tools/ios-carrier-manifest.mts`); +const version = readFileSync(`${root}/src/native/runtime/VERSION`, 'utf8').trim(); +const tag = `liboliphaunt-native-v${version}`; +const assets = [ + ['base-xcframework', `liboliphaunt-${version}-apple-spm-xcframework.zip`, 'zip', 'liboliphaunt.xcframework'], + ['runtime-resources', `liboliphaunt-${version}-runtime-resources-ios-datum64.tar.gz`, 'tar.gz', 'oliphaunt'], +].map(([role, name, format, member]) => ({role, name, format, member, bytes:1, sha256:'a'.repeat(64), + url:`https://github.com/f0rr0/oliphaunt/releases/download/${tag}/${name}`})); +writeFileSync(process.argv[3], JSON.stringify({schema:'oliphaunt-react-native-ios-carrier-v1', + base:{product:'liboliphaunt-native', version, tag, assets}, carriers:[], extensions:[], + legal:{base:iosBaseLegalMetadata(), extensions:[]}})); +JS +for scenario in missing same-sha invalid-sha corrupt unchanged-ios changed-ios; do + expected=false + cp "$scratch/carrier.json" target/ci/ios-carrier/manifest.json + printf '%s\n' "$MOON_HEAD" > target/ci/ios-carrier/source-sha + export FIXTURE_IOS_CHANGED=false + case "$scenario" in + missing) rm target/ci/ios-carrier/manifest.json ;; + same-sha) expected=true ;; + invalid-sha) printf 'invalid\n' > target/ci/ios-carrier/source-sha ;; + corrupt) printf '{invalid\n' > target/ci/ios-carrier/manifest.json ;; + unchanged-ios) printf '%s\n' "$before" > target/ci/ios-carrier/source-sha; expected=true ;; + changed-ios) printf '%s\n' "$before" > target/ci/ios-carrier/source-sha; export FIXTURE_IOS_CHANGED=true ;; + esac + bash "$root/tools/ci/ci-plan.sh" > "$scratch/$scenario.out" 2> "$scratch/$scenario.err" + bun -e 'import assert from "node:assert/strict"; const output=await Bun.file(process.argv[2]).text(); assert(output.includes(`reuse_ios_carrier=${process.argv[1]}`)); const jobs=JSON.parse(output.match(/^jobs=(.+)$/m)[1]); assert.equal(jobs.includes("liboliphaunt-native-ios-abi"), process.argv[1] !== "true")' "$expected" "$scratch/$scenario.out" +done +echo 'Frozen iOS metadata: same-SHA reuse and cold, corrupt, changed-input fallbacks passed' diff --git a/tools/ci/ci_plan.mts b/tools/ci/ci_plan.mts index 31c62ddfb..992bc58ac 100644 --- a/tools/ci/ci_plan.mts +++ b/tools/ci/ci_plan.mts @@ -7,6 +7,11 @@ // targets are CI execution details, not source projects. import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; +import { qualificationRequestKey } from '../../.github/scripts/release-candidate-lib.mts'; +import { + publishedConsumerInventory, + validPublishedConsumerInventory, +} from '../../src/native/sdks/ts/tools/published-consumer.mts'; import { brokerRuntimeMatrix, extensionArtifactsNativeMatrix, @@ -29,13 +34,8 @@ import { extensionSqlNames, extensionSqlNamesForProducts, } from '../release/release-artifact-targets.mts'; -import { affectedNames, triggeringProjectNames, triggeringTaskNames } from './affected.mts'; import { loadProducts, moonProjectsById } from '../release/release-graph.mts'; -import { qualificationRequestKey } from '../../.github/scripts/release-candidate-lib.mts'; -import { - publishedConsumerInventory, - validPublishedConsumerInventory, -} from '../../src/native/sdks/ts/tools/published-consumer.mts'; +import { affectedNames, triggeringProjectNames, triggeringTaskNames } from './affected.mts'; const ROOT = path.resolve(import.meta.dir, '../..'); const PREFIX = 'ci_plan.mts'; @@ -166,7 +166,9 @@ const RELEASE_ONLY_TARGETS = new Set( .map((task) => task.target), ); export const BUILDER_JOBS = new Set( - Object.keys(CI_JOB_TARGETS).filter((job) => job !== NATIVE_EXTENSION_LIFECYCLE_JOB), + Object.keys(CI_JOB_TARGETS).filter( + (job) => ![NATIVE_EXTENSION_LIFECYCLE_JOB, 'wasix-release-regression'].includes(job), + ), ); const JOBS_BY_TARGET = (() => { const jobs = new Map(); @@ -179,6 +181,7 @@ const DEPENDENTS_BY_TARGET = (() => { const dependents = new Map(); for (const task of TASKS_BY_TARGET.values()) { for (const dependency of task.deps ?? []) { + if (dependency.cacheStrategy === 'ignored') continue; const target = typeof dependency === 'string' ? dependency : dependency.target; if (typeof target === 'string') { dependents.set(target, [...(dependents.get(target) ?? []), task.target]); @@ -272,18 +275,29 @@ export function addRequiredJobs(jobs) { return jobs; } -export function planJobsForAffected(tasks, excludedTargets = RELEASE_ONLY_TARGETS) { +export function planJobsForAffected( + tasks, + excludedTargets = RELEASE_ONLY_TARGETS, + reuseIosCarrier = process.env.OLIPHAUNT_REUSE_IOS_CARRIER === 'true', +) { const jobs = new Set(ALWAYS_JOBS); - const directlySelectedJobs = jobsForTargets(requiredTasksForAffected(tasks, excludedTargets), { - allowedJobs: ALL_BUILDER_JOBS, - }); + const directlySelectedJobs = jobsForTargets( + requiredTasksForAffected(tasks, excludedTargets, reuseIosCarrier), + { + allowedJobs: ALL_BUILDER_JOBS, + }, + ); for (const job of directlySelectedJobs) { jobs.add(job); } return jobs; } -export function requiredTasksForAffected(tasks, excludedTargets = RELEASE_ONLY_TARGETS) { +export function requiredTasksForAffected( + tasks, + excludedTargets = RELEASE_ONLY_TARGETS, + reuseIosCarrier = process.env.OLIPHAUNT_REUSE_IOS_CARRIER === 'true', +) { const selected = new Set( [...downstreamTaskClosure(tasks, excludedTargets)].filter((target) => JOBS_BY_TARGET.has(target), @@ -294,16 +308,18 @@ export function requiredTasksForAffected(tasks, excludedTargets = RELEASE_ONLY_T const target = pending.pop(); const task = TASKS_BY_TARGET.get(target); if (!task) fail(`affected Moon selection references missing target ${target}`); - const dependencies = taskDependencyTargets(task); - // Selecting the portable WASIX job also requires full same-run lifecycle - // evidence. Include every producer downloaded by wasix-release-regression. + const dependencies = taskDependencyTargets(task).filter( + (dependency) => + !( + reuseIosCarrier && + target === 'oliphaunt-react-native:package' && + dependency === 'liboliphaunt-native:finalize-runtime-ios-abi' + ), + ); + // Every planned portable runtime requires fresh lifecycle qualification. + // Its owner task declares the producer closure; do not repeat it here. if (task.tags?.includes('ci-liboliphaunt-wasix-runtime')) { - dependencies.push( - 'liboliphaunt-wasix:runtime-aot', - 'extension-artifacts-wasix:build-target', - 'extension-artifacts-wasix:build-aot', - 'postgres-tools-wasix:build-aot', - ); + dependencies.push(...CI_JOB_TARGETS['wasix-release-regression']); } for (const dependency of dependencies) { if (!selected.has(dependency)) { @@ -514,7 +530,7 @@ export function planForReleaseProducts( excludedTargets.delete('oliphaunt-js:test-consumer-published'); roots.add('oliphaunt-js:test-consumer-published'); } - const tasks = requiredTasksForAffected(roots, excludedTargets); + const tasks = requiredTasksForAffected(roots, excludedTargets, false); for (const target of downstreamTaskClosure(roots, excludedTargets)) { const task = TASKS_BY_TARGET.get(target); if ( @@ -533,7 +549,7 @@ export function planForReleaseProducts( } } } - const jobs = planJobsForAffected(roots, excludedTargets); + const jobs = planJobsForAffected(roots, excludedTargets, false); const selectedExtensionProducts = selectedExtensionProductsForPlan(projects, roots, jobs); const plan = renderPlanWithSelection({ jobs, @@ -700,8 +716,8 @@ export function planForFullRun({ new Set(['liboliphaunt-wasix-runtime', 'liboliphaunt-wasix-aot']), ); if (wasmTarget === 'linux-x64-gnu') { - // The workflow selects release regression for the Linux host target. - focusedJobs.add('extension-artifacts-wasix'); + focusedJobs.add('wasix-release-regression'); + addRequiredJobs(focusedJobs); } return { jobs: focusedJobs, @@ -716,7 +732,7 @@ export function planForFullRun({ BASE_JOBS, BUILDER_JOBS, WASM_RUNTIME_JOBS, - new Set([NATIVE_EXTENSION_LIFECYCLE_JOB]), + new Set([NATIVE_EXTENSION_LIFECYCLE_JOB, 'wasix-release-regression']), ); addRequiredJobs(jobs); return { @@ -791,9 +807,8 @@ export function extensionArtifactsNativeMatrixForPlan( jobs.has('extension-packages') ? undefined : (selectedTargets ?? undefined), selectedExtensionProducts ?? undefined, ); - if (!jobs.has(NATIVE_EXTENSION_LIFECYCLE_JOB)) { - return matrix; - } + const android = matrix.include.some((row) => row.target.startsWith('android-')); + if (!jobs.has(NATIVE_EXTENSION_LIFECYCLE_JOB) && !android) return matrix; const exactProducts = new Set(exactExtensionProducts()); const requiredTargets = new Set(['linux-x64-gnu']); @@ -897,6 +912,11 @@ export function renderPlanWithSelection({ jobs: sorted(jobs), builder_jobs: sorted(new Set([...jobs].filter((job) => BUILDER_JOBS.has(job)))), e2e_jobs: mobileE2eJobsForPlan(jobs), + reuse_ios_carrier: + qualificationMode === AFFECTED_QUALIFICATION_MODE && + process.env.OLIPHAUNT_REUSE_IOS_CARRIER === 'true' && + jobs.has('react-native-sdk-package') && + !jobs.has('liboliphaunt-native-ios-abi'), job_targets: jobTargetsForJobs( jobs, qualificationMode === PRODUCT_QUALIFICATION_MODE @@ -997,12 +1017,13 @@ export function renderPlanWithSelection({ ['extension_artifacts_native_matrix', ['linux', 'android', 'ios', 'other']], ['liboliphaunt_native_desktop_runtime_matrix', ['linux', 'other']], ['broker_runtime_matrix', ['linux', 'other']], + ['liboliphaunt_wasix_aot_runtime_matrix', ['linux', 'other']], ]) { for (const group of groups) { plan[`${matrix}_${group}`] = { include: plan[matrix].include.filter((row) => { const family = - row.target === 'linux-x64-gnu' + (row.target_id ?? row.target) === 'linux-x64-gnu' ? 'linux' : groups.includes('android') && row.target.startsWith('android-') ? 'android' @@ -1015,6 +1036,8 @@ export function renderPlanWithSelection({ } } + plan.wasix_napi_targets = plan.wasix_napi_runtime_matrix.include.map((row) => row.target); + for (const family of ['android', 'ios']) { plan[`mobile_extension_package_native_targets_${family}_csv`] = plan.mobile_extension_package_native_targets diff --git a/tools/ci/moon.yml b/tools/ci/moon.yml index 2a3a7d10f..ad7d22997 100644 --- a/tools/ci/moon.yml +++ b/tools/ci/moon.yml @@ -11,6 +11,6 @@ tasks: test: tags: ["quality", "unit"] command: "bash tools/ci/start-android-emulator-ci.test.sh" - inputs: ["start-android-emulator-ci.sh", "start-android-emulator-ci.test.sh"] + inputs: ["start-android-emulator-ci.sh", "start-android-emulator-ci.test.sh", "/tools/dev/acquisition.sh"] options: runFromWorkspaceRoot: true diff --git a/tools/ci/start-android-emulator-ci.sh b/tools/ci/start-android-emulator-ci.sh index cdb84dc38..2d8cbcaa7 100755 --- a/tools/ci/start-android-emulator-ci.sh +++ b/tools/ci/start-android-emulator-ci.sh @@ -58,8 +58,11 @@ need_cmd avdmanager need_cmd adb ensure_kvm_access -yes | sdkmanager --licenses >/dev/null || true -sdkmanager --install "emulator" "$image" +# Package acquisition and emulator boot have separate budgets. +. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../dev" && pwd)/acquisition.sh" +oliphaunt_acquisition_start 'Android emulator packages' 1800 +yes | oliphaunt_acquisition_run 60 sdkmanager --licenses >/dev/null || true +oliphaunt_acquisition_run 1800 sdkmanager --install "emulator" "$image" need_cmd emulator mkdir -p "$ANDROID_AVD_HOME" diff --git a/tools/ci/workflow-caches.test.mts b/tools/ci/workflow-caches.test.mts new file mode 100644 index 000000000..76f444bfa --- /dev/null +++ b/tools/ci/workflow-caches.test.mts @@ -0,0 +1,155 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(import.meta.dir, '../..'); +const yaml = (file) => Bun.YAML.parse(readFileSync(path.join(root, file), 'utf8')); +const workflow = yaml('.github/workflows/ci.yml'); +const common = 'source src/wasix/postmaster/lib/common.sh\n'; + +test('the shared builder label satisfies Postmaster without a second Docker build', () => { + const scratch = mkdtempSync(path.join(tmpdir(), 'oliphaunt-builder-cache-')); + try { + const action = yaml('.github/actions/setup-wasix-builder/action.yml'); + const output = path.join(scratch, 'output'); + execFileSync('bash', ['-eu', '-c', action.runs.steps[0].run], { + cwd: root, + env: { ...process.env, GITHUB_OUTPUT: output }, + }); + const recipe = readFileSync(output, 'utf8').trim().split('=')[1]; + assert.match(recipe, /^[a-f0-9]{64}$/u); + const build = action.runs.steps.find(({ uses }) => + uses?.startsWith('docker/build-push-action@'), + ); + const label = build.with.labels.replace('${{ steps.recipe.outputs.sha256 }}', recipe); + const docker = path.join(scratch, 'docker'); + // A cache hit may inspect the image, but must not invoke Docker build. + writeFileSync( + docker, + `#!/usr/bin/env bash +set -eu +test "$1 $2" = 'image inspect' +test "$4" = '{{ index .Config.Labels "dev.oliphaunt.wasix-builder.recipe-sha256" }}' +test "$5" = "$EXPECTED_IMAGE" +test "\${BUILDER_LABEL%%=*}" = dev.oliphaunt.wasix-builder.recipe-sha256 +printf '%s\\n' "\${BUILDER_LABEL#*=}" +`, + { mode: 0o755 }, + ); + const image = execFileSync( + 'bash', + ['-eu', '-c', `${common}printf '%s' "$FRESH_WASIX_DOCKER_IMAGE"`], + { + cwd: root, + encoding: 'utf8', + }, + ); + assert.ok(build.with.tags.trim().split('\n').includes(image)); + execFileSync( + 'bash', + [ + '-eu', + '-c', + `${common}fresh_docker_bin() { printf '%s' "$FAKE_DOCKER"; }; fresh_ensure_docker_image`, + ], + { + cwd: root, + env: { ...process.env, FAKE_DOCKER: docker, BUILDER_LABEL: label, EXPECTED_IMAGE: image }, + }, + ); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +}); + +test('Postmaster caches Cargo target directories without cleaning their shared parent', () => { + const targets = execFileSync( + 'bash', + [ + '-eu', + '-c', + `${common}printf '%s\\n' "$FRESH_WORK_ROOT/runtime/wasmer/target" "$FRESH_POSTMASTER_EXECUTOR_TARGET_DIR" "$FRESH_POSTMASTER_COMPILER_TARGET_DIR"`, + ], + { cwd: root, encoding: 'utf8' }, + ) + .trim() + .split('\n'); + for (const id of ['wasix-postmaster-portable', 'wasix-postmaster-target']) { + const steps = workflow.jobs[id].steps; + const rustIndex = steps.findIndex(({ uses }) => uses === './.github/actions/setup-rust'); + const mappings = steps[rustIndex].with['cache-workspaces'] + .trim() + .split('\n') + .map((line) => { + const [workspace, target] = line.split('->').map((part) => part.trim()); + return { + workspace: path.resolve(root, workspace), + target: path.resolve(root, workspace, target), + }; + }); + for (const target of targets) + assert.ok( + mappings.some((mapping) => mapping.target === target), + `${id}: missing ${target}`, + ); + for (const a of mappings) { + for (const b of mappings) { + if (a !== b) + assert.ok( + !b.target.startsWith(`${a.target}${path.sep}`), + 'a parent Cargo cleanup would delete the nested cache', + ); + } + } + for (const { workspace } of mappings) + assert.ok(readFileSync(path.join(workspace, 'Cargo.toml'), 'utf8')); + } +}); + +test('Cargo cache metadata needs no generated Wasmer checkout at restore time', () => { + const scratch = mkdtempSync(path.join(tmpdir(), 'oliphaunt-cache-workspace-')); + try { + const executor = path.join(root, 'src/wasix/postmaster/executor'); + for (const entry of ['Cargo.toml', 'Cargo.lock', 'src']) + cpSync(path.join(executor, entry), path.join(scratch, entry), { recursive: true }); + const lock = readFileSync(path.join(scratch, 'Cargo.lock'), 'utf8'); + // rust-cache collects workspace members without dependency resolution on restore. + const metadata = JSON.parse( + execFileSync('cargo', ['metadata', '--all-features', '--no-deps', '--format-version', '1'], { + cwd: scratch, + encoding: 'utf8', + }), + ); + assert.equal(metadata.workspace_members.length, 1); + assert.equal(readFileSync(path.join(scratch, 'Cargo.lock'), 'utf8'), lock); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +}); + +test('Rust setup follows the repository pin and rejects an unpinned override', () => { + const scratch = mkdtempSync(path.join(tmpdir(), 'oliphaunt-rust-pin-')); + try { + const step = yaml('.github/actions/setup-rust-tools/action.yml').runs.steps.find( + ({ id }) => id === 'toolchain', + ); + const output = path.join(scratch, 'output'); + writeFileSync(path.join(scratch, 'rust-toolchain.toml'), '[toolchain]\nchannel = "9.8.7"\n'); + const run = (version) => + execFileSync('bash', ['-eu', '-c', step.run], { + cwd: scratch, + env: { ...process.env, TOOLCHAIN_INPUT: version, GITHUB_OUTPUT: output }, + stdio: 'pipe', + }); + run(''); + assert.equal(readFileSync(output, 'utf8'), 'version=9.8.7\n'); + run('1.99.0'); + assert.match(readFileSync(output, 'utf8'), /version=1[.]99[.]0/u); + assert.throws(() => run('stable'), /expected a pinned Rust version/u); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +}); diff --git a/tools/ci/workflow-moon-transfers.test.mts b/tools/ci/workflow-moon-transfers.test.mts index 742dd29fe..a8c79a96f 100644 --- a/tools/ci/workflow-moon-transfers.test.mts +++ b/tools/ci/workflow-moon-transfers.test.mts @@ -13,8 +13,8 @@ import { } from 'node:fs'; import path from 'node:path'; import test from 'node:test'; -import { CI_JOB_TARGETS } from './ci_plan.mts'; import { resolveExecution } from '../../.github/scripts/resolve-planned-moon-execution.mts'; +import { CI_JOB_TARGETS } from './ci_plan.mts'; const ROOT = path.resolve(import.meta.dir, '../..'); const workflow = Bun.YAML.parse(readFileSync(path.join(ROOT, '.github/workflows/ci.yml'), 'utf8')); @@ -27,6 +27,64 @@ const tasks = new Map( ]), ); +if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) + test('workflow-wide environment does not force product builds during planning', () => { + for (const task of tasks.values()) { + for (const input of task.inputs ?? []) { + if (typeof input !== 'string' || !input.startsWith('$')) continue; + assert( + !Object.hasOwn(workflow.env ?? {}, input.slice(1)), + `${input} affects ${task.target}; scope it to the consuming build job`, + ); + } + } + }); + +if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) + test('jobs without a Moon cache have no cacheable local task subtree', () => { + for (const [id, job] of Object.entries(workflow.jobs)) { + if ( + !job.steps?.some( + (step) => + step.uses === './.github/actions/setup-moon' && step.with?.['task-cache'] === 'false', + ) + ) + continue; + const roots = CI_JOB_TARGETS[id] ?? []; + if (!roots.length) { + assert( + !job.steps.some((step) => + /run-(?:planned-moon-job|moon-targets)[.]sh/u.test(step.run ?? ''), + ), + `${id} executes unclassified Moon work`, + ); + continue; + } + const transferred = [ + ...new Set( + job.steps.flatMap((step) => + JSON.parse(step.env?.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON ?? '[]'), + ), + ), + ]; + const execution = resolveExecution(roots, transferred, tasks); + assert.deepEqual( + execution.localDependencies, + [], + `${id} has local work that could reuse the cache`, + ); + // The adapter always executes paths consuming transferred artifacts with + // MOON_CACHE=off; only localDependencies use normal Moon caching. + assert.ok(execution.transferred.length > 0, `${id} runs without an artifact boundary`); + } + const action = Bun.YAML.parse( + readFileSync(path.join(ROOT, '.github/actions/setup-moon/action.yml'), 'utf8'), + ); + assert.equal(action.inputs['task-cache'].default, 'true'); + const cache = action.runs.steps.find((step) => step.uses?.startsWith('actions/cache@')); + assert.equal(cache.if, `\${{ inputs.task-cache == 'true' }}`); + }); + if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) test('artifact production waits for source check and test results without a dependency cycle', () => { const jobs = workflow.jobs; @@ -102,6 +160,34 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) assert.deepEqual(execution.targets, ['extension-packages:package']); }); +if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) + test('WASIX consumers build query once before running against transferred SDK artifacts', () => { + const step = workflow.jobs['wasix-ts-sdk-package'].steps.find( + (step) => step.name === 'Test WASIX TypeScript consumers', + ); + const execution = resolveExecution( + CI_JOB_TARGETS['wasix-ts-sdk-package'], + JSON.parse(step.env.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON), + tasks, + ); + // SDK transfers cut off their upstream query build, so every independently + // selectable consumer must retain its own edge to the shared producer. + for (const target of CI_JOB_TARGETS['wasix-ts-sdk-package']) { + assert(dependencies(target).includes('oliphaunt-query-ts:build'), target); + } + assert.deepEqual(execution.localDependencies, ['oliphaunt-query-ts:build']); + for (const target of [...execution.localDependencies, ...execution.targets]) { + assert( + ![ + 'wasix-browser-host:build', + 'oliphaunt-wasix-ts:build', + 'oliphaunt-wasix-ts:package', + ].includes(target), + target, + ); + } + }); + function dependencies(target) { const task = tasks.get(target); assert.ok(task, `workflow root ${target} must exist in Moon`); @@ -119,7 +205,13 @@ function dependencies(target) { if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) test('downloaded Moon dependencies are explicit reachable handoffs', () => { - for (const [workflowJob, job] of Object.entries(workflow.jobs)) { + const host = Bun.YAML.parse( + readFileSync(path.join(ROOT, '.github/workflows/wasix-host.yml'), 'utf8'), + ); + for (const [workflowJob, job] of Object.entries({ + ...workflow.jobs, + 'wasix-host': { ...host.jobs.build, needs: ['liboliphaunt-wasix-runtime'] }, + })) { const steps = job.steps ?? []; for (const [index, step] of steps.entries()) { const run = String(step.run ?? ''); @@ -130,7 +222,9 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) const rawTransfers = step.env?.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON; if (rawTransfers === undefined) continue; - const plannedJob = run.match(/run-planned-moon-job[.]sh ([a-z0-9-]+)/u)?.[1]; + const plannedJob = + run.match(/run-planned-moon-job[.]sh ([a-z0-9-]+)/u)?.[1] ?? + (run.includes('collect-wasix-evidence.sh') ? 'wasix-release-regression' : undefined); assert.ok(plannedJob, `${workflowJob} transferred handoff must use the planned-job runner`); const transfers = JSON.parse(rawTransfers); assert.ok(Array.isArray(transfers) && transfers.length > 0); @@ -180,6 +274,59 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) } }); +if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) + test('SDK runtime suites execute in artifact-consuming jobs without rebuilding their producers', () => { + for (const [job, roots, forbidden] of [ + [ + 'native-consumers', + [ + 'oliphaunt-rust:test-integration', + 'oliphaunt-mobile-bindings:test-native', + 'oliphaunt-swift:test-native', + 'oliphaunt-kotlin:test-native-bindings', + ], + [ + 'liboliphaunt-native:build-runtime-desktop-target', + 'oliphaunt-broker:build-release-assets', + ], + ], + [ + 'wasix-release-regression', + ['oliphaunt-wasix-rust:test-integration'], + [ + 'liboliphaunt-wasix:compiler-output', + 'liboliphaunt-wasix:runtime-aot', + 'database-resources:build-icu-data', + ], + ], + ]) { + const step = workflow.jobs[job].steps.find( + (step) => step.env?.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON, + ); + const available = JSON.parse(step.env.OLIPHAUNT_MOON_TRANSFERRED_DEPS_JSON); + for (const root of roots) { + assert(CI_JOB_TARGETS[job].includes(root), `${root} has no executable hosted owner`); + const reachable = new Set(dependencies(root)); + for (const dependency of reachable) + for (const parent of dependencies(dependency)) reachable.add(parent); + const execution = resolveExecution( + [root], + available.filter((target) => reachable.has(target)), + tasks, + ); + assert(execution.targets.includes(root)); + const executed = [...execution.targets, ...execution.localDependencies]; + for (const target of forbidden) + assert(!executed.includes(target), `${root} rebuilds ${target}`); + assert.equal( + tasks.get(root).options.cache, + false, + `${root} must execute against this run's artifacts`, + ); + } + } + }); + const phase = process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE; const scratch = process.argv[2]; if (phase) { diff --git a/tools/dev/acquisition.sh b/tools/dev/acquisition.sh new file mode 100644 index 000000000..1e9e02c1b --- /dev/null +++ b/tools/dev/acquisition.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env sh +# One budget per acquisition, shared by retries, mirrors and lock waits. +# POSIX shell and curl suffice for bootstrap downloads; other subprocesses use +# GNU timeout (already required for source Git operations). + +oliphaunt_acquisition_start() { + oliphaunt_acquisition_label=$1 + oliphaunt_acquisition_seconds=${OLIPHAUNT_ACQUISITION_TIMEOUT_SECONDS:-$2} + case "$oliphaunt_acquisition_seconds" in + ''|0*|*[!0-9]*|?????*) echo 'acquisition timeout must be an integer from 1 to 7200 seconds' >&2; return 2 ;; + esac + if [ "$oliphaunt_acquisition_seconds" -lt 1 ] || [ "$oliphaunt_acquisition_seconds" -gt 7200 ]; then + echo 'acquisition timeout must be an integer from 1 to 7200 seconds' >&2 + return 2 + fi + oliphaunt_acquisition_end=$(( $(date +%s) + oliphaunt_acquisition_seconds )) + if [ -n "${oliphaunt_acquisition_deadline:-}" ] && + [ "$oliphaunt_acquisition_deadline" -lt "$oliphaunt_acquisition_end" ]; then + oliphaunt_acquisition_end=$oliphaunt_acquisition_deadline + fi + oliphaunt_acquisition_deadline=$oliphaunt_acquisition_end +} + +oliphaunt_acquisition_remaining() ( + remaining=$(( ${oliphaunt_acquisition_deadline:?start an acquisition first} - $(date +%s) )) + if [ "$remaining" -le 0 ]; then + echo "acquisition deadline exhausted: $oliphaunt_acquisition_label" >&2 + exit 124 + fi + [ "$remaining" -le "$1" ] || remaining=$1 + printf '%s\n' "$remaining" +) + +oliphaunt_acquisition_sleep() ( + remaining=$(oliphaunt_acquisition_remaining 7200) || exit $? + if [ "$1" -ge "$remaining" ]; then + echo "acquisition retry would exceed deadline: $oliphaunt_acquisition_label" >&2 + exit 124 + fi + sleep "$1" +) + +# Git for Windows can have System32/timeout.exe ahead of GNU timeout in PATH. +# Prefer Coreutils and fall back to the shell's bundled /usr/bin copy. +oliphaunt_acquisition_timeout() ( + for candidate in gtimeout timeout /usr/bin/timeout; do + timer=$(command -v "$candidate") || continue + case "$("$timer" --version 2>/dev/null)" in + *'GNU coreutils'*) printf '%s\n' "$timer"; exit 0 ;; + esac + done + echo 'acquisition requires GNU timeout (brew install coreutils on macOS; use Git Bash on Windows)' >&2 + exit 127 +) + +oliphaunt_acquisition_run() ( + timer=$(oliphaunt_acquisition_timeout) || exit $? + seconds=$(oliphaunt_acquisition_remaining "$1") || exit $? + shift + status=0 + "$timer" --kill-after=5 "${seconds}s" "$@" || status=$? + if [ "$status" = 124 ] || [ "$status" = 137 ]; then + echo "acquisition command timed out after ${seconds}s: $oliphaunt_acquisition_label" >&2 + fi + exit "$status" +) + +# CAP ATTEMPTS DELAY CURL ARGS...; CAP bounds this endpoint, not each retry. +# curl's retry-max-time allows a final transfer to overrun its timer. Run single +# attempts with the remaining time instead, without needing a bootstrap runtime. +oliphaunt_acquisition_curl() ( + cap=$1 attempts=$2 delay=$3 curl_command=$4 + shift 4 + endpoint_deadline=$(( $(date +%s) + cap )) + if [ "$endpoint_deadline" -lt "$oliphaunt_acquisition_deadline" ]; then + oliphaunt_acquisition_deadline=$endpoint_deadline + fi + attempt=1 status=0 + while [ "$attempt" -le "$attempts" ]; do + seconds=$(oliphaunt_acquisition_remaining "$cap") || exit $? + if "$curl_command" --disable --retry 0 --max-time "$seconds" "$@"; then + exit 0 + else + status=$? + fi + # Cancellation is not a transient transport failure. + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac + oliphaunt_acquisition_remaining "$cap" >/dev/null || exit $? + [ "$attempt" -lt "$attempts" ] || break + oliphaunt_acquisition_sleep "$delay" || exit $? + attempt=$((attempt + 1)) + done + exit "$status" +) diff --git a/tools/dev/acquisition.test.sh b/tools/dev/acquisition.test.sh new file mode 100644 index 000000000..012b9dc7a --- /dev/null +++ b/tools/dev/acquisition.test.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(git rev-parse --show-toplevel)" +. "$root/tools/dev/acquisition.sh" +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT + +# A fake clock exercises shared budgets without waiting through retry delays. +printf '100\n' > "$scratch/clock" +date() { cat "$scratch/clock"; } +sleep() { printf '%s\n' "$(( $(date +%s) + $1 ))" > "$scratch/clock"; } +unset OLIPHAUNT_ACQUISITION_TIMEOUT_SECONDS oliphaunt_acquisition_deadline +oliphaunt_acquisition_start fixture 20 +for invalid in '' 0 -1 08 abc 7201 99999999999999999999; do + if OLIPHAUNT_ACQUISITION_TIMEOUT_SECONDS="$invalid" oliphaunt_acquisition_start invalid 20 2>/dev/null; then + # An empty override deliberately selects the caller's default. + [ -z "$invalid" ] || exit 1 + fi +done +oliphaunt_acquisition_start fixture 20 +curl_attempt() { + local seconds="" argument + while [ "$#" -gt 0 ]; do + argument=$1; shift + case "$argument" in + --max-time) seconds=$1; shift ;; + --retry) [ "$1" = 0 ]; shift ;; + esac + done + printf '%s\n' "$seconds" >> "$scratch/attempts" + # Model an attempt that consumes seven seconds then loses its connection. + sleep 7 + return 7 +} +status=0 +oliphaunt_acquisition_curl 20 10 3 curl_attempt || status=$? +[ "$status" = 124 ] +printf '20\n10\n' > "$scratch/expected" +cmp "$scratch/expected" "$scratch/attempts" +[ "$(date +%s)" = 117 ] +# An interrupted transfer must not be retried. +curl_interrupted() { printf 'called\n' >> "$scratch/cancelled"; return 143; } +status=0 +oliphaunt_acquisition_curl 20 10 0 curl_interrupted || status=$? +[ "$status" = 143 ] && [ "$(wc -l < "$scratch/cancelled")" -eq 1 ] +# Starting a mirror/sub-operation cannot replenish its parent's remaining time. +oliphaunt_acquisition_start mirror 100 +[ "$(oliphaunt_acquisition_remaining 100)" = 3 ] +printf '121\n' > "$scratch/clock" +status=0 +oliphaunt_acquisition_run 60 touch "$scratch/late-command" || status=$? +[ "$status" = 124 ] && [ ! -e "$scratch/late-command" ] + +# Exhaustion on the final attempt is still a deadline, not a fallback-triggering +# transport error. Endpoint expiry leaves the parent's budget for another mirror. +printf '100\n' > "$scratch/clock" +unset oliphaunt_acquisition_deadline +oliphaunt_acquisition_start endpoint 20 +status=0 +oliphaunt_acquisition_curl 7 1 0 curl_attempt || status=$? +[ "$status" = 124 ] +[ "$(oliphaunt_acquisition_remaining 20)" = 13 ] + +# A non-GNU program named timeout (Windows System32) must not mask Coreutils. +# Intercept discovery so this also tests a Mac with only gtimeout installed. +real_timeout=$(oliphaunt_acquisition_timeout) +for available in gtimeout /usr/bin/timeout missing; do + command() { + [ "$1" = -v ] || return 2 + if [ "$2" = "$available" ]; then printf '%s\n' "$real_timeout" + elif [ "$2" = timeout ]; then printf '%s\n' false + else return 1; fi + } + if [ "$available" = missing ]; then + status=0 + oliphaunt_acquisition_timeout 2>/dev/null || status=$? + [ "$status" = 127 ] + else + [ "$(oliphaunt_acquisition_timeout)" = "$real_timeout" ] + fi + unset -f command +done + +# Real GNU timeout must terminate descendants, preserving ordinary exit codes. +unset -f date sleep +unset oliphaunt_acquisition_deadline +oliphaunt_acquisition_start process 30 +status=0 +oliphaunt_acquisition_run 10 sh -c 'exit 7' || status=$? +[ "$status" = 7 ] +status=0 +oliphaunt_acquisition_run 1 sh -c 'sleep 3; touch "$1"' sh "$scratch/orphan" || status=$? +[ "$status" = 124 ] +sleep 3 +[ ! -e "$scratch/orphan" ] +echo 'Acquisition deadlines: shared retries, nested budgets, expired admission and process cleanup passed' diff --git a/tools/dev/android-sdk.toml b/tools/dev/android-sdk.toml index ce8f41c36..3a5fea936 100644 --- a/tools/dev/android-sdk.toml +++ b/tools/dev/android-sdk.toml @@ -2,6 +2,8 @@ command_line_tools_build = "14742923" command_line_tools_revision = "20.0" ndk = "27.0.12077973" +# React Native / Expo app toolchain; native runtime artifacts retain their own NDK. +expo_ndk = "27.1.12297006" cmake = "3.22.1" compile_sdk = "36" build_tools = "36.0.0" diff --git a/tools/dev/extract-maestro.mts b/tools/dev/extract-maestro.mts deleted file mode 100644 index 50f6a4456..000000000 --- a/tools/dev/extract-maestro.mts +++ /dev/null @@ -1,27 +0,0 @@ -import { mkdirSync, writeFileSync } from 'node:fs'; -import path from 'node:path'; -import { readPortableArchiveEntries } from '../packaging/portable-archive.mts'; -const [archive, destination, version] = process.argv.slice(2); -try { - const entries = readPortableArchiveEntries(archive, { - maxArchiveBytes: 400_000_000, - maxExpandedBytes: 800_000_000, - maxEntries: 4096, - }); - for (const name of ['maestro/bin/maestro', 'maestro/lib/maestro-cli-' + version + '.jar']) { - if (!entries.get(name)?.isFile || !entries.get(name)?.size) - throw new Error('missing expected archive entry: ' + name); - } - for (const entry of entries.values()) { - if (entry.name !== 'maestro' && !entry.name.startsWith('maestro/')) - throw new Error('unsafe Maestro archive path: ' + entry.name); - } - for (const entry of entries.values()) { - const output = path.join(destination, entry.name); - mkdirSync(entry.isDirectory ? output : path.dirname(output), { recursive: true }); - if (entry.isFile) writeFileSync(output, entry.data(), { flag: 'wx', mode: 0o644 }); - } -} catch (error) { - console.error(`invalid Maestro archive: ${error.message}`); - process.exitCode = 1; -} diff --git a/tools/dev/install-pinned-js-runtime.sh b/tools/dev/install-pinned-js-runtime.sh index 3be389cf7..c90f4044c 100755 --- a/tools/dev/install-pinned-js-runtime.sh +++ b/tools/dev/install-pinned-js-runtime.sh @@ -48,6 +48,8 @@ if [ ! -f "$curl_platform_flags" ] || [ -L "$curl_platform_flags" ]; then fi # shellcheck source=tools/dev/curl-platform-flags.sh disable=SC1091 . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "$tool bootstrap" 300 manifest_value() { local section="$1" @@ -284,20 +286,22 @@ for candidate_url in "$url" ${mirror_url:+"$mirror_url"}; do curl_args=( --fail --location --silent --show-error --proto '=https' --proto-redir '=https' - --retry 6 --retry-all-errors --retry-max-time 120 - --connect-timeout 20 --max-time 180 --max-filesize 200000000 + --connect-timeout 20 --max-filesize 200000000 ) if [ -n "$curl_platform_tls_flag" ]; then curl_args+=("$curl_platform_tls_flag") fi curl_args+=(--remove-on-error --output "$archive" "$candidate_url") - if "${OLIPHAUNT_PINNED_TOOL_CURL:-curl}" "${curl_args[@]}"; then + if oliphaunt_acquisition_curl 120 7 2 "${OLIPHAUNT_PINNED_TOOL_CURL:-curl}" "${curl_args[@]}"; then actual_archive_sha256="$(sha256_file "$archive")" if [ "$actual_archive_sha256" = "$archive_sha256" ]; then downloaded=1 break fi echo "$tool archive checksum mismatch from $candidate_url; trying the next pinned origin" >&2 + else + status=$? + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac fi done [ "$downloaded" = "1" ] || fail "could not download the verified $tool $version $target archive" diff --git a/tools/dev/install-pinned-js-runtime.test.sh b/tools/dev/install-pinned-js-runtime.test.sh index a91a6839b..de0cbf5da 100755 --- a/tools/dev/install-pinned-js-runtime.test.sh +++ b/tools/dev/install-pinned-js-runtime.test.sh @@ -4,6 +4,10 @@ set -euo pipefail root="$(git rev-parse --show-toplevel)" installer="$root/tools/dev/install-pinned-js-runtime.sh" tmp="$(mktemp -d)" +mkdir -p "$tmp/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$tmp/no-delay/sleep" +chmod +x "$tmp/no-delay/sleep" +export PATH="$tmp/no-delay:$PATH" trap 'rm -rf "$tmp"' EXIT HUP INT TERM diff --git a/tools/dev/install-pinned-maintainer-tool.sh b/tools/dev/install-pinned-maintainer-tool.sh index ac899f21a..36b7ccacd 100755 --- a/tools/dev/install-pinned-maintainer-tool.sh +++ b/tools/dev/install-pinned-maintainer-tool.sh @@ -272,6 +272,8 @@ if cache_valid_release; then exit 0 fi +. "$(dirname "${BASH_SOURCE[0]}")/acquisition.sh" +oliphaunt_acquisition_start "$tool bootstrap" 180 curl_command="${OLIPHAUNT_MAINTAINER_TOOLS_CURL:-curl}" command -v "$curl_command" >/dev/null 2>&1 || { echo "missing required download command: $curl_command" >&2; exit 1; } command -v mktemp >/dev/null 2>&1 || { echo "missing required command: mktemp" >&2; exit 1; } @@ -282,17 +284,12 @@ archive="$temporary_root/archive.partial" extract_root="$temporary_root/extracted" mkdir -p "$extract_root" set +e -"$curl_command" \ +oliphaunt_acquisition_curl 180 5 2 "$curl_command" \ --fail \ --location \ --silent \ --show-error \ - --retry 4 \ - --retry-all-errors \ - --retry-delay 2 \ - --retry-max-time 120 \ --connect-timeout 20 \ - --max-time 180 \ --max-filesize "$max_archive_bytes" \ --proto '=https' \ --proto-redir '=https' \ diff --git a/tools/dev/install-pinned-winflexbison.sh b/tools/dev/install-pinned-winflexbison.sh index fe1c39460..eb04b41d4 100755 --- a/tools/dev/install-pinned-winflexbison.sh +++ b/tools/dev/install-pinned-winflexbison.sh @@ -28,6 +28,8 @@ esac fail "missing regular curl platform policy: $curl_platform_flags" # shellcheck source=tools/dev/curl-platform-flags.sh disable=SC1091 . "$curl_platform_flags" +. "${curl_platform_flags%/*}/acquisition.sh" +oliphaunt_acquisition_start "winflexbison bootstrap" 180 manifest_value() { local section="$1" @@ -176,13 +178,12 @@ trap 'exit 143' TERM curl_args=( --fail --location --silent --show-error --proto '=https' --proto-redir '=https' - --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 120 - --connect-timeout 20 --max-time 180 --max-filesize 2000000 + --connect-timeout 20 --max-filesize 2000000 ) curl_platform_tls_flag="$(oliphaunt_curl_platform_tls_flag)" if [ -n "$curl_platform_tls_flag" ]; then curl_args+=("$curl_platform_tls_flag"); fi curl_args+=(--remove-on-error --output "$archive" "$url") -"$curl_command" "${curl_args[@]}" || fail "could not download pinned winflexbison $version" +oliphaunt_acquisition_curl 180 6 2 "$curl_command" "${curl_args[@]}" || fail "could not download pinned winflexbison $version" actual_bytes="$(wc -c <"$archive" | tr -d '[:space:]')" [ "$actual_bytes" = "$archive_bytes" ] || fail "winflexbison archive size mismatch: expected $archive_bytes, got $actual_bytes" diff --git a/tools/dev/install-pinned-winflexbison.test.sh b/tools/dev/install-pinned-winflexbison.test.sh index 7aef06433..3cd06fd0f 100755 --- a/tools/dev/install-pinned-winflexbison.test.sh +++ b/tools/dev/install-pinned-winflexbison.test.sh @@ -5,6 +5,10 @@ root="$(git rev-parse --show-toplevel)" installer="$root/tools/dev/install-pinned-winflexbison.sh" extractor="$root/tools/dev/extract-pinned-zip.sh" tmp="$(mktemp -d)" +mkdir -p "$tmp/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$tmp/no-delay/sleep" +chmod +x "$tmp/no-delay/sleep" +export PATH="$tmp/no-delay:$PATH" trap 'rm -rf "$tmp"' EXIT HUP INT TERM mkdir -p "$tmp/fixtures" "$tmp/config" "$tmp/bin" @@ -70,7 +74,7 @@ payload="$(run_installer)" [ -x "$payload/win_flex.exe" ] [ -x "$payload/win_bison.exe" ] [ -f "$payload/data/README.md" ] -grep -Fxq -- "--retry-all-errors" "$tmp/curl-args.log" +grep -Fxq -- "--retry" "$tmp/curl-args.log" grep -Fxq -- "=https" "$tmp/curl-args.log" # A complete verified cache is network-independent. diff --git a/tools/dev/maestro.toml b/tools/dev/maestro.toml deleted file mode 100644 index 320d6448a..000000000 --- a/tools/dev/maestro.toml +++ /dev/null @@ -1,15 +0,0 @@ -[toolchain] -maestro = "2.6.0" -install_url = "https://github.com/mobile-dev-inc/Maestro/releases/download/cli-2.6.0/maestro.zip" -sha256 = "80185105a5d7e227e3b3fbcf225f45b312508ea676a9fc8e1b1aa1cac8b9ff6e" -license = "Apache-2.0" -cloud_required = false - -[decision] -status = "accepted" -date = "2026-06-08" -scope = "react-native-installed-app-e2e" -runner = "open-source-maestro-cli" -reopen_only_if = "Replacing the mobile E2E implementation or proving an actual requirement cannot be met by the pinned open-source CLI." -stop_rule = "Do not re-check Maestro, Detox, Appium, EAS-only flows, or hosted mobile E2E providers during routine implementation/review work." -default_path = "free, public-checkout reproducible, GitHub-hosted emulator/simulator E2E" diff --git a/tools/dev/maintainer-tool-install.test.sh b/tools/dev/maintainer-tool-install.test.sh index 8d998f70e..7b14e7dae 100644 --- a/tools/dev/maintainer-tool-install.test.sh +++ b/tools/dev/maintainer-tool-install.test.sh @@ -3,6 +3,10 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." repo="$PWD" scratch=$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-maintainer-tools-XXXXXX") +mkdir -p "$scratch/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$scratch/no-delay/sleep" +chmod +x "$scratch/no-delay/sleep" +export PATH="$scratch/no-delay:$PATH" trap 'rm -rf "$scratch"' EXIT FAKE_REAL_MV="$(command -v mv)" export FAKE_REAL_MV diff --git a/tools/dev/moon.yml b/tools/dev/moon.yml index e693443aa..5d0ef0128 100644 --- a/tools/dev/moon.yml +++ b/tools/dev/moon.yml @@ -21,24 +21,23 @@ tasks: tags: - quality - unit - script: "set -e\nbash tools/dev/setup-android-sdk.test.sh\nbash tools/dev/setup-maestro.test.sh\n" + script: "set -e\nbash tools/dev/setup-android-sdk.test.sh\n" inputs: + - acquisition* - setup-android-sdk* - - setup-maestro* - - extract-maestro.mts - bun.sh - node-info.mts - extract-pinned-zip.* - /tools/packaging/testdata/zip-fixture.mts - /tools/packaging/portable-archive.mts - - "/tools/dev/{android-sdk,maestro}.toml" + - "/tools/dev/android-sdk.toml" options: runFromWorkspaceRoot: true test: tags: - quality - unit - script: "set -eu\nfor test in tools/dev/bun.test.sh tools/dev/extract-pinned-zip.test.sh tools/dev/install-pinned-js-runtime.test.sh tools/dev/install-pinned-winflexbison.test.sh .github/actions/setup-moon/install-pinned-node.test.sh .github/actions/setup-moon/install-pinned-toolchain.test.sh .github/actions/setup-npm-publisher/install.test.sh .github/scripts/setup-native-build-tools.test.sh; do\n bash \"$test\"\ndone\nbash tools/dev/maintainer-tool-install.test.sh\n" + script: "set -eu\nfor test in tools/dev/acquisition.test.sh tools/dev/bun.test.sh tools/dev/extract-pinned-zip.test.sh tools/dev/install-pinned-js-runtime.test.sh tools/dev/install-pinned-winflexbison.test.sh .github/actions/setup-moon/install-pinned-node.test.sh .github/actions/setup-moon/install-pinned-toolchain.test.sh .github/actions/setup-npm-publisher/install.test.sh .github/scripts/setup-native-build-tools.test.sh; do\n bash \"$test\"\ndone\nbash tools/dev/maintainer-tool-install.test.sh\n" inputs: - install-pinned* - install-actionlint.sh @@ -50,6 +49,7 @@ tasks: - bun.test.sh - node-info.mts - curl-platform-flags.sh + - acquisition* - /.prototools - /.moon/toolchains.yml - /.github/actions/setup-moon/** diff --git a/tools/dev/setup-android-sdk.sh b/tools/dev/setup-android-sdk.sh index e83ecec7e..1e47b68fd 100755 --- a/tools/dev/setup-android-sdk.sh +++ b/tools/dev/setup-android-sdk.sh @@ -22,6 +22,9 @@ Options: --ndk-version Must match the authoritative toolchain manifest. --cmake-version Must match the authoritative toolchain manifest. --compile-sdk Must match the authoritative toolchain manifest. + --native-tools Install NDK and CMake (default true). + --expo Also install the manifest-pinned Expo app NDK. + --expo-ndk-version Assert the generated app uses the pinned Expo NDK. -h, --help Show this help. The command-line-tools URLs and SHA-256 checksums are intentionally not @@ -32,6 +35,8 @@ EOF root="$(git rev-parse --show-toplevel 2>/dev/null)" || fail "must run inside the Oliphaunt git checkout" cd "$root" +. "$root/tools/dev/acquisition.sh" +oliphaunt_acquisition_start "Android SDK setup" 1800 manifest="${OLIPHAUNT_ANDROID_TOOLCHAIN_MANIFEST:-$root/tools/dev/android-sdk.toml}" extractor="${OLIPHAUNT_ANDROID_ZIP_EXTRACTOR:-$root/tools/dev/extract-pinned-zip.sh}" @@ -78,6 +83,9 @@ sdk_root="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/android-sdk}}" ndk_version="$pinned_ndk" cmake_version="$pinned_cmake" compile_sdk="$pinned_compile_sdk" +native_tools=true +expo=false +expo_ndk_input="" sdkmanager_install_attempts="${ANDROID_SDKMANAGER_INSTALL_ATTEMPTS:-4}" sdkmanager_retry_delay="${ANDROID_SDKMANAGER_RETRY_DELAY:-5}" @@ -103,6 +111,18 @@ while [ "$#" -gt 0 ]; do compile_sdk="$2" shift 2 ;; + --native-tools) + [ "$#" -ge 2 ] || fail "--native-tools requires a value" + native_tools="$2" + shift 2 + ;; + --expo) expo=true; shift ;; + --expo-ndk-version) + [ "$#" -ge 2 ] && [ -n "$2" ] || fail "--expo-ndk-version requires a value" + expo=true + expo_ndk_input="$2" + shift 2 + ;; -h | --help) usage exit 0 @@ -113,6 +133,16 @@ while [ "$#" -gt 0 ]; do esac done +case "$native_tools" in true|false) ;; *) fail "--native-tools must be true or false" ;; esac +[ "$expo" = false ] || [ "$native_tools" = true ] || fail "--expo requires native tools" +expo_ndk="" +if [ "$expo" = true ]; then + expo_ndk="$(manifest_package expo_ndk)" + [ -z "$expo_ndk_input" ] || [ "$expo_ndk_input" = "$expo_ndk" ] || + fail "Expo app NDK $expo_ndk_input differs from manifest pin $expo_ndk; update the pin before building" + case "$expo_ndk" in ''|.*|*.|*..*|*[!0-9.]*) fail "invalid Expo NDK pin" ;; esac +fi + [ -n "$sdk_root" ] || fail "Android SDK root is empty" case "$sdk_root" in /|.|..) fail "unsafe Android SDK root: $sdk_root" ;; esac [ "$ndk_version" = "$pinned_ndk" ] || @@ -165,6 +195,7 @@ case "$cmdline_entry_count" in *[!0-9]*|'') fail "$manifest $section.entry_count must be numeric" ;; esac +oliphaunt_acquisition_timeout >/dev/null require java require mktemp command -v "$curl_bin" >/dev/null 2>&1 || fail "missing required command: $curl_bin" @@ -201,7 +232,7 @@ sdkmanager_version() { local binary="$1" local output [ -f "$binary" ] && [ ! -L "$binary" ] && [ -x "$binary" ] || return 1 - output="$("$binary" --sdk_root="$sdk_root" --version 2>/dev/null)" || return 1 + output="$(oliphaunt_acquisition_run 30 "$binary" --sdk_root="$sdk_root" --version 2>/dev/null)" || return 1 printf '%s\n' "$output" | awk ' { sub(/\r$/, "") @@ -267,11 +298,10 @@ install_cmdline_tools() { for candidate_url in "$cmdline_url" "$cmdline_mirror_url"; do rm -f "$archive" echo "Downloading pinned Android command-line tools: $candidate_url" - if "$curl_bin" \ + if oliphaunt_acquisition_curl 240 6 2 "$curl_bin" \ --fail --location --silent --show-error \ --proto '=https' --proto-redir '=https' \ - --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 180 \ - --connect-timeout 20 --max-time 300 --max-filesize 220000000 \ + --connect-timeout 20 --max-filesize 220000000 \ --remove-on-error --output "$archive" "$candidate_url"; then actual="$(sha256_file "$archive")" if [ "$actual" = "$cmdline_sha256" ]; then @@ -279,6 +309,9 @@ install_cmdline_tools() { break fi echo "Android command-line-tools checksum mismatch from $candidate_url; trying the next pinned origin" >&2 + else + status=$? + case "$status" in 126|127|129|130|137|143) exit "$status" ;; esac fi done [ "$downloaded" = "1" ] || @@ -382,9 +415,10 @@ cmake_valid() { } ndk_valid() { - local directory="$sdk_root/ndk/$pinned_ndk" + local version="${1:-$pinned_ndk}" + local directory="$sdk_root/ndk/$version" local clang count=0 - package_revision_valid "$directory" "$pinned_ndk" || return 1 + package_revision_valid "$directory" "$version" || return 1 for clang in "$directory"/toolchains/llvm/prebuilt/*/bin/clang; do [ -e "$clang" ] || continue usable_executable "$clang" || return 1 @@ -393,12 +427,15 @@ ndk_valid() { [ "$count" -eq 1 ] } + sdk_packages_valid() { platform_tools_valid && platform_valid && - build_tools_valid && - cmake_valid && - ndk_valid + build_tools_valid || return 1 + if [ "$native_tools" = true ]; then + cmake_valid && ndk_valid || return 1 + [ "$expo" = false ] || ndk_valid "$expo_ndk" || return 1 + fi } cleanup_invalid_sdk_packages() { @@ -407,28 +444,33 @@ cleanup_invalid_sdk_packages() { platform_valid || rm -rf "$sdk_root/platforms/android-$pinned_compile_sdk" build_tools_valid || rm -rf "$sdk_root/build-tools/$pinned_build_tools" - cmake_valid || - rm -rf "$sdk_root/cmake/$pinned_cmake" - ndk_valid || - rm -rf "$sdk_root/ndk/$pinned_ndk" + if [ "$native_tools" = true ]; then + cmake_valid || rm -rf "$sdk_root/cmake/$pinned_cmake" + ndk_valid || rm -rf "$sdk_root/ndk/$pinned_ndk" + if [ "$expo" = true ]; then + ndk_valid "$expo_ndk" || rm -rf "$sdk_root/ndk/$expo_ndk" + fi + fi } install_sdk_packages() { local attempt=1 + local packages=(platform-tools "platforms;android-$pinned_compile_sdk" "build-tools;$pinned_build_tools") + if [ "$native_tools" = true ]; then + packages+=("cmake;$pinned_cmake" "ndk;$pinned_ndk") + [ "$expo" = false ] || packages+=("ndk;$expo_ndk") + fi while [ "$attempt" -le "$sdkmanager_install_attempts" ]; do cleanup_invalid_sdk_packages echo "Installing exact Android SDK package identities (attempt $attempt/$sdkmanager_install_attempts)" - if "$sdkmanager_bin" --sdk_root="$sdk_root" --install \ - "platform-tools" \ - "platforms;android-$pinned_compile_sdk" \ - "build-tools;$pinned_build_tools" \ - "cmake;$pinned_cmake" \ - "ndk;$pinned_ndk" && sdk_packages_valid; then + if oliphaunt_acquisition_run 1800 "$sdkmanager_bin" --sdk_root="$sdk_root" --install \ + "${packages[@]}" && sdk_packages_valid; then return 0 fi + cleanup_invalid_sdk_packages if [ "$attempt" -lt "$sdkmanager_install_attempts" ]; then echo "Android SDK package install or validation failed; repairing before retry" >&2 - sleep "$sdkmanager_retry_delay" + oliphaunt_acquisition_sleep "$sdkmanager_retry_delay" || exit $? fi attempt=$((attempt + 1)) done @@ -437,10 +479,15 @@ install_sdk_packages() { if ! sdk_packages_valid; then echo "Accepting Android SDK licenses" - yes | "$sdkmanager_bin" --sdk_root="$sdk_root" --licenses >/dev/null || true + yes | oliphaunt_acquisition_run 1800 "$sdkmanager_bin" --sdk_root="$sdk_root" --licenses >/dev/null || true install_sdk_packages fi sdk_packages_valid || fail "Android SDK package cache is invalid after repair" echo "ANDROID_HOME=$sdk_root" -echo "ANDROID_NDK_HOME=$sdk_root/ndk/$pinned_ndk" +if [ "$native_tools" = true ]; then + echo "ANDROID_NDK_HOME=$sdk_root/ndk/$pinned_ndk" + if [ -n "${GITHUB_ENV:-}" ]; then + echo "ANDROID_NDK_HOME=$sdk_root/ndk/$pinned_ndk" >> "$GITHUB_ENV" + fi +fi diff --git a/tools/dev/setup-android-sdk.test.sh b/tools/dev/setup-android-sdk.test.sh index f89784513..95be4a183 100755 --- a/tools/dev/setup-android-sdk.test.sh +++ b/tools/dev/setup-android-sdk.test.sh @@ -5,6 +5,10 @@ root="$(git rev-parse --show-toplevel)" installer="$root/tools/dev/setup-android-sdk.sh" extractor="$root/tools/dev/extract-pinned-zip.sh" tmp="$(mktemp -d)" +mkdir -p "$tmp/no-delay" +printf '#!/bin/sh\nexit 0\n' > "$tmp/no-delay/sleep" +chmod +x "$tmp/no-delay/sleep" +export PATH="$tmp/no-delay:$PATH" trap 'rm -rf "$tmp"' EXIT HUP INT TERM @@ -19,7 +23,7 @@ const write = (name, data) => writeFileSync(root + '/' + name, data); function archive(name, version, layout = 'cmdline-tools') { const entries = { - [layout + '/bin/sdkmanager']: "#!/usr/bin/env bash\nset -euo pipefail\nsdk_root=\"\"\noperation=\"\"\npackages=()\nfor argument in \"$@\"; do\n case \"$argument\" in\n --sdk_root=*) sdk_root=\"${argument#--sdk_root=}\" ;;\n --version) operation=version ;;\n --licenses) operation=licenses ;;\n --install) operation=install ;;\n *) packages+=(\"$argument\") ;;\n esac\ndone\n[ -n \"$sdk_root\" ]\ncase \"$operation\" in\n version)\n printf '{version}\\n'\n ;;\n licenses)\n exit 0\n ;;\n install)\n expected=(\n platform-tools\n 'platforms;android-36'\n 'build-tools;36.0.0'\n 'cmake;3.22.1'\n 'ndk;27.0.12077973'\n )\n [ \"${#packages[@]}\" = \"${#expected[@]}\" ]\n for index in \"${!expected[@]}\"; do\n [ \"${packages[$index]}\" = \"${expected[$index]}\" ]\n done\n mkdir -p \\\n \"$sdk_root/platform-tools\" \\\n \"$sdk_root/platforms/android-36\" \\\n \"$sdk_root/build-tools/36.0.0\" \\\n \"$sdk_root/cmake/3.22.1/bin\" \\\n \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/platform-tools/adb\"\n chmod +x \"$sdk_root/platform-tools/adb\"\n printf 'AndroidVersion.ApiLevel=36\\n' > \"$sdk_root/platforms/android-36/source.properties\"\n printf 'fake-android-jar\\n' > \"$sdk_root/platforms/android-36/android.jar\"\n printf 'Pkg.Revision=36.0.0\\n' > \"$sdk_root/build-tools/36.0.0/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/aapt2\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/zipalign\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/apksigner\"\n chmod +x \\\n \"$sdk_root/build-tools/36.0.0/aapt2\" \\\n \"$sdk_root/build-tools/36.0.0/zipalign\" \\\n \"$sdk_root/build-tools/36.0.0/apksigner\"\n printf 'Pkg.Revision = 3.22.1\\n' > \"$sdk_root/cmake/3.22.1/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/cmake/3.22.1/bin/cmake\"\n chmod +x \"$sdk_root/cmake/3.22.1/bin/cmake\"\n printf 'Pkg.Revision = 27.0.12077973\\n' > \"$sdk_root/ndk/27.0.12077973/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\"\n chmod +x \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\"\n count=0\n [ ! -f \"$sdk_root/fake-install-count\" ] || count=\"$(cat \"$sdk_root/fake-install-count\")\"\n printf '%s\\n' \"$((count + 1))\" > \"$sdk_root/fake-install-count\"\n ;;\n *)\n exit 2\n ;;\nesac\n".replace('{version}', version), + [layout + '/bin/sdkmanager']: "#!/usr/bin/env bash\nset -euo pipefail\nsdk_root=\"\"\noperation=\"\"\npackages=()\nfor argument in \"$@\"; do\n case \"$argument\" in\n --sdk_root=*) sdk_root=\"${argument#--sdk_root=}\" ;;\n --version) operation=version ;;\n --licenses) operation=licenses ;;\n --install) operation=install ;;\n *) packages+=(\"$argument\") ;;\n esac\ndone\n[ -n \"$sdk_root\" ]\ncase \"$operation\" in\n version)\n printf '{version}\\n'\n ;;\n licenses)\n exit 0\n ;;\n install)\n expected=(\n platform-tools\n 'platforms;android-36'\n 'build-tools;36.0.0'\n 'cmake;3.22.1'\n 'ndk;27.0.12077973'\n )\n case \"${#packages[@]}\" in\n 3) expected=(\"${expected[@]:0:3}\") ;;\n 6) expected+=('ndk;27.1.12297006') ;;\n esac\n [ \"${#packages[@]}\" = \"${#expected[@]}\" ]\n for index in \"${!expected[@]}\"; do\n [ \"${packages[$index]}\" = \"${expected[$index]}\" ]\n done\n mkdir -p \\\n \"$sdk_root/platform-tools\" \\\n \"$sdk_root/platforms/android-36\" \\\n \"$sdk_root/build-tools/36.0.0\" \\\n \"$sdk_root/cmake/3.22.1/bin\" \\\n \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/platform-tools/adb\"\n chmod +x \"$sdk_root/platform-tools/adb\"\n printf 'AndroidVersion.ApiLevel=36\\n' > \"$sdk_root/platforms/android-36/source.properties\"\n printf 'fake-android-jar\\n' > \"$sdk_root/platforms/android-36/android.jar\"\n printf 'Pkg.Revision=36.0.0\\n' > \"$sdk_root/build-tools/36.0.0/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/aapt2\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/zipalign\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/build-tools/36.0.0/apksigner\"\n chmod +x \\\n \"$sdk_root/build-tools/36.0.0/aapt2\" \\\n \"$sdk_root/build-tools/36.0.0/zipalign\" \\\n \"$sdk_root/build-tools/36.0.0/apksigner\"\n printf 'Pkg.Revision = 3.22.1\\n' > \"$sdk_root/cmake/3.22.1/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/cmake/3.22.1/bin/cmake\"\n chmod +x \"$sdk_root/cmake/3.22.1/bin/cmake\"\n printf 'Pkg.Revision = 27.0.12077973\\n' > \"$sdk_root/ndk/27.0.12077973/source.properties\"\n printf '%s\\n' '#!/bin/sh' 'exit 0' > \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\"\n chmod +x \"$sdk_root/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\"\n if [ \"${#packages[@]}\" = 3 ]; then\n rm -rf \"$sdk_root/ndk\" \"$sdk_root/cmake\"\n elif [ \"${#packages[@]}\" = 6 ]; then\n cp -R \"$sdk_root/ndk/27.0.12077973\" \"$sdk_root/ndk/27.1.12297006\"\n printf 'Pkg.Revision = 27.1.12297006\\n' > \"$sdk_root/ndk/27.1.12297006/source.properties\"\n fi\n count=0\n [ ! -f \"$sdk_root/fake-install-count\" ] || count=\"$(cat \"$sdk_root/fake-install-count\")\"\n printf '%s\\n' \"$((count + 1))\" > \"$sdk_root/fake-install-count\"\n ;;\n *)\n exit 2\n ;;\nesac\n".replace('{version}', version), [layout + '/bin/avdmanager']: '#!/usr/bin/env bash\nset -euo pipefail\nexit 0\n', [layout + '/bin/apkanalyzer']: '#!/usr/bin/env bash\nset -euo pipefail\nexit 0\n', [layout + '/source.properties']: 'Pkg.Revision=20.0\n', @@ -34,7 +38,7 @@ for (const [name, digest] of [ ['android-bad-sha.toml','0'.repeat(64)], ['android-wrong-version.toml',archive('android-wrong-version.zip','19.0')], ['android-wrong-layout.toml',archive('android-wrong-layout.zip','20.0','not-cmdline-tools')], -]) write('config/' + name, "[packages]\ncommand_line_tools_build = \"14742923\"\ncommand_line_tools_revision = \"20.0\"\nndk = \"27.0.12077973\"\ncmake = \"3.22.1\"\ncompile_sdk = \"36\"\nbuild_tools = \"36.0.0\"\n\n[command_line_tools.linux]\nurl = \"https://dl.google.com/android/repository/commandlinetools-linux-14742923_latest.zip\"\nmirror_url = \"https://edgedl.me.gvt1.com/edgedl/android/repository/commandlinetools-linux-14742923_latest.zip\"\nsha256 = \"{digest}\"\nentry_count = \"5\"\n\n[command_line_tools.mac]\nurl = \"https://dl.google.com/android/repository/commandlinetools-mac-14742923_latest.zip\"\nmirror_url = \"https://edgedl.me.gvt1.com/edgedl/android/repository/commandlinetools-mac-14742923_latest.zip\"\nsha256 = \"{digest}\"\nentry_count = \"5\"\n".replaceAll('{digest}', digest)); +]) write('config/' + name, "[packages]\ncommand_line_tools_build = \"14742923\"\ncommand_line_tools_revision = \"20.0\"\nndk = \"27.0.12077973\"\nexpo_ndk = \"27.1.12297006\"\ncmake = \"3.22.1\"\ncompile_sdk = \"36\"\nbuild_tools = \"36.0.0\"\n\n[command_line_tools.linux]\nurl = \"https://dl.google.com/android/repository/commandlinetools-linux-14742923_latest.zip\"\nmirror_url = \"https://edgedl.me.gvt1.com/edgedl/android/repository/commandlinetools-linux-14742923_latest.zip\"\nsha256 = \"{digest}\"\nentry_count = \"5\"\n\n[command_line_tools.mac]\nurl = \"https://dl.google.com/android/repository/commandlinetools-mac-14742923_latest.zip\"\nmirror_url = \"https://edgedl.me.gvt1.com/edgedl/android/repository/commandlinetools-mac-14742923_latest.zip\"\nsha256 = \"{digest}\"\nentry_count = \"5\"\n".replaceAll('{digest}', digest)); TS @@ -94,7 +98,7 @@ run_android() { --sdk-root "${SDK_ROOT:-$tmp/sdk}" \ --ndk-version 27.0.12077973 \ --cmake-version 3.22.1 \ - --compile-sdk 36 + --compile-sdk 36 "$@" } # The official mirror is a bounded fallback, and installed identities are exact. @@ -220,4 +224,39 @@ for retry_case in attempts delay; do [ ! -s "$tmp/curl.log" ] done +# Replay installs no native compilers; Expo explicitly installs both pinned NDKs. +SDK_ROOT="$tmp/replay-sdk" run_android --native-tools false > "$tmp/replay.out" +[ ! -d "$tmp/replay-sdk/ndk" ] && [ ! -d "$tmp/replay-sdk/cmake" ] +[ -x "$tmp/replay-sdk/platform-tools/adb" ] +SDK_ROOT="$tmp/expo-sdk" run_android --expo-ndk-version 27.1.12297006 > "$tmp/expo.out" +for ndk in 27.0.12077973 27.1.12297006; do + grep -qx "Pkg.Revision = $ndk" "$tmp/expo-sdk/ndk/$ndk/source.properties" +done +SDK_ROOT="$tmp/expo-sdk" CURL_MODE=fail-all run_android --expo > "$tmp/expo-cached.out" +grep -qx 1 "$tmp/expo-sdk/fake-install-count" +if SDK_ROOT="$tmp/replay-sdk" run_android --native-tools false --expo >/dev/null 2>&1; then + echo 'accepted Expo without native tools' >&2; exit 1 +fi +if run_android --expo-ndk-version 99.0.0 > "$tmp/skew.out" 2>&1; then + echo 'accepted unexpected Expo NDK' >&2; exit 1 +fi +grep -q 'differs from manifest pin' "$tmp/skew.out" +# A license request that exhausts the budget must not start package installation. +cp -R "$tmp/sdk" "$tmp/deadline-sdk" +sed 's/ exit 0/ printf "9999999\\n" > "$DEADLINE_CLOCK"; exit 0/' \ + "$tmp/sdk/cmdline-tools/latest/bin/sdkmanager" > "$tmp/deadline-sdk/cmdline-tools/latest/bin/sdkmanager" +rm "$tmp/deadline-sdk/build-tools/36.0.0/apksigner" +cat > "$tmp/bin/date" <<'DATE' +#!/bin/sh +cat "$DEADLINE_CLOCK" +DATE +chmod +x "$tmp/bin/date" +printf '1000000\n' > "$tmp/clock" +if PATH="$tmp/bin:$PATH" DEADLINE_CLOCK="$tmp/clock" SDK_ROOT="$tmp/deadline-sdk" run_android > "$tmp/deadline.out" 2>&1; then + echo 'expired SDK setup succeeded' >&2; exit 1 +fi +grep -q 'deadline' "$tmp/deadline.out" +cmp "$tmp/sdk/fake-install-count" "$tmp/deadline-sdk/fake-install-count" +[ -x "$tmp/deadline-sdk/ndk/27.0.12077973/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" ] +[ ! -e "$tmp/deadline-sdk/build-tools/36.0.0" ] echo "Android SDK bootstrap fault tests passed" diff --git a/tools/dev/setup-maestro.sh b/tools/dev/setup-maestro.sh deleted file mode 100755 index c722e0ab9..000000000 --- a/tools/dev/setup-maestro.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root="$(git rev-parse --show-toplevel 2>/dev/null)" || { - echo "must run inside the Oliphaunt git checkout" >&2 - exit 1 -} -cd "$root" - -need_cmd() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required command: $1" >&2 - exit 1 - } -} - -need_cmd curl -need_cmd java -need_cmd mktemp - -export MAESTRO_CLI_NO_ANALYTICS=true -export MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED=true -maestro_bin="$HOME/.maestro/bin/maestro" -maestro_manifest="tools/dev/maestro.toml" - -manifest_value() { - local key="$1" - local count - local value - count="$(grep -Ec "^[[:space:]]*${key}[[:space:]]*=" "$maestro_manifest" || true)" - [ "$count" = "1" ] || return 1 - value="$(sed -n "s/^[[:space:]]*${key}[[:space:]]*=[[:space:]]*\"\([^\"]*\)\"[[:space:]]*$/\\1/p" "$maestro_manifest")" - [ -n "$value" ] || return 1 - printf '%s\n' "$value" -} - -version="$(manifest_value maestro || true)" -maestro_url="$(manifest_value install_url || true)" -maestro_sha256="$(manifest_value sha256 || true)" -if [ -z "$version" ] || [ -z "$maestro_url" ] || [ -z "$maestro_sha256" ]; then - echo "$maestro_manifest must contain exactly one quoted maestro version, install_url, and sha256 pin" >&2 - exit 1 -fi - -normalized_version="${version#cli-}" -stable_maestro_version_re='^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z]+)*$' -if ! [[ "$normalized_version" =~ $stable_maestro_version_re ]] || { - [ "$version" != "$normalized_version" ] && [ "$version" != "cli-$normalized_version" ]; -}; then - echo "$maestro_manifest contains an invalid Maestro version: $version" >&2 - exit 1 -fi -expected_url="https://github.com/mobile-dev-inc/Maestro/releases/download/cli-$normalized_version/maestro.zip" -if [ "$maestro_url" != "$expected_url" ]; then - echo "$maestro_manifest install_url must be the exact release asset for cli-$normalized_version" >&2 - exit 1 -fi -if [ "${#maestro_sha256}" -ne 64 ] || [[ "$maestro_sha256" =~ [^0-9A-Fa-f] ]]; then - echo "$maestro_manifest sha256 must be exactly 64 hexadecimal characters" >&2 - exit 1 -fi -maestro_sha256="$(printf '%s' "$maestro_sha256" | tr '[:upper:]' '[:lower:]')" - -maestro_version() { - local binary="$1" - local output - local detected - output="$("$binary" --version 2>/dev/null)" || return 1 - detected="$(printf '%s\n' "$output" | awk 'NF { value = $NF } END { print value }')" - detected="${detected#cli-}" - [ -n "$detected" ] || return 1 - printf '%s\n' "$detected" -} - -maestro_sha256_file() { - local path="$1" - if command -v shasum >/dev/null 2>&1; then - shasum -a 256 "$path" | awk '{print tolower($1)}' - elif command -v sha256sum >/dev/null 2>&1; then - sha256sum "$path" | awk '{print tolower($1)}' - else - echo "Maestro installation requires shasum or sha256sum" >&2 - return 127 - fi -} - -export MAESTRO_VERSION="$normalized_version" -if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then - echo "Maestro installation requires shasum or sha256sum" >&2 - exit 127 -fi -umask 077 -mkdir -p "$HOME" -install_root="$HOME/.maestro" -temporary_root="$(mktemp -d "$HOME/.maestro.install.XXXXXX")" -previous_root="$temporary_root/previous" -had_previous=0 -promotion_started=0 -cleanup() { - local cleanup_status="$?" - trap - EXIT HUP INT TERM - if [ "$promotion_started" = "1" ] && [ "$had_previous" = "1" ] && \ - [ ! -e "$install_root" ] && [ ! -L "$install_root" ] && \ - { [ -e "$previous_root" ] || [ -L "$previous_root" ]; }; then - if ! mv "$previous_root" "$install_root"; then - echo "could not restore the previous Maestro installation; it remains at $previous_root" >&2 - [ "$cleanup_status" -ne 0 ] || cleanup_status=1 - exit "$cleanup_status" - fi - fi - rm -rf "$temporary_root" - exit "$cleanup_status" -} -trap cleanup EXIT -trap 'exit 129' HUP -trap 'exit 130' INT -trap 'exit 143' TERM - -archive="$temporary_root/maestro.zip" -extract_root="$temporary_root/extracted" -mkdir -p "$extract_root" -curl \ - --fail \ - --location \ - --silent \ - --show-error \ - --retry 4 \ - --retry-all-errors \ - --retry-delay 3 \ - --retry-max-time 300 \ - --connect-timeout 20 \ - --max-time 300 \ - --max-filesize 400000000 \ - --proto '=https' \ - --proto-redir '=https' \ - --tlsv1.2 \ - --remove-on-error \ - --output "$archive" \ - "$maestro_url" - -actual_sha256="$(maestro_sha256_file "$archive")" -if [ "$actual_sha256" != "$maestro_sha256" ]; then - echo "Maestro $normalized_version archive checksum mismatch: expected $maestro_sha256, got $actual_sha256" >&2 - exit 1 -fi - -bash tools/dev/bun.sh tools/dev/extract-maestro.mts "$archive" "$extract_root" "$normalized_version" - -candidate_root="$extract_root/maestro" -candidate_bin="$candidate_root/bin/maestro" -candidate_jar="$candidate_root/lib/maestro-cli-$normalized_version.jar" -if [ ! -f "$candidate_bin" ] || [ -L "$candidate_bin" ]; then - echo "Maestro archive did not produce a regular launcher at $candidate_bin" >&2 - exit 1 -fi -if [ ! -f "$candidate_jar" ] || [ -L "$candidate_jar" ]; then - echo "Maestro archive did not produce the pinned CLI jar at $candidate_jar" >&2 - exit 1 -fi -chmod 0755 "$candidate_bin" -candidate_version="$(maestro_version "$candidate_bin" || true)" -if [ "$candidate_version" != "$normalized_version" ]; then - echo "Maestro archive launcher version mismatch: expected $normalized_version, got ${candidate_version:-}" >&2 - exit 1 -fi - -if [ -e "$install_root" ] || [ -L "$install_root" ]; then - promotion_started=1 - had_previous=1 - mv "$install_root" "$previous_root" -fi -if mv "$candidate_root" "$install_root"; then - : -else - promotion_status=$? - if [ "$had_previous" = "1" ] && ! mv "$previous_root" "$install_root"; then - trap - EXIT - echo "Maestro promotion failed and rollback failed; the previous installation remains at $previous_root" >&2 - exit "$promotion_status" - fi - echo "Maestro promotion failed; the previous installation was restored" >&2 - exit "$promotion_status" -fi - -[ -x "$maestro_bin" ] || { - echo "maestro install did not produce $maestro_bin" >&2 - exit 1 -} - -if [ -n "${GITHUB_PATH:-}" ]; then - printf '%s\n' "$HOME/.maestro/bin" >>"$GITHUB_PATH" -fi - -"$maestro_bin" --version diff --git a/tools/dev/setup-maestro.test.sh b/tools/dev/setup-maestro.test.sh deleted file mode 100755 index 0ec1e810c..000000000 --- a/tools/dev/setup-maestro.test.sh +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root="$(git rev-parse --show-toplevel)" -installer="$root/tools/dev/setup-maestro.sh" -manifest="$root/tools/dev/maestro.toml" -configured_version="$(sed -n 's/^[[:space:]]*maestro[[:space:]]*=[[:space:]]*"\([^"]*\)"[[:space:]]*$/\1/p' "$manifest")" -expected_version="${configured_version#cli-}" -expected_sha256="$(sed -n 's/^[[:space:]]*sha256[[:space:]]*=[[:space:]]*"\([^"]*\)"[[:space:]]*$/\1/p' "$manifest")" -expected_url="$(sed -n 's/^[[:space:]]*install_url[[:space:]]*=[[:space:]]*"\([^"]*\)"[[:space:]]*$/\1/p' "$manifest")" -if [ -z "$expected_version" ] || [ -z "$expected_sha256" ] || [ -z "$expected_url" ]; then - echo "setup-maestro.test.sh: missing Maestro release metadata in $manifest" >&2 - exit 1 -fi -test_root="$(mktemp -d "${TMPDIR:-/tmp}/oliphaunt-maestro-test.XXXXXX")" -trap 'rm -rf "$test_root"' EXIT - -fail() { - echo "setup-maestro.test.sh: $*" >&2 - exit 1 -} - -assert_contains() { - local path="$1" - local expected="$2" - grep -F -- "$expected" "$path" >/dev/null || fail "$path did not contain: $expected" -} - -assert_curl_arg() { - local path="$1" - local expected="$2" - grep -Fx -- "$expected" "$path" >/dev/null || fail "curl did not receive argument: $expected" -} - -fake_bin="$test_root/fake-bin" -mkdir -p "$fake_bin" - -cat >"$fake_bin/curl" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -printf '%s\n' "$@" >"$MAESTRO_TEST_CURL_ARGS" -if [ "${MAESTRO_TEST_CURL_EXIT:-0}" != "0" ]; then - exit "$MAESTRO_TEST_CURL_EXIT" -fi -output="" -while [ "$#" -gt 0 ]; do - case "$1" in - --output) - output="$2" - shift 2 - ;; - *) shift ;; - esac -done -[ -n "$output" ] || exit 64 -cp "$MAESTRO_TEST_ARCHIVE" "$output" -SH - -cat >"$fake_bin/shasum" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -last="${!#}" -printf '%s %s\n' "$MAESTRO_TEST_SHA256" "$last" -SH - -cat >"$fake_bin/java" <<'SH' -#!/usr/bin/env bash -exit 0 -SH - -cat >"$fake_bin/maestro" <<'SH' -#!/usr/bin/env bash -printf '%s\n' "${MAESTRO_TEST_AMBIENT_VERSION:-cli-0.0.0}" -SH - -cat >"$fake_bin/mv" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -count=0 -if [ -f "$MAESTRO_TEST_MV_COUNTER" ]; then - count="$(cat "$MAESTRO_TEST_MV_COUNTER")" -fi -count=$((count + 1)) -printf '%s\n' "$count" >"$MAESTRO_TEST_MV_COUNTER" -if [ "${MAESTRO_TEST_MV_FAIL_SECOND:-0}" = "1" ] && [ "$count" = "2" ]; then - exit 73 -fi -exec /bin/mv "$@" -SH - -pinned_bun="$(bash "$root/tools/dev/bun.sh" "$root/tools/dev/node-info.mts" executable)" -ln -s "$pinned_bun" "$fake_bin/bun" -chmod 0755 "$fake_bin"/* - -fallback_bin="$test_root/fallback-bin" -no_hash_bin="$test_root/no-hash-bin" -mkdir -p "$fallback_bin" "$no_hash_bin" -for command_name in bash git grep sed tr awk mkdir mktemp dirname chmod rm cp cat; do - command_path="$(command -v "$command_name")" - ln -s "$command_path" "$fallback_bin/$command_name" - ln -s "$command_path" "$no_hash_bin/$command_name" -done -for helper in bun curl java maestro mv; do - cp "$fake_bin/$helper" "$fallback_bin/$helper" - cp "$fake_bin/$helper" "$no_hash_bin/$helper" -done -cp "$fake_bin/shasum" "$fallback_bin/sha256sum" - -make_archive() { - local output="$1" - local launcher_version="$2" - local shape="$3" - bash "$root/tools/dev/bun.sh" - "$output" "$launcher_version" "$shape" "$expected_version" <<'TS' -import {writeFileSync} from 'node:fs'; -import {zipArchive} from './tools/packaging/testdata/zip-fixture.mts'; -const [output,version,shape,archiveVersion] = process.argv.slice(2); -const rows = [{name:'maestro/bin/maestro',data:"#!/usr/bin/env bash\nprintf '" + version + "\\n'\n",externalAttributes:0o100755<<16}]; -if (shape !== 'missing-jar') rows.push({name:'maestro/lib/maestro-cli-'+archiveVersion+'.jar',data:'mock jar'}); -if (shape === 'traversal') rows.push({name:'maestro/../escape',data:'escape'}); -writeFileSync(output,zipArchive(rows)); -TS -} - -valid_archive="$test_root/valid.zip" -wrong_version_archive="$test_root/wrong-version.zip" -missing_jar_archive="$test_root/missing-jar.zip" -traversal_archive="$test_root/traversal.zip" -corrupt_archive="$test_root/corrupt.zip" -make_archive "$valid_archive" "$expected_version" valid -make_archive "$wrong_version_archive" "999.999.999" valid -make_archive "$missing_jar_archive" "$expected_version" missing-jar -make_archive "$traversal_archive" "$expected_version" traversal -printf 'not a zip archive\n' >"$corrupt_archive" - -run_case() { - local name="$1" - local configured_version="$2" - local archive="$3" - local reported_sha256="$4" - local fail_second_mv="${5:-0}" - local curl_exit="${6:-0}" - local manifest_mode="${7:-pinned}" - local hash_mode="${8:-shasum}" - local ambient_version="${9:-cli-0.0.0}" - local case_path - case "$hash_mode" in - shasum) case_path="$fake_bin:$PATH" ;; - sha256sum) case_path="$fallback_bin" ;; - none) case_path="$no_hash_bin" ;; - *) fail "unknown hash mode: $hash_mode" ;; - esac - - CASE_ROOT="$test_root/cases/$name" - CASE_REPO="$CASE_ROOT/repo" - CASE_HOME="$CASE_ROOT/home" - CASE_LOG="$CASE_ROOT/setup.log" - CASE_CURL_ARGS="$CASE_ROOT/curl-args" - CASE_GITHUB_PATH="$CASE_ROOT/github-path" - mkdir -p "$CASE_REPO/tools/dev" "$CASE_REPO/tools/dev" "$CASE_HOME/.maestro" - cp "$installer" "$CASE_REPO/tools/dev/setup-maestro.sh" - cp "$root/tools/dev/extract-maestro.mts" "$CASE_REPO/tools/dev/extract-maestro.mts" - cp "$root/tools/dev/bun.sh" "$CASE_REPO/tools/dev/bun.sh" - cp "$root/.prototools" "$CASE_REPO/.prototools" - mkdir -p "$CASE_REPO/tools/packaging" - cp "$root/tools/packaging/portable-archive.mts" "$CASE_REPO/tools/packaging/portable-archive.mts" - case "$manifest_mode" in - pinned) - printf '[toolchain]\nmaestro = "%s"\ninstall_url = "%s"\nsha256 = "%s"\n' \ - "$configured_version" "$expected_url" "$expected_sha256" \ - >"$CASE_REPO/tools/dev/maestro.toml" - ;; - unpinned) - printf '[toolchain]\nmaestro = "%s"\n' \ - "$configured_version" >"$CASE_REPO/tools/dev/maestro.toml" - ;; - wrong-url) - printf '[toolchain]\nmaestro = "%s"\ninstall_url = "https://example.invalid/maestro.zip"\nsha256 = "%s"\n' \ - "$configured_version" "$expected_sha256" \ - >"$CASE_REPO/tools/dev/maestro.toml" - ;; - invalid-sha) - printf '[toolchain]\nmaestro = "%s"\ninstall_url = "%s"\nsha256 = "not-a-sha256"\n' \ - "$configured_version" "$expected_url" \ - >"$CASE_REPO/tools/dev/maestro.toml" - ;; - *) fail "unknown manifest mode: $manifest_mode" ;; - esac - printf 'previous installation\n' >"$CASE_HOME/.maestro/previous-marker" - git -C "$CASE_REPO" init -q - - if ( - cd "$CASE_REPO" - env \ - PATH="$case_path" \ - HOME="$CASE_HOME" \ - GITHUB_PATH="$CASE_GITHUB_PATH" \ - MAESTRO_TEST_ARCHIVE="$archive" \ - MAESTRO_TEST_CURL_ARGS="$CASE_CURL_ARGS" \ - MAESTRO_TEST_CURL_EXIT="$curl_exit" \ - MAESTRO_TEST_SHA256="$reported_sha256" \ - MAESTRO_TEST_MV_COUNTER="$CASE_ROOT/mv-count" \ - MAESTRO_TEST_MV_FAIL_SECOND="$fail_second_mv" \ - MAESTRO_TEST_AMBIENT_VERSION="$ambient_version" \ - bash "$CASE_REPO/tools/dev/setup-maestro.sh" - ) >"$CASE_LOG" 2>&1; then - CASE_STATUS=0 - else - CASE_STATUS=$? - fi -} - -assert_previous_preserved() { - [ -f "$CASE_HOME/.maestro/previous-marker" ] || fail "$CASE_ROOT did not preserve the previous installation" -} - -assert_no_staging_dirs() { - local leftover - leftover="$(find "$CASE_HOME" -maxdepth 1 -name '.maestro.install.*' -print -quit)" - [ -z "$leftover" ] || fail "$CASE_ROOT left installation staging behind: $leftover" -} - -run_case success "$configured_version" "$valid_archive" "$expected_sha256" -[ "$CASE_STATUS" = "0" ] || fail "valid pinned archive failed: $(cat "$CASE_LOG")" -[ -x "$CASE_HOME/.maestro/bin/maestro" ] || fail "valid install did not promote the launcher" -[ -f "$CASE_HOME/.maestro/lib/maestro-cli-$expected_version.jar" ] || fail "valid install did not promote the pinned jar" -[ ! -e "$CASE_HOME/.maestro/previous-marker" ] || fail "valid install retained stale installation contents" -[ "$("$CASE_HOME/.maestro/bin/maestro" --version)" = "$expected_version" ] || fail "promoted launcher has the wrong version" -[ "$(cat "$CASE_GITHUB_PATH")" = "$CASE_HOME/.maestro/bin" ] || fail "valid install wrote the wrong GitHub PATH entry" -assert_no_staging_dirs -for argument in \ - --fail \ - --location \ - --retry-all-errors \ - --retry-max-time \ - --connect-timeout \ - --max-time \ - --max-filesize \ - --proto \ - --proto-redir \ - --tlsv1.2 \ - --remove-on-error \ - '=https' \ - "$expected_url"; do - assert_curl_arg "$CASE_CURL_ARGS" "$argument" -done - -run_case prefixed-version "cli-$expected_version" "$valid_archive" "$expected_sha256" -[ "$CASE_STATUS" = "0" ] || fail "cli-prefixed configured version was not preserved" - -run_case same-version-ambient "$expected_version" "$valid_archive" "$expected_sha256" 0 0 pinned shasum "cli-$expected_version" -[ "$CASE_STATUS" = "0" ] || fail "a same-version ambient Maestro prevented the pinned installation" -[ -s "$CASE_CURL_ARGS" ] || fail "a same-version ambient Maestro bypassed the pinned archive download" -[ -x "$CASE_HOME/.maestro/bin/maestro" ] || fail "same-version ambient repair did not promote the pinned launcher" -assert_no_staging_dirs - -run_case sha256sum-fallback "$expected_version" "$valid_archive" "$expected_sha256" 0 0 pinned sha256sum -[ "$CASE_STATUS" = "0" ] || fail "sha256sum-only environment failed: $(cat "$CASE_LOG")" - -run_case missing-hash-command "$expected_version" "$valid_archive" "$expected_sha256" 0 0 pinned none -[ "$CASE_STATUS" = "127" ] || fail "missing hash utilities did not fail with status 127" -assert_contains "$CASE_LOG" "requires shasum or sha256sum" -[ ! -e "$CASE_CURL_ARGS" ] || fail "missing hash utilities reached the network" -assert_previous_preserved -assert_no_staging_dirs - -run_case checksum-mismatch "$expected_version" "$valid_archive" "0000000000000000000000000000000000000000000000000000000000000000" -[ "$CASE_STATUS" != "0" ] || fail "checksum mismatch unexpectedly succeeded" -assert_contains "$CASE_LOG" "archive checksum mismatch" -assert_previous_preserved -assert_no_staging_dirs - -run_case corrupt-archive "$expected_version" "$corrupt_archive" "$expected_sha256" -[ "$CASE_STATUS" != "0" ] || fail "corrupt archive unexpectedly succeeded" -assert_contains "$CASE_LOG" "invalid Maestro archive" -assert_previous_preserved -assert_no_staging_dirs - -run_case missing-layout "$expected_version" "$missing_jar_archive" "$expected_sha256" -[ "$CASE_STATUS" != "0" ] || fail "archive with a missing CLI jar unexpectedly succeeded" -assert_contains "$CASE_LOG" "missing expected archive entry" -assert_previous_preserved -assert_no_staging_dirs - -run_case wrong-version "$expected_version" "$wrong_version_archive" "$expected_sha256" -[ "$CASE_STATUS" != "0" ] || fail "archive with the wrong launcher version unexpectedly succeeded" -assert_contains "$CASE_LOG" "launcher version mismatch" -assert_previous_preserved -assert_no_staging_dirs - -run_case traversal "$expected_version" "$traversal_archive" "$expected_sha256" -[ "$CASE_STATUS" != "0" ] || fail "archive with path traversal unexpectedly succeeded" -assert_contains "$CASE_LOG" "unsafe archive member" -assert_previous_preserved -assert_no_staging_dirs - -run_case unpinned-version 999.999.998 "$valid_archive" "$expected_sha256" 0 0 unpinned -[ "$CASE_STATUS" != "0" ] || fail "unpinned manifest unexpectedly succeeded" -assert_contains "$CASE_LOG" "must contain exactly one quoted maestro version, install_url, and sha256 pin" -[ ! -e "$CASE_CURL_ARGS" ] || fail "unpinned configured version reached the network" -assert_previous_preserved -assert_no_staging_dirs - -run_case wrong-release-url "$expected_version" "$valid_archive" "$expected_sha256" 0 0 wrong-url -[ "$CASE_STATUS" != "0" ] || fail "manifest with a non-release URL unexpectedly succeeded" -assert_contains "$CASE_LOG" "install_url must be the exact release asset for cli-$expected_version" -[ ! -e "$CASE_CURL_ARGS" ] || fail "invalid release URL reached the network" -assert_previous_preserved -assert_no_staging_dirs - -run_case invalid-sha "$expected_version" "$valid_archive" "$expected_sha256" 0 0 invalid-sha -[ "$CASE_STATUS" != "0" ] || fail "manifest with an invalid checksum unexpectedly succeeded" -assert_contains "$CASE_LOG" "sha256 must be exactly 64 hexadecimal characters" -[ ! -e "$CASE_CURL_ARGS" ] || fail "invalid checksum metadata reached the network" -assert_previous_preserved -assert_no_staging_dirs - -run_case promotion-rollback "$expected_version" "$valid_archive" "$expected_sha256" 1 -[ "$CASE_STATUS" != "0" ] || fail "failed atomic promotion unexpectedly succeeded" -assert_contains "$CASE_LOG" "previous installation was restored" -assert_previous_preserved -assert_no_staging_dirs - -run_case transport-failure "$expected_version" "$valid_archive" "$expected_sha256" 0 22 -[ "$CASE_STATUS" != "0" ] || fail "transport failure unexpectedly succeeded" -assert_previous_preserved -assert_no_staging_dirs - -echo "setup-maestro installer tests passed" diff --git a/tools/release/public-consumer-smoke.test.mts b/tools/release/public-consumer-smoke.test.mts index 2017f69da..189c1555e 100644 --- a/tools/release/public-consumer-smoke.test.mts +++ b/tools/release/public-consumer-smoke.test.mts @@ -100,7 +100,7 @@ if (fixtureMode === 'prepare-npm') { JSON.stringify({ lock: frozen, plan: publicConsumerPlan(frozen, ['sdk'], graph(products)), - deadlineMilliseconds: Date.now() + (scenario === 'timeout' ? 2000 : 30000), + deadlineMilliseconds: scenario === 'expired' ? 0 : Date.now() + 60000, }), ); process.exit(0); diff --git a/tools/release/public-consumer-smoke.test.sh b/tools/release/public-consumer-smoke.test.sh index d48298edf..3789dc69f 100644 --- a/tools/release/public-consumer-smoke.test.sh +++ b/tools/release/public-consumer-smoke.test.sh @@ -12,6 +12,19 @@ while IFS= read -r -d '' entry; do clean+=("$entry"); done < "$scratch/cargo/env (cd "$scratch/cargo"; "${clean[@]}" cargo generate-lockfile) [[ -f "$scratch/cargo/Cargo.lock" ]] mkdir "$scratch/bin" +export PUBLIC_PROBE_TIMEOUT="$(command -v gtimeout || command -v timeout)" +cat > "$scratch/bin/gtimeout" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +# Stage/startup has its own generous budget. Shorten only the deliberate +# hanging consumer, while still exercising the real process-group timeout. +if [[ " $* " == *" npm install "* && -n "${HANG_PUBLIC_PROBE:-}" ]]; then + sleep 2 + set -- "$1" 2s "${@:3}" +fi +exec "$PUBLIC_PROBE_TIMEOUT" "$@" +SH +chmod +x "$scratch/bin/gtimeout" cat > "$scratch/bin/npm" <<'SH' #!/usr/bin/env bash set -euo pipefail @@ -28,16 +41,18 @@ SH chmod +x "$scratch/bin/npm" export PATH="$scratch/bin:$PATH" SENSITIVE_TOKEN=must-not-survive CARGO_REGISTRY_TOKEN=must-not-survive export PUBLIC_PROBE_COUNTER="$scratch/attempts" PUBLIC_PROBE_FIXTURE="$source_root/tools/release/public-consumer-smoke.test.mts" -for mode in success fail timeout; do +for mode in success fail timeout expired; do mkdir "$scratch/$mode" bun tools/release/public-consumer-smoke.test.mts prepare-npm "$scratch/$mode" "$mode" unset FAIL_PUBLIC_PROBE HANG_PUBLIC_PROBE PUBLIC_PROBE_CHILD expected=0 + if [[ "$mode" == expired ]]; then expected=1; fi if [[ "$mode" == fail ]]; then export FAIL_PUBLIC_PROBE=1; expected=7; fi if [[ "$mode" == timeout ]]; then export HANG_PUBLIC_PROBE=1 PUBLIC_PROBE_CHILD="$scratch/child-pid"; expected=124; fi status=0 bash tools/release/public-consumer-smoke.sh --surface "$scratch/$mode" npm > "$scratch/$mode/output" 2>&1 || status=$? if [[ "$status" != "$expected" ]]; then cat "$scratch/$mode/output" >&2; exit 1; fi + if [[ "$mode" == expired ]]; then grep -q "shared public-consumer deadline reached" "$scratch/$mode/output"; fi bun tools/release/public-consumer-smoke.test.mts assert-npm "$scratch/$mode" "$mode" done [[ "$(wc -l < "$scratch/attempts")" -eq 3 ]] diff --git a/tools/release/release_plan.mts b/tools/release/release_plan.mts index ca8ef94ae..fd84abb4e 100644 --- a/tools/release/release_plan.mts +++ b/tools/release/release_plan.mts @@ -57,6 +57,9 @@ function printGithubOutput(plan) { console.log( `has_extension_artifacts=${String(extensionArtifactProducts.length > 0).toLowerCase()}`, ); + console.log( + `requires_native_extension_lifecycle_evidence=${extensionArtifactProductsForReleaseProducts(products, { family: 'native', prefix: TOOL }).length > 0}`, + ); console.log(`products_json=${JSON.stringify(products)}`); console.log(`extension_products_json=${JSON.stringify(extensionProducts)}`); console.log(