diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml index 986446a6b55..f36c58b84ec 100644 --- a/.github/actions/ccache-setup/action.yml +++ b/.github/actions/ccache-setup/action.yml @@ -62,36 +62,10 @@ runs: install -y --no-install-recommends ccache; then echo "ccache installed offline from the staged .deb bundle" else - # Same defence in depth as install-apt-deps: Acquire timeouts drop - # a stalled connection, `timeout` hard-kills a wedged apt-get, and - # only then does the retry loop get a non-zero exit to act on. - # 60s+90s x2: ccache is a ~700 KB package, and this loop has no - # budget input of its own, so it has to stay small enough to chain - # after install-apt-deps inside a 10-minute job. - - # No wolfSSL job installs from the runner's Google/Microsoft apt repos, - # and a bad index on either fails apt-get update for everyone. Drop them. - grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \ - /etc/apt/sources.list.d/ 2>/dev/null | xargs -r sudo rm -vf || true - APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 - -o Acquire::https::Timeout=30) - ok="" - for i in 1 2; do - sudo dpkg --configure -a >/dev/null 2>&1 || true - if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 60 \ - apt-get "${APT_OPTS[@]}" update -q && \ - sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 90 \ - apt-get "${APT_OPTS[@]}" install -y \ - --no-install-recommends ccache; then - ok=1 - break - fi - echo "::warning::ccache apt install failed (attempt $i/2)" - # No sleep after the last attempt - this loop is not budgeted by - # the caller and chains onto install-apt-deps in the same job. - [ "$i" -eq 2 ] || sleep 5 - done - [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; } + # Short limits: this follows install-apt-deps in 10-minute jobs. + "$GITHUB_ACTION_PATH/../../scripts/apt-install.sh" --tries 2 \ + --update-timeout 60 --install-timeout 90 --drop-vendor-sources \ + --no-install-recommends ccache fi elif [ "${{ runner.os }}" = "macOS" ]; then brew install ccache diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml index c929729999d..e7712a236db 100644 --- a/.github/actions/install-apt-deps/action.yml +++ b/.github/actions/install-apt-deps/action.yml @@ -13,10 +13,10 @@ inputs: Nominal wall-clock for the whole retry loop, split across the attempts as per-command deadlines, so a wedged mirror is reported by this action instead of the job being cancelled around it. The loop overshoots it by - retry-delay plus 10s of SIGKILL grace, and the per-command floors make - values below retries*80 inert. This, plus pull-timeout, plus any - ccache-setup in the same job, has to fit the caller's timeout-minutes - - the defaults need ~16 minutes. + the retry delays plus up to 20s of SIGKILL grace per attempt, and the + per-command floors make values below retries*80 inert. This, plus + pull-timeout, plus any ccache-setup in the same job, has to fit the + caller's timeout-minutes - the defaults need ~16 minutes. required: false default: '600' pull-timeout: @@ -37,11 +37,12 @@ inputs: description: > Tag of a prebuilt .deb bundle published to ghcr.io/wolfssl/wolfssl-ci-debs by the ci-deps-image workflow - (e.g. "ubuntu-24.04-minimal"). When set, the packages are installed - from that bundle with no network access at all and the apt path below - is skipped. The install is all-or-nothing, so any failure - bundle - missing, not public, or not covering every requested package - falls - back to the apt path. Always safe to set; leave empty to use apt only. + (e.g. "ubuntu-24.04-minimal"). When set, on x64 runners, the packages + are installed from that bundle with no network access at all and the + apt path below is skipped. The install is all-or-nothing, so any + failure - bundle missing, not public, or not covering every requested + package - falls back to the apt path. Always safe to set; leave empty + to use apt only. .github/scripts/check-ci-deps.py checks on every PR that the tag exists and carries every package named above. required: false @@ -78,7 +79,8 @@ runs: # packages). - name: Install from the ghcr .deb bundle (offline) id: ghcr - if: inputs.ghcr-debs-tag != '' + # Bundles hold amd64 .debs, so other runners go straight to apt. + if: inputs.ghcr-debs-tag != '' && runner.arch == 'X64' shell: bash run: | set -u @@ -200,67 +202,18 @@ runs: if: steps.ghcr.outputs.satisfied != 'true' shell: bash run: | - RETRIES=${{ inputs.retries }} - DELAY=${{ inputs.retry-delay }} - BUDGET=${{ inputs.budget-seconds }} + # Update gets half of each attempt (20s..90s, apt's 3 x 30s ladder). + PER=$(( ${{ inputs.budget-seconds }} / ${{ inputs.retries }} )) + UPD=$((PER / 2)) + [ "$UPD" -le 90 ] || UPD=90 + [ "$UPD" -ge 20 ] || UPD=20 + INS=$((PER - UPD)) + [ "$INS" -ge 40 ] || INS=40 NO_REC="" if [ "${{ inputs.no-install-recommends }}" = "true" ]; then NO_REC="--no-install-recommends" fi - - # A wedged mirror hangs apt rather than failing it, so the retry loop - # below never fired and the job burned its whole budget instead. - # Defend in depth: apt drops a stalled connection after 30s and retries - # it (Acquire timeouts - this is what actually detects a wedge, in - # ~90s), `timeout` hard-kills an apt-get that wedged outside its own - # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a - # retry can land on a different mirror. apt resumes from - # archives/partial/, so a killed transfer is not restarted from - # scratch. - - # No wolfSSL job installs from the runner's Google/Microsoft apt repos, - # and a bad index on either fails apt-get update for everyone. Drop them. - grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \ - /etc/apt/sources.list.d/ 2>/dev/null | xargs -r sudo rm -vf || true - APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 - -o Acquire::https::Timeout=30) - - DEADLINE=$(($(date +%s) + BUDGET)) - - # sudo resets the environment, so DEBIAN_FRONTEND has to ride along - # on each privileged command rather than being exported once. - for i in $(seq 1 $RETRIES); do - # Split what is LEFT over the attempts still to come, rather than - # slicing the budget up front: an attempt that ends early hands its - # remainder to the next one instead of dropping it. - # - # update gets half of an attempt, capped at 90s because that is what - # apt's own Acquire ladder (3 retries x a 30s timeout) needs to work - # through a stalled mirror and move on; install gets the rest. A - # sixth of an attempt used to be 50s of the sssd job's 600s budget, - # so update was killed mid-transfer twice and the job failed with - # 80% of its budget unspent. The floors keep a small budget usable; - # they are what makes it overshoot. - PER=$(( (DEADLINE - $(date +%s)) / (RETRIES - i + 1) )) - UPD=$((PER / 2)) - [ "$UPD" -le 90 ] || UPD=90 - [ "$UPD" -ge 20 ] || UPD=20 - INS=$((PER - UPD)) - [ "$INS" -ge 40 ] || INS=40 - # A previous attempt killed mid-unpack leaves dpkg needing this. - sudo dpkg --configure -a >/dev/null 2>&1 || true - if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $UPD \ - apt-get "${APT_OPTS[@]}" update -q && \ - sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $INS \ - apt-get "${APT_OPTS[@]}" install -y \ - $NO_REC ${{ inputs.packages }}; then - exit 0 - fi - if [ "$i" -eq "$RETRIES" ] || [ "$(date +%s)" -ge "$DEADLINE" ]; then - echo "::error::apt-get failed after $i attempt(s) in ${BUDGET}s" - exit 1 - fi - echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..." - sleep $DELAY - DELAY=$((DELAY * 2)) - done + "$GITHUB_ACTION_PATH/../../scripts/apt-install.sh" \ + --tries ${{ inputs.retries }} --delay ${{ inputs.retry-delay }} \ + --update-timeout "$UPD" --install-timeout "$INS" \ + --drop-vendor-sources $NO_REC ${{ inputs.packages }} diff --git a/.github/ci-deps/packages-ubuntu-24.04-full.txt b/.github/ci-deps/packages-ubuntu-24.04-full.txt index 9e4cbc106cf..0964fdba86b 100644 --- a/.github/ci-deps/packages-ubuntu-24.04-full.txt +++ b/.github/ci-deps/packages-ubuntu-24.04-full.txt @@ -8,7 +8,9 @@ autoconf autoconf-archive automake autopoint +autotools-dev bc +bison bubblewrap build-essential ccache @@ -20,12 +22,14 @@ g++-10 g++-11 g++-12 g++-9 +g++-multilib gcc-10 gcc-11 gcc-12 gcc-9 gcc-multilib gettext +git gyp jq krb5-admin-server @@ -88,9 +92,11 @@ ninja-build pkg-config pkgconf psmisc +python3 python3-docutils python3-impacket python3-ldb +python3-pip python3-psutil python3-yaml shellcheck diff --git a/.github/scripts/apt-install.sh b/.github/scripts/apt-install.sh new file mode 100755 index 00000000000..c12d4bf2bf5 --- /dev/null +++ b/.github/scripts/apt-install.sh @@ -0,0 +1,105 @@ +#!/bin/sh +# Install apt packages with retries and hard timeouts, so a stuck mirror fails +# an attempt instead of hanging the job. +# +# usage: apt-install.sh [options] [--] [apt-get install args...] +# --tries N attempts (default 3) +# --update-timeout S limit for each apt-get update (default 90) +# --install-timeout S limit for each apt-get install (default 300) +# --delay S wait before the first retry, doubled after (5) +# --no-update skip apt-get update +# --drop-vendor-sources remove the runner's Google/Microsoft apt sources +# --warn-only report the last failure as a warning, not an error +# The remaining args go to apt-get install as is, so --no-install-recommends +# and --download-only work. With none, only apt-get update runs. +set -u + +tries=3 +update_timeout=90 +install_timeout=300 +delay=5 +update=1 +drop_vendor=0 +level=error + +while [ $# -gt 0 ]; do + case $1 in + --tries) tries=$2; shift ;; + --update-timeout) update_timeout=$2; shift ;; + --install-timeout) install_timeout=$2; shift ;; + --delay) delay=$2; shift ;; + --no-update) update=0 ;; + --drop-vendor-sources) drop_vendor=1 ;; + --warn-only) level=warning ;; + --) shift; break ;; + *) break ;; + esac + shift +done + +for n in "$tries" "$update_timeout" "$install_timeout" "$delay"; do + case $n in + ''|*[!0-9]*) echo "apt-install.sh: not a number: '$n'" >&2; exit 2 ;; + esac +done +for n in "$tries" "$update_timeout" "$install_timeout"; do + [ "$n" -gt 0 ] || { + echo "apt-install.sh: must be positive: '$n'" >&2 + exit 2 + } +done + +export DEBIAN_FRONTEND=noninteractive +if [ "$(id -u)" -eq 0 ]; then + as_root() { "$@"; } +else + # sudo resets the environment, so DEBIAN_FRONTEND has to go through it. + as_root() { sudo DEBIAN_FRONTEND=noninteractive "$@"; } +fi + +apt_get() { + limit=$1 + shift + as_root timeout -k 10 "$limit" apt-get -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 "$@" +} + +attempt() { + # An attempt killed mid-unpack leaves dpkg needing this. + as_root dpkg --configure -a || true + if [ "$update" -eq 1 ]; then + step=update + apt_get "$update_timeout" update -q || return + fi + [ $# -gt 0 ] || return 0 + step=install + apt_get "$install_timeout" install -y -q "$@" +} + +if [ "$drop_vendor" -eq 1 ]; then + # No job uses these, and a bad index on either fails apt-get update. + grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \ + /etc/apt/sources.list.d/ 2>/dev/null | + while read -r f; do as_root rm -vf "$f"; done +fi + +i=1 +while :; do + rc=0 + attempt "$@" || rc=$? + [ "$rc" -ne 0 ] || exit 0 + # A killed apt-get prints nothing, so name the timeout. + case $rc in + 124|137) why="timed out" ;; + *) why="failed with exit code $rc" ;; + esac + [ "$i" -lt "$tries" ] || break + echo "::warning::apt-get $step $why (attempt $i/$tries)," \ + "retrying in ${delay}s" + sleep "$delay" + delay=$((delay * 2)) + i=$((i + 1)) +done + +echo "::$level::apt-get $step $why (attempt $i/$tries), giving up${*:+ on: $*}" +exit "$rc" diff --git a/.github/scripts/download-deb-closure.sh b/.github/scripts/download-deb-closure.sh index d4f72e46158..881dc8cd252 100755 --- a/.github/scripts/download-deb-closure.sh +++ b/.github/scripts/download-deb-closure.sh @@ -16,27 +16,14 @@ set -uo pipefail LIST=${1:?package list} DEST=${2:?destination directory} +APT_INSTALL="$(dirname "$0")/apt-install.sh" + mapfile -t PKGS < <(grep -vE '^[[:space:]]*#|^[[:space:]]*$' "$LIST") echo "Packages (${#PKGS[@]}): ${PKGS[*]}" -export DEBIAN_FRONTEND=noninteractive # Not rm -rf: in the container this directory is a bind mount. mkdir -p "$DEST" && rm -f "$DEST"/*.deb apt-get clean -# No wolfSSL job installs from the runner's Google/Microsoft apt repos, and a -# bad index on either fails apt-get update for everyone. Drop them. Already -# root here, so no sudo; the container images carry neither repo, so this is a -# no-op there. -grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \ - /etc/apt/sources.list.d/ 2>/dev/null | xargs -r rm -vf || true -# A single stalled mirror connection once hung -full for ~20 min (it normally -# finishes in a few). retry() only re-runs on a non-zero exit, so a hang never -# tripped it. Defend in depth: apt drops a stalled connection after 30s and -# retries it (Acquire timeouts), `timeout` hard-kills a wedged apt-get, then -# retry() re-runs from scratch. -APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 - -o Acquire::https::Timeout=30) -retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; } -retry timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q +"$APT_INSTALL" --tries 5 --update-timeout 120 --drop-vendor-sources # Download each package's closure independently (requested package + any # dependency not already installed) without installing. Per package, not one # resolve of the whole list, so one unbundleable package - e.g. a conflict in @@ -44,8 +31,8 @@ retry timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q # apt for anything missing. skipped=0 for pkg in "${PKGS[@]}"; do - retry timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y --download-only "$pkg" \ - || { echo "::warning::could not download $pkg"; skipped=$((skipped+1)); } + "$APT_INSTALL" --tries 5 --no-update --warn-only --download-only "$pkg" \ + || skipped=$((skipped+1)) done cp /var/cache/apt/archives/*.deb "$DEST/" 2>/dev/null || true # The steps that index and package these run unprivileged, and in the diff --git a/.github/workflows/afalg.yml b/.github/workflows/afalg.yml index 2b47ac75013..886eb92633f 100644 --- a/.github/workflows/afalg.yml +++ b/.github/workflows/afalg.yml @@ -82,8 +82,8 @@ jobs: done if [ -n "$missing" ]; then grep -rn 'algif_' /etc/modprobe.d /lib/modprobe.d || true - sudo apt-get update -qq || true - sudo apt-get install -y "linux-modules-extra-$(uname -r)" || true + .github/scripts/apt-install.sh --tries 2 --warn-only \ + --drop-vendor-sources "linux-modules-extra-$(uname -r)" || true for m in $missing; do if sudo modprobe --ignore-install "$m"; then echo "modprobe $m: ok after modules-extra" diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml index 782958c6589..a4842cadb77 100644 --- a/.github/workflows/ci-deps-image.yml +++ b/.github/workflows/ci-deps-image.yml @@ -46,6 +46,7 @@ on: branches: [ master ] paths: - '.github/ci-deps/**' + - '.github/scripts/apt-install.sh' - '.github/scripts/download-deb-closure.sh' - '.github/workflows/ci-deps-image.yml' workflow_dispatch: @@ -223,6 +224,11 @@ jobs: echo "rebuild=true" >> "$GITHUB_OUTPUT" fi + - uses: actions/checkout@v5 + if: steps.check.outputs.rebuild == 'true' + with: + sparse-checkout: .github/scripts + - name: Resolve and download the .deb closure if: steps.check.outputs.rebuild == 'true' shell: bash @@ -236,23 +242,15 @@ jobs: # check step so it stays in step with the pkgset the gate compares. read -r -a PKGS <<< "${{ steps.check.outputs.pkgs }} linux-headers-$K" echo "Packages: ${PKGS[*]}" - export DEBIAN_FRONTEND=noninteractive rm -rf debs && mkdir -p debs sudo apt-get clean - APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) - # 2 attempts, not 5: this job's timeout-minutes is 20, and an - # attempt cut off mid-flight reports nothing. The explicit return - # keeps the real exit code (124 from timeout) and stops a bare - # `sleep` from making an exhausted retry look like success. - retry() { local i rc=0; for i in 1 2; do "$@" && return 0; rc=$?; [ "$i" -eq 2 ] || sleep 5; done; return "$rc"; } - retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q # The whole set is required and this bundle is small, so resolve it as # one closure and let any download failure fail the job. We push only # on success, so a transient mirror error keeps the last good bundle # rather than publishing a partial one - which the kernel-label skip # would then pin in place until the kernel next changes (~monthly). - retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \ - --download-only "${PKGS[@]}" + .github/scripts/apt-install.sh --tries 2 --update-timeout 60 \ + --drop-vendor-sources --download-only "${PKGS[@]}" sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files" test -n "$(ls debs/*.deb 2>/dev/null)" # headers are never preinstalled diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml index d8556d89475..5a4d4f6527a 100644 --- a/.github/workflows/cross-library.yml +++ b/.github/workflows/cross-library.yml @@ -58,37 +58,21 @@ jobs: image: ${{ inputs.container }} timeout-minutes: 20 steps: + # No git yet, so this is a REST download, only to get apt-install.sh. + - name: Fetch the apt script + uses: actions/checkout@v5 + with: + path: apt-script + # Minimal-image containers ship without git/toolchain; install them # before checkout. Product-specific extras come from apt_packages. - name: Install build tools run: | - set -eux - export DEBIAN_FRONTEND=noninteractive - # These containers are bare images with no bash, so this step runs - # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it - # word-splits. - # A wedged mirror hangs apt instead of failing it. Acquire timeouts - # drop a stalled connection (and are what actually detects a wedge), - # `timeout` hard-kills apt-get if it wedges outside its own I/O loop, - # and the loop then retries - re-reading the mirror list. Two - # attempts at 60s+300s fit inside this job's timeout-minutes; apt - # resumes from archives/partial/, so a killed transfer is not lost. - APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30" - for i in 1 2; do - # An attempt killed mid-unpack leaves dpkg needing this, or the - # next apt-get aborts instantly in debSystem::Lock(). - dpkg --configure -a >/dev/null 2>&1 || true - if timeout -k 10 60 apt-get $APT_OPTS update -q && \ - timeout -k 10 300 apt-get $APT_OPTS install -y \ - --no-install-recommends \ - build-essential autoconf automake libtool pkg-config \ - git ca-certificates ${{ inputs.apt_packages }}; then - break - fi - test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; } - echo "::warning::apt-get failed (attempt $i/2)" - sleep 5 - done + # Two attempts fit inside this job's timeout-minutes. + sh apt-script/.github/scripts/apt-install.sh --tries 2 \ + --update-timeout 60 --no-install-recommends \ + build-essential autoconf automake libtool pkg-config \ + git ca-certificates ${{ inputs.apt_packages }} # Building only needs the commit under test, not history. The break check # that needs history runs in the compile job, not here. @@ -144,37 +128,21 @@ jobs: matrix: ref_mode: [ head, latest ] steps: + # No git yet, so this is a REST download, only to get apt-install.sh. + - name: Fetch the apt script + uses: actions/checkout@v5 + with: + path: apt-script + # Minimal-image containers ship without git/toolchain; install them # before checkout. Product-specific extras come from apt_packages. - name: Install build tools run: | - set -eux - export DEBIAN_FRONTEND=noninteractive - # These containers are bare images with no bash, so this step runs - # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it - # word-splits. - # A wedged mirror hangs apt instead of failing it. Acquire timeouts - # drop a stalled connection (and are what actually detects a wedge), - # `timeout` hard-kills apt-get if it wedges outside its own I/O loop, - # and the loop then retries - re-reading the mirror list. Two - # attempts at 60s+300s fit inside this job's timeout-minutes; apt - # resumes from archives/partial/, so a killed transfer is not lost. - APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30" - for i in 1 2; do - # An attempt killed mid-unpack leaves dpkg needing this, or the - # next apt-get aborts instantly in debSystem::Lock(). - dpkg --configure -a >/dev/null 2>&1 || true - if timeout -k 10 60 apt-get $APT_OPTS update -q && \ - timeout -k 10 300 apt-get $APT_OPTS install -y \ - --no-install-recommends \ - build-essential autoconf automake libtool pkg-config \ - git ca-certificates ${{ inputs.apt_packages }}; then - break - fi - test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; } - echo "::warning::apt-get failed (attempt $i/2)" - sleep 5 - done + # Two attempts fit inside this job's timeout-minutes. + sh apt-script/.github/scripts/apt-install.sh --tries 2 \ + --update-timeout 60 --no-install-recommends \ + build-essential autoconf automake libtool pkg-config \ + git ca-certificates ${{ inputs.apt_packages }} # This job does not build wolfSSL, but the latest leg still checks out # wolfSSL history because check-break.sh scans commit messages here. The diff --git a/.github/workflows/cross-library/README.md b/.github/workflows/cross-library/README.md index 4dc553683fc..476988ed6a7 100644 --- a/.github/workflows/cross-library/README.md +++ b/.github/workflows/cross-library/README.md @@ -29,7 +29,9 @@ default branch and at its latest tagged release. The engine (`cross-library.yml`) runs one job in a clean container (`ubuntu:24.04` by default; a caller may pass `debian:13`): -1. **Install build tools** with `apt-get` (`+ apt_packages` from the caller). +1. **Install build tools** with `.github/scripts/apt-install.sh` + (`+ apt_packages` from the caller). The image has no git yet, so a first + checkout downloads the script through the REST API. 2. **Checkout wolfSSL** (full history + tags, for the break check below). 3. **`build-wolfssl.sh`** builds this checkout's wolfSSL with the product's required `wolfssl_configure` flags and installs it to a local dir. diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml index 26df28dfbaf..18b60c0c8fd 100644 --- a/.github/workflows/falcon-interop.yml +++ b/.github/workflows/falcon-interop.yml @@ -87,7 +87,7 @@ jobs: if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 needs: build_liboqs - timeout-minutes: 15 + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -95,37 +95,6 @@ jobs: - { name: 'default', opts: '--enable-falcon' } - { name: 'smallest-mem', opts: '--enable-falcon=smallest-mem' } steps: - - name: Install build tools - run: | - # A wedged mirror hangs apt instead of failing it. Acquire timeouts - # drop a stalled connection, `timeout` hard-kills apt-get if it - # wedges anyway, and the loop then retries against a fresh mirror. - # Two attempts at 60s+300s stay inside this job's timeout-minutes. - - # No wolfSSL job installs from the runner's Google/Microsoft apt repos, - # and a bad index on either fails apt-get update for everyone. Drop them. - grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \ - /etc/apt/sources.list.d/ 2>/dev/null | xargs -r sudo rm -vf || true - APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 - -o Acquire::https::Timeout=30) - apt_retry() { - local i - for i in 1 2; do - # An attempt killed mid-unpack leaves dpkg needing this, or the - # next apt-get aborts instantly in debSystem::Lock(). - sudo dpkg --configure -a >/dev/null 2>&1 || true - if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ - sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then - return 0 - fi - echo "::warning::apt-get failed (attempt $i/2)" - sleep 5 - done - echo "::error::apt-get failed after 2 attempts" - return 1 - } - apt_retry ninja-build - # Check out wolfSSL first: actions/checkout runs "git clean -ffdx", which # would delete an untracked oqs-install/ placed in the workspace by the # cache/artifact steps below. Restoring liboqs after the checkout keeps it. @@ -134,6 +103,12 @@ jobs: with: fetch-depth: 1 + - name: Install build tools + uses: ./.github/actions/install-apt-deps + with: + packages: ninja-build + ghcr-debs-tag: ubuntu-24.04-full + - name: Restore liboqs from cache uses: actions/cache/restore@v5 id: cache @@ -209,42 +184,17 @@ jobs: # this is the only config where the verify path is all there is. - { name: 'verify-only', runner: ubuntu-latest, opts: '--enable-falcon', cppflags: '-DWOLFSSL_FALCON_VERIFY_ONLY' } steps: - - name: Install build tools - run: | - # A wedged mirror hangs apt instead of failing it. Acquire timeouts - # drop a stalled connection, `timeout` hard-kills apt-get if it - # wedges anyway, and the loop then retries against a fresh mirror. - # Two attempts at 60s+300s stay inside this job's timeout-minutes. - - # No wolfSSL job installs from the runner's Google/Microsoft apt repos, - # and a bad index on either fails apt-get update for everyone. Drop them. - grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \ - /etc/apt/sources.list.d/ 2>/dev/null | xargs -r sudo rm -vf || true - APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 - -o Acquire::https::Timeout=30) - apt_retry() { - local i - for i in 1 2; do - # An attempt killed mid-unpack leaves dpkg needing this, or the - # next apt-get aborts instantly in debSystem::Lock(). - sudo dpkg --configure -a >/dev/null 2>&1 || true - if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ - sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then - return 0 - fi - echo "::warning::apt-get failed (attempt $i/2)" - sleep 5 - done - echo "::error::apt-get failed after 2 attempts" - return 1 - } - apt_retry autoconf automake libtool - - name: Checkout wolfSSL uses: actions/checkout@v5 with: fetch-depth: 1 + - name: Install build tools + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool + ghcr-debs-tag: ubuntu-24.04-minimal + # Robustness guard: an AVX2-compiled binary SIGILLs on a non-AVX2 host. # GitHub x86 runners always have AVX2+FMA, but fail loudly if that changes. - name: Require AVX2 on the runner diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml index 728e386bb2e..2a1bab2f7d6 100644 --- a/.github/workflows/sbom.yml +++ b/.github/workflows/sbom.yml @@ -820,45 +820,21 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Install build deps + SBOM validators + # bison + autotools-dev are required by strace's ./bootstrap. + # gcc-multilib + g++-multilib give strace's --enable-mpers=check + # the 32-bit/x32 compilers it needs - without them mpers is + # silently downgraded and bomtrace3 traces only native-arch + # syscalls, diverging from what bomsh's devcontainer produces. + # The rest mirror bomsh's .devcontainer/Dockerfile bomtrace3 + # stage. + - name: Install build deps + uses: ./.github/actions/install-apt-deps + with: + packages: build-essential autoconf automake libtool bison autotools-dev gcc-multilib g++-multilib python3 python3-pip git + ghcr-debs-tag: ubuntu-24.04-full + + - name: Install SBOM validators run: | - # A wedged mirror hangs apt instead of failing it. Acquire timeouts - # drop a stalled connection, `timeout` hard-kills apt-get if it - # wedges anyway, and the loop then retries against a fresh mirror. - # Two attempts at 60s+300s stay inside this job's timeout-minutes. - - # No wolfSSL job installs from the runner's Google/Microsoft apt repos, - # and a bad index on either fails apt-get update for everyone. Drop them. - grep -rlE 'dl\.google\.com|packages\.microsoft\.com' \ - /etc/apt/sources.list.d/ 2>/dev/null | xargs -r sudo rm -vf || true - APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 - -o Acquire::https::Timeout=30) - apt_retry() { - local i - for i in 1 2; do - # An attempt killed mid-unpack leaves dpkg needing this, or the - # next apt-get aborts instantly in debSystem::Lock(). - sudo dpkg --configure -a >/dev/null 2>&1 || true - if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ - sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then - return 0 - fi - echo "::warning::apt-get failed (attempt $i/2)" - sleep 5 - done - echo "::error::apt-get failed after 2 attempts" - return 1 - } - # bison + autotools-dev are required by strace's ./bootstrap. - # gcc-multilib + g++-multilib give strace's --enable-mpers=check - # the 32-bit/x32 compilers it needs - without them mpers is - # silently downgraded and bomtrace3 traces only native-arch - # syscalls, diverging from what bomsh's devcontainer produces. - # The rest mirror bomsh's .devcontainer/Dockerfile bomtrace3 - # stage. - apt_retry build-essential autoconf automake libtool \ - bison autotools-dev gcc-multilib g++-multilib \ - python3 python3-pip git python3 -m pip install --user --upgrade pip python3 -m pip install --user 'spdx-tools==0.8.*' echo "$HOME/.local/bin" >> "$GITHUB_PATH"