ui-smoke phase 3: screenshots, writable config mirrors, homing settle

.github/workflows/ci.yml

@@ -51,6 +51,18 @@ jobs:
       run: |
         set -x
         scripts/rip-environment runtests -p
+    - name: Upload UI smoke screenshots
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: ui-smoke-screenshots-gcc
+        path: |
+          tests/ui-smoke/**/screenshot.png
+          tests/ui-smoke/**/confirm.png
+          tests/ui-smoke/**/diff.png
+          tests/ui-smoke/**/diff-abs.png
+        if-no-files-found: ignore
+        retention-days: 14
     - name: Verify no untracked or modified files after test
       run: |
         .github/scripts/verify-clean-repo.sh
@@ -112,6 +124,18 @@ jobs:
       run: |
         set -x
         scripts/rip-environment runtests -p
+    - name: Upload UI smoke screenshots
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: ui-smoke-screenshots-clang
+        path: |
+          tests/ui-smoke/**/screenshot.png
+          tests/ui-smoke/**/confirm.png
+          tests/ui-smoke/**/diff.png
+          tests/ui-smoke/**/diff-abs.png
+        if-no-files-found: ignore
+        retention-days: 14
     - name: Verify no untracked or modified files after test
       run: |
         .github/scripts/verify-clean-repo.sh

debian/control.top.in

@@ -47,6 +47,9 @@ Build-Depends:
     tk@TCLTK_VERSION@-dev,
     xvfb <!nocheck>,
     x11-xserver-utils <!nocheck>,
+    x11-utils <!nocheck>,
+    gdb <!nocheck>,
+    imagemagick <!nocheck>,
     python3-opengl <!nocheck>,
     python3-pyqt5 <!nocheck>,
     python3-pyqt5.qsci <!nocheck>,

scripts/runtests.in

@@ -369,7 +369,7 @@ while getopts cnuvsph opt; do
     case "$opt" in
     c) CLEAN_ONLY=1 ;;
     n) NOCLEAN=1 ;;
-    u) NOSUDO=true ;;
+    u) NOSUDO=true; export RUNTESTS_NOSUDO=1 ;;
     v) VERBOSE=1 ;;
     s) STOP=1 ;;
     p) PRINT=1 ;;

tests/ui-smoke/.gitignore

@@ -5,3 +5,11 @@ linuxcnc.err
 linuxcnc.pid
 ui-smoke.out
 ui-smoke.err
+# confirm.png is the per-run shot; diff.png and diff-abs.png are its
+# comparison to the committed reference.png (which IS tracked, so it is
+# not listed here).
+screenshot.png
+confirm.png
+ui-smoke-qt.png
+diff.png
+diff-abs.png

tests/ui-smoke/README

@@ -19,6 +19,35 @@ Shared helpers live in _lib/:
   checkresult.sh      shared pass/fail predicate
   skip-if-missing.sh  shared skip predicate
 
+Failure diagnostics (failure path only, no cost on a green run):
+  crashdump.sh   arms a core dump and prints a native backtrace if a GUI
+                 segfaults (the Qt/dbus/GL frames PYTHONFAULTHANDLER misses)
+  screenshot.sh  photographs the Xvfb root window before teardown. On a
+                 failure it captures the cause (a GUI hung on a blocking
+                 modal leaves no core and no traceback); on a clean Phase 2
+                 run it captures confirm.png, the GUI in its post-movement
+                 idle state (final DRO / toolpath) for visual confirmation.
+                 CI uploads both as build artifacts (ui-smoke-screenshots-*
+                 in ci.yml).
+  compare.sh     on a clean run, compares confirm.png to the committed
+                 known-good reference.png (ImageMagick) and writes a
+                 highlighted diff.png, also uploaded as an artifact.
+
+Reference images:
+  reference.png  committed per-GUI known-good shot (in a --run-program test
+                 directory); diff.png is confirm.png compared against it.
+                 The comparison NEVER fails a test: freetype/font versions
+                 differ across distros, so some drift is expected. The diff
+                 is only recorded, not used to gate the test.
+
+  Create or refresh the references on a built run-in-place tree:
+      . scripts/rip-environment
+      tests/ui-smoke/_lib/make-references.sh           # all run-program GUIs
+      tests/ui-smoke/_lib/make-references.sh axis      # just one
+  This sets UI_SMOKE_UPDATE_REFERENCE=1, which makes compare.sh save each
+  clean confirm shot as that test's reference.png. Review the PNGs before
+  committing; they are baselines from the generating machine's fonts.
+
 Skip vs fail policy: the only condition we skip on is xvfb-run absence
 (rare local dev env). Python and gi typelib deps the GUIs need are
 declared in debian/control under !nocheck so apt-get build-dep

tests/ui-smoke/_lib/compare.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+# Known-good image comparison for the UI smoke confirm shots. Complements
+# screenshot.sh: that grabs confirm.png on a clean run; this compares it to
+# a committed reference.png and writes two visual diffs. Like screenshot.sh
+# it carries no state and is a logged no-op whenever it cannot run, so it can
+# never turn a pass into a fail.
+#
+# Two diff images, both at 0% fuzz so nothing is hidden:
+#   diff.png      red highlight over a faded copy of the reference (where it
+#                 changed, ImageMagick "compare" style)
+#   diff-abs.png  absolute per-channel difference |b - a|, black where equal
+#                 and bright where it changed (unbiased, magnitude preserved)
+#
+# Policy: we never fail a test on the image difference. freetype/font
+# versions differ across distros, so some drift is expected; the diffs are
+# here to record what changed, not to gate. The function always returns 0.
+#
+# Local "make a known-good image" workflow: run a test with
+# UI_SMOKE_UPDATE_REFERENCE=1 (see make-references.sh) and the freshly
+# captured shot is saved as the committed reference instead of compared.
+
+# Set IM_COMPARE and IM_CONVERT to the IM7 or IM6 entry points; return 1
+# if ImageMagick is absent.
+_im_tools() {
+    if command -v magick >/dev/null 2>&1; then
+        IM_COMPARE="magick compare"; IM_CONVERT="magick"
+    elif command -v compare >/dev/null 2>&1; then
+        IM_COMPARE="compare"; IM_CONVERT="convert"
+    else
+        return 1
+    fi
+}
+
+# compare_to_reference <shot> <reference> <diff> <diff_abs>
+# Compare the captured shot to the committed reference, writing a red-highlight
+# diff and an absolute-difference diff. Always returns 0.
+compare_to_reference() {
+    shot="$1"
+    reference="$2"
+    diff="$3"
+    diff_abs="$4"
+
+    if [ ! -s "$shot" ]; then
+        echo "compare: no shot at $shot, skipping"
+        return 0
+    fi
+
+    # Update mode: adopt this shot as the new known-good reference.
+    if [ "${UI_SMOKE_UPDATE_REFERENCE:-}" = "1" ]; then
+        if cp -f "$shot" "$reference"; then
+            echo "compare: saved reference $reference (UI_SMOKE_UPDATE_REFERENCE=1)"
+        else
+            echo "compare: failed to save reference $reference"
+        fi
+        return 0
+    fi
+
+    if [ ! -s "$reference" ]; then
+        echo "compare: no reference at $reference yet, skipping (run with UI_SMOKE_UPDATE_REFERENCE=1 to create one)"
+        return 0
+    fi
+
+    _im_tools || {
+        echo "compare: no ImageMagick available, skipping"
+        return 0
+    }
+
+    # Red-highlight diff at 0% fuzz: -metric AE counts every differing pixel
+    # (interpretable), and the same call writes diff.png. compare exits 0
+    # (identical), 1 (differ) or 2 (error, e.g. the shots are different
+    # sizes). We log the outcome and always succeed.
+    metric=$($IM_COMPARE -metric AE -fuzz 0% "$reference" "$shot" "$diff" 2>&1)
+    rc=$?
+    case "$rc" in
+        0) echo "compare: $shot matches $reference (AE=$metric)" ;;
+        1) echo "compare: $shot differs from $reference (AE=$metric differing pixels); diff at $diff" ;;
+        *) echo "compare: could not compare $shot to $reference (rc=$rc): $metric" ;;
+    esac
+
+    # Absolute-difference diff |b - a|: black where equal, bright where it
+    # changed. Unbiased (direction does not matter) and keeps magnitude.
+    if $IM_CONVERT "$reference" "$shot" -compose difference -composite "$diff_abs" 2>/dev/null; then
+        echo "compare: absolute-difference diff at $diff_abs"
+    else
+        echo "compare: could not write absolute-difference diff $diff_abs"
+    fi
+    return 0
+}

tests/ui-smoke/_lib/crashdump.sh

@@ -1,45 +1,59 @@
 #!/bin/bash
-# Native crash capture for the UI smoke launchers. A GUI segfault is the
-# failure these tests most need to explain, and it lands in C/C++ (Qt,
-# dbus, GL) where PYTHONFAULTHANDLER stops at the event-loop frame. Arm a
-# core dump before launch; after the run, if the GUI left a core, print a
-# native backtrace into the log so CI shows the faulting frame directly.
-# Source with LIB_DIR set; runs only on the failure path, so green runs
-# pay nothing.
+# Native crash capture for the UI smoke launchers. A GUI segfault lands in
+# C/C++ (Qt, dbus, GL); PYTHONFAULTHANDLER (set in launch-env.sh) prints a
+# Python traceback to linuxcnc.err naming the frame that called in, which
+# is the reliable, environment-independent crash signal and is surfaced in
+# every failure log. This helper adds a best-effort native backtrace on
+# top: arm a core dump before launch, and after the run, if a readable
+# core from this run is present, gdb-print its backtrace. The core only
+# materialises when we can point kernel.core_pattern at a writable dir,
+# which needs root; non-root runs (CI, local -u) keep the Python traceback
+# and skip the native one. Source with LIB_DIR set; runs only on the
+# failure path, so green runs pay nothing.
 
 crashdump_arm() {
     CORE_DIR="$(mktemp -d -t ui-smoke-cores.XXXXXX)"
     export CORE_DIR
     ulimit -c unlimited 2>/dev/null || true
-    # core_pattern is global; only set it if already root. Never sudo:
-    # the suite must run unattended. Non-root falls back to a cwd "core".
-    if [ "$(id -u)" = 0 ]; then
+    # core_pattern is global; only set it if already root, never sudo
+    # (the suite must run unattended), and skip under runtests -u so a
+    # user opting out of root tweaks is respected even when invoking the
+    # suite from a root shell. Non-root falls back to a cwd "core".
+    if [ "$(id -u)" = 0 ] && [ "${RUNTESTS_NOSUDO:-0}" != "1" ]; then
         sysctl -w "kernel.core_pattern=$CORE_DIR/core.%e.%p" >/dev/null 2>&1 || true
     fi
 }
 
 crashdump_report() {
     [ -n "${CORE_DIR:-}" ] || return 0
-    local core
-    # shellcheck disable=SC2012  # mktemp dir, no odd filenames
-    core=$(ls -t "$CORE_DIR"/core* ./core* /tmp/core* 2>/dev/null | head -1)
-    if [ -n "$core" ]; then
+    local c core=""
+    # Only trust a core we know is from this run and can actually read:
+    # one the kernel wrote into our fresh CORE_DIR (root path, where we set
+    # core_pattern), or a relative "core" in the cwd that postdates arming.
+    # A broad /tmp glob would pick up a stale or foreign core (often root-
+    # owned), and gdb would just print "Permission denied".
+    for c in "$CORE_DIR"/core*; do
+        [ -e "$c" ] && [ -r "$c" ] && { core="$c"; break; }
+    done
+    if [ -z "$core" ]; then
+        for c in ./core*; do
+            [ -e "$c" ] && [ -r "$c" ] && [ "$c" -nt "$CORE_DIR" ] && { core="$c"; break; }
+        done
+    fi
+    if [ -n "$core" ] && command -v gdb >/dev/null 2>&1; then
         echo "=== crash: native backtrace ($core) ==="
-        # gdb reads the core; pull it in if missing, only when root.
-        if ! command -v gdb >/dev/null 2>&1 && [ "$(id -u)" = 0 ]; then
-            apt-get install -y -q gdb >/dev/null 2>&1 || true
-        fi
-        if command -v gdb >/dev/null 2>&1; then
-            # "bt" first: gdb auto-selects the faulting thread on a SIGSEGV
-            # core. "thread apply all bt" after gives the rest.
-            gdb -batch -nx \
-                -ex "bt" \
-                -ex "echo \n=== all threads ===\n" \
-                -ex "thread apply all bt" \
-                "$(command -v python3)" "$core" 2>&1 | head -400
-        else
-            echo "(gdb unavailable; core left at $core)"
-        fi
+        # "bt" first: gdb auto-selects the faulting thread on a SIGSEGV
+        # core. "thread apply all bt" after gives the rest.
+        gdb -batch -nx \
+            -ex "bt" \
+            -ex "echo \n=== all threads ===\n" \
+            -ex "thread apply all bt" \
+            "$(command -v python3)" "$core" 2>&1 | head -400
+    else
+        # No readable core (the common non-root case). The Python
+        # faulthandler traceback in linuxcnc.err already names the crash
+        # site; the native backtrace is only a best-effort extra.
+        echo "=== crash: no readable core dump; see the Python traceback in linuxcnc.err above ==="
     fi
     rm -rf "$CORE_DIR"
 }

tests/ui-smoke/_lib/drive.py

@@ -32,6 +32,13 @@
 # stability check catches that.
 STATE_STABILITY_S = 0.5
 STATE_RETRY_BUDGET = 6
+# Pause after homing before requesting AUTO. gmoccapy only enables AUTO
+# once it has processed the all-homed signal in its own event loop (and
+# re-asserts MANUAL itself on that signal). Requesting AUTO before then is
+# rejected: it bounces back to MANUAL with an "It is not possible to
+# change to Auto Mode" warning. ensure_mode would retry and win, but the
+# warning lingers on screen; this settle lets the GUI catch up first.
+POST_HOME_SETTLE_S = 2.0
 
 # linuxcnc launcher PID, written to linuxcnc.pid by the launcher and read
 # once at startup. The driver watches it so a GUI crash, which tears
@@ -297,6 +304,11 @@ def run_program(cmd, stat, ngc_path, expect_delta_mm, tol, run_timeout):
     if not home_all(cmd, stat, timeout=60.0):
         return False
 
+    # Let the GUI react to the all-homed transition before requesting AUTO,
+    # so it does not reject the mode change (see POST_HOME_SETTLE_S).
+    time.sleep(POST_HOME_SETTLE_S)
+    stat.poll()
+
     if not ensure_mode(cmd, stat, linuxcnc.MODE_AUTO, "MODE_AUTO"):
         return False
 

tests/ui-smoke/_lib/gmoccapy-prepare.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Sourced by the gmoccapy ui-smoke tests (smoke and quit) to run gmoccapy
+# against a writable copy of its sim config. Sets GMOCCAPY_INI to the
+# mirrored ini path; the caller then execs run-gui.sh or quit-launch.sh
+# with "$GMOCCAPY_INI". Must be sourced with LIB_DIR already set.
+#
+# gmoccapy writes its preferences file next to the config: with no
+# PREFERENCE_FILE_PATH in the ini, getiniinfo falls back to
+# <config-dir>/<MACHINE>.pref. CI mounts the workspace read-only for the
+# runtime user, so that write raises PermissionError partway through
+# __init__ (during _get_pref_data, before the MDIHistory widget's
+# _hal_init runs). gmoccapy pops an error dialog and limps on in a
+# half-initialised state: the interp-idle handler then hits a widget with
+# no .stat and throws a second dialog. Both vanish once the config dir is
+# writable. Mirror it to tmp, same fix qtdragon-prepare.sh uses.
+
+: "${LIB_DIR:?gmoccapy-prepare.sh must be sourced with LIB_DIR set}"
+
+SRC_DIR="$(cd "$LIB_DIR/../../../configs/sim/gmoccapy" && pwd)"
+
+WORK_DIR="$(mktemp -d -t ui-smoke-gmoccapy.XXXXXX)"
+trap 'rm -rf "$WORK_DIR"' EXIT
+cp -r "$SRC_DIR/." "$WORK_DIR/"
+
+# Seed the preference file (config dir + <MACHINE>.pref; MACHINE=gmoccapy)
+# so the first-run "Important change(s)" modal stays hidden. That dialog
+# runs a nested gtk loop, so under xvfb it never gets dismissed: it sits
+# on top of the UI in the confirm shot and, worse, swallows the SIGTERM
+# in the quit test (the loop keeps running after main_quit). A real user
+# ticks "Don't show this again" once; hide_startup_messsage replicates
+# that. The triple-s key matches gmoccapy's own (sic).
+cat >"$WORK_DIR/gmoccapy.pref" <<'PREF'
+[DEFAULT]
+hide_startup_messsage = 99
+PREF
+
+# Consumed by the sourcing test.sh, which execs the launcher with it.
+# shellcheck disable=SC2034
+GMOCCAPY_INI="$WORK_DIR/gmoccapy.ini"

tests/ui-smoke/_lib/launch-env.sh

@@ -29,3 +29,10 @@ export SDL_AUDIODRIVER=dummy
 # names the line; for a C/C++ crash (Qt, dbus, GL) it shows the Python
 # frame that called in. The native side is captured by crashdump.sh.
 export PYTHONFAULTHANDLER=1
+
+# Xvfb virtual screen for the launchers' xvfb-run. There is no window
+# manager under xvfb-run, so a GUI's maximize() is a no-op and it renders
+# at its natural size; a screen smaller than the window clips the grab
+# (the failure/confirm screenshot then misses panels). 1920x1080 fits
+# every sim GUI so the whole window is captured.
+export UI_SMOKE_XVFB_SCREEN="${UI_SMOKE_XVFB_SCREEN:-1920x1080x24}"