imagecache: fall back to legacy mount on pre-6.5 kernels

internal/imagecache/README.md

@@ -118,11 +118,13 @@ staging the virtio-fs lower (micro-VM):
    legitimately list the same diffID twice — are collapsed to the topmost
    occurrence, which overlayfs otherwise rejects with `ELOOP`), `upperdir` /
    `workdir` are the bundle-local dirs, holding this actor's private writes.
-   The mount uses the new mount API (`fsopen` + one `fsconfig` `lowerdir+`
-   append per layer) rather than `mount(2)`, whose single-page option-string
-   cap the digest-derived layer paths would hit at ~34 layers. **Minimum
-   supported kernel: Linux 6.5** (`lowerdir+`); every current GKE channel
-   ships ≥ 6.6 (Stable: COS 121 LTS).
+   The mount prefers the new mount API (`fsopen` + one `fsconfig` `lowerdir+`
+   append per layer), which lifts `mount(2)`'s single-page option-string cap
+   that the digest-derived layer paths would hit at ~34 layers. `lowerdir+`
+   needs Linux 6.5+ (every current GKE channel ships >= 6.6, Stable: COS 121
+   LTS); on older kernels the first `lowerdir+` attempt fails immediately and
+   ateom falls back to a plain `mount(2)` call, which keeps the ~34-layer
+   cap.
 3. **ExtraDirs** are created through the mount (landing in the upper), again
    under `os.Root` confinement.
 4. **Implicit-parent metadata repair.** A layer tar routinely omits entries

internal/imagecache/bundle_linux.go

@@ -22,10 +22,12 @@ package imagecache
 import (
 	"errors"
 	"fmt"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 
 	"golang.org/x/sys/unix"
 )
@@ -97,21 +99,49 @@ func SetupBundleRootfs(bundlePath string) error {
 	return nil
 }
 
+// lowerdirPlusOption is the fsconfig parameter name mountOverlay uses for
+// each lowerdir. It is a var, not a literal, only so tests can force it to
+// an unrecognized name and exercise the pre-6.5-kernel fallback below
+// without needing an actual old kernel.
+var lowerdirPlusOption = "lowerdir+"
+
+var warnLegacyMountOnce sync.Once
+
 // mountOverlay attaches an overlay of lowers (top-most first) with the given
-// upper/work dirs at mountpoint, using the new mount API rather than
-// mount(2): appending lowerdirs one fsconfig(2) call at a time sidesteps
-// mount(2)'s single-page option-string cap, which digest-derived layer paths
-// (~114 bytes each) would hit at roughly 34 layers.
+// upper/work dirs at mountpoint. It prefers the new mount API, appending
+// lowerdirs one fsconfig(2) call at a time to sidestep mount(2)'s
+// single-page option-string cap, which digest-derived layer paths (~114
+// bytes each) would hit at roughly 34 layers.
 //
-// Minimum supported kernel: Linux 6.5, where overlayfs gained the
-// incremental "lowerdir+" option. Every current GKE channel is at or above
-// it (Stable runs COS 121 LTS on kernel 6.6; Regular and Rapid run COS
-// 125/129 on 6.12).
+// Kernels older than 6.5 do not recognize the incremental "lowerdir+"
+// parameter at all: the kernel queues "Unknown parameter 'lowerdir+'" on
+// the fs context log and the very first attempt to set it fails, before any
+// path is looked at or any superblock created. That specific diagnostic is
+// used as the fall-back signal rather than a separate startup probe. A bad
+// lowerdir path fails the same call with a different error (e.g. ENOENT,
+// since 6.5 resolves the path while parsing the parameter) and is reported
+// as-is instead of being misread as a kernel-version problem.
 func mountOverlay(mountpoint string, lowers []string, upper, work string) error {
+	if len(lowers) == 0 {
+		return fmt.Errorf("mountOverlay: no lower layers")
+	}
+
 	fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC)
 	if err != nil {
 		return fmt.Errorf("while opening overlay fs context: %w", err)
 	}
+
+	if err := unix.FsconfigSetString(fsfd, lowerdirPlusOption, lowers[0]); err != nil {
+		log := fsContextLog(fsfd)
+		unix.Close(fsfd)
+		if strings.Contains(log, "Unknown parameter") {
+			warnLegacyMountOnce.Do(func() {
+				slog.Warn("kernel lacks overlayfs lowerdir+ (need >= 6.5); using legacy mount, images beyond ~34 layers will fail")
+			})
+			return mountOverlayLegacy(mountpoint, lowers, upper, work)
+		}
+		return fmt.Errorf("while setting overlay %s=%q: %w%s", lowerdirPlusOption, lowers[0], err, log)
+	}
 	defer unix.Close(fsfd)
 
 	set := func(key, val string) error {
@@ -120,8 +150,8 @@ func mountOverlay(mountpoint string, lowers []string, upper, work string) error
 		}
 		return nil
 	}
-	for _, lower := range lowers {
-		if err := set("lowerdir+", lower); err != nil {
+	for _, lower := range lowers[1:] {
+		if err := set(lowerdirPlusOption, lower); err != nil {
 			return err
 		}
 	}
@@ -146,10 +176,44 @@ func mountOverlay(mountpoint string, lowers []string, upper, work string) error
 	return nil
 }
 
+// legacyOptionsPageCap is mount(2)'s per-call option-string limit: the
+// kernel copies the "data" argument through a single page.
+const legacyOptionsPageCap = 4096
+
+// overlayMountOptions builds the mount(2) overlayfs option string for
+// kernels without "lowerdir+" support. lowers must already be top-first and
+// deduplicated (see overlayLowerDirs).
+func overlayMountOptions(lowers []string, upper, work string) (string, error) {
+	for _, p := range append([]string{upper, work}, lowers...) {
+		if strings.ContainsAny(p, ":,") {
+			return "", fmt.Errorf("overlay path %q contains mount option separators", p)
+		}
+	}
+	opts := "lowerdir=" + strings.Join(lowers, ":") + ",upperdir=" + upper + ",workdir=" + work
+	if len(opts) >= legacyOptionsPageCap {
+		return "", fmt.Errorf("overlay options for %d layers are %d bytes, at or over mount(2)'s %d-byte page cap; kernel lacks overlayfs lowerdir+, upgrade to Linux >= 6.5 to support longer layer chains", len(lowers), len(opts), legacyOptionsPageCap)
+	}
+	return opts, nil
+}
+
+// mountOverlayLegacy mounts the overlay via a plain mount(2) call, for
+// kernels older than 6.5 that do not support the incremental "lowerdir+"
+// fsconfig option.
+func mountOverlayLegacy(mountpoint string, lowers []string, upper, work string) error {
+	opts, err := overlayMountOptions(lowers, upper, work)
+	if err != nil {
+		return err
+	}
+	if err := unix.Mount("overlay", mountpoint, "overlay", 0, opts); err != nil {
+		return fmt.Errorf("while mounting overlay rootfs at %q (%s): %w", mountpoint, opts, err)
+	}
+	return nil
+}
+
 // fsContextLog drains the human-readable message log the kernel queues on an
 // fs context fd (one "e/w/i "-prefixed message per read, ENODATA when empty)
-// and renders it for appending to an error. mount(2) had no equivalent — a
-// failed overlay mount was a bare errno; here the kernel says which option
+// and renders it for appending to an error. mount(2) had no equivalent,
+// a failed overlay mount was a bare errno; here the kernel says which option
 // it rejected and why.
 func fsContextLog(fsfd int) string {
 	var msgs []string

internal/imagecache/bundle_linux_test.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"golang.org/x/sys/unix"
@@ -289,3 +290,99 @@ func TestSetupBundleRootfs_ManyLayers(t *testing.T) {
 		t.Fatalf("UnmountAllUnder: %v", err)
 	}
 }
+
+// Legacy mount(2) path, exercised directly so it is covered even on kernels
+// where mountOverlay's lowerdir+ attempt succeeds and never falls back to
+// it. Needs CAP_SYS_ADMIN.
+func TestMountOverlayLegacy(t *testing.T) {
+	roottest.Require(t, "mount/unmount")
+	layer := t.TempDir()
+	writeLayer(t, layer, map[string]string{"from-layer.txt": "hello"}, nil)
+
+	bundle := t.TempDir()
+	rootfs := filepath.Join(bundle, "rootfs")
+	upper := filepath.Join(bundle, "upper")
+	work := filepath.Join(bundle, "work")
+	for _, d := range []string{rootfs, upper, work} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			t.Fatalf("mkdir %s: %v", d, err)
+		}
+	}
+	if err := FinalizeLayer(layer); err != nil {
+		t.Fatalf("FinalizeLayer: %v", err)
+	}
+
+	if err := mountOverlayLegacy(rootfs, overlayLowerDirs([]string{layer}), upper, work); err != nil {
+		t.Fatalf("mountOverlayLegacy: %v", err)
+	}
+	t.Cleanup(func() { _ = UnmountAllUnder(bundle) })
+
+	if got, err := os.ReadFile(filepath.Join(rootfs, "from-layer.txt")); err != nil || string(got) != "hello" {
+		t.Errorf("layer content not visible through overlay: %q (%v)", got, err)
+	}
+}
+
+// Forces mountOverlay's first fsconfig call to fail with an unrecognized
+// parameter name, the same failure a pre-6.5 kernel gives for real, and
+// checks it falls back to a working legacy mount rather than erroring out.
+// Needs CAP_SYS_ADMIN.
+func TestMountOverlay_FallsBackOnUnsupportedLowerdirPlus(t *testing.T) {
+	roottest.Require(t, "mount/unmount")
+	orig := lowerdirPlusOption
+	lowerdirPlusOption = "not-a-real-overlay-option+"
+	t.Cleanup(func() { lowerdirPlusOption = orig })
+
+	layer := t.TempDir()
+	writeLayer(t, layer, map[string]string{"from-layer.txt": "hello"}, nil)
+	if err := FinalizeLayer(layer); err != nil {
+		t.Fatalf("FinalizeLayer: %v", err)
+	}
+
+	bundle := t.TempDir()
+	rootfs := filepath.Join(bundle, "rootfs")
+	upper := filepath.Join(bundle, "upper")
+	work := filepath.Join(bundle, "work")
+	for _, d := range []string{rootfs, upper, work} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			t.Fatalf("mkdir %s: %v", d, err)
+		}
+	}
+
+	if err := mountOverlay(rootfs, overlayLowerDirs([]string{layer}), upper, work); err != nil {
+		t.Fatalf("mountOverlay: %v", err)
+	}
+	t.Cleanup(func() { _ = UnmountAllUnder(bundle) })
+
+	if got, err := os.ReadFile(filepath.Join(rootfs, "from-layer.txt")); err != nil || string(got) != "hello" {
+		t.Errorf("layer content not visible through overlay after fallback: %q (%v)", got, err)
+	}
+}
+
+func TestOverlayMountOptions(t *testing.T) {
+	if _, err := overlayMountOptions([]string{"a:b"}, "/upper", "/work"); err == nil {
+		t.Error("lower path with separator: want error, got nil")
+	}
+	if _, err := overlayMountOptions([]string{"/a"}, "up,per", "/work"); err == nil {
+		t.Error("upper path with separator: want error, got nil")
+	}
+
+	opts, err := overlayMountOptions([]string{"/a", "/b"}, "/upper", "/work")
+	if err != nil {
+		t.Fatalf("overlayMountOptions: %v", err)
+	}
+	if want := "lowerdir=/a:/b,upperdir=/upper,workdir=/work"; opts != want {
+		t.Errorf("opts = %q, want %q", opts, want)
+	}
+
+	long := make([]string, 64)
+	for i := range long {
+		long[i] = filepath.Join("/sha256", fmt.Sprintf("%064d", i), "fs")
+	}
+	_, err = overlayMountOptions(long, "/upper", "/work")
+	if err == nil {
+		t.Fatal("over-page-cap options: want error, got nil")
+	}
+	if !strings.Contains(err.Error(), "lowerdir+") {
+		t.Errorf("over-page-cap error = %q, want a mention of lowerdir+ / kernel upgrade", err.Error())
+	}
+}