sp x86_64: separate lane selection from vector-register ownership

configure.ac

@@ -5111,7 +5111,13 @@ then
         AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_AESNI"
         if test "$KERNEL_MODE_DEFAULTS" = "yes"
         then
-                AM_CFLAGS="$AM_CFLAGS -DWC_C_DYNAMIC_FALLBACK"
+            # FIPS v7 and later pin one lane at build time; dev builds may
+            # still switch.  Older validated modules keep what they were
+            # validated with.  Tested numerically, and on the same condition
+            # settings.h uses, so a new v7.x or lean-* string cannot slip past.
+            AS_IF([test "${HAVE_FIPS_VERSION_MAJOR:-0}" -ge 7 && \
+                   test "x$ENABLED_FIPS_DEV" != "xyes"],[],
+                [AM_CFLAGS="$AM_CFLAGS -DWC_C_DYNAMIC_FALLBACK"])
         fi
         if test "$CC" != "icc"
         then
@@ -14223,6 +14229,14 @@ AM_CFLAGS="$AM_CFLAGS $EXTRA_CFLAGS"
 AM_CCASFLAGS="$AM_CCASFLAGS $EXTRA_CCASFLAGS"
 AM_LDFLAGS="$AM_LDFLAGS $EXTRA_LDFLAGS"
 
+# FIPS v7 and later never switch lanes at run time, however the define
+# arrives.  Same condition as the auto-add above and as settings.h.
+AS_IF([test "${HAVE_FIPS_VERSION_MAJOR:-0}" -ge 7 && \
+       test "x$ENABLED_FIPS_DEV" != "xyes"],
+    [AS_CASE([" $AM_CPPFLAGS $AM_CFLAGS $CPPFLAGS $CFLAGS "],
+        [*-DWC_C_DYNAMIC_FALLBACK*],
+        [AC_MSG_ERROR([WC_C_DYNAMIC_FALLBACK is not allowed with --enable-fips=$FIPS_VERSION; use --enable-fips=dev or --enable-fips=dev-no-post])])])
+
 CREATE_HEX_VERSION
 AC_SUBST([AM_CPPFLAGS])
 AC_SUBST([AM_CFLAGS])

linuxkm/lkcapi_aes_glue.c

@@ -2235,8 +2235,14 @@ static int ccmAesAead_rfc4309_loaded = 0;
     #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream).
 #endif
 
-#if defined(WOLFSSL_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK) && !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS)
-    #error LKCAPI registration of AES-XTS with AESNI requires WC_C_DYNAMIC_FALLBACK.
+/* AES-XTS asm needs a vector save on every call.  Without the fallback the
+ * whole context is pinned to C at setkey, so no save is ever taken. */
+#if defined(WOLFSSL_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK) && \
+    !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS)
+    #define WC_LINUXKM_XTS_NO_AESNI
+    #ifndef WC_FLAG_DONT_USE_VECTOR_OPS
+        #error AES-XTS without WC_C_DYNAMIC_FALLBACK needs WC_FLAG_DONT_USE_VECTOR_OPS.
+    #endif
 #endif
 
 struct km_AesXtsCtx {
@@ -2286,6 +2292,15 @@ static int km_AesXtsSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
     int err;
     struct km_AesXtsCtx * ctx = crypto_skcipher_ctx(tfm);
 
+#ifdef WC_LINUXKM_XTS_NO_AESNI
+    /* Set before the key schedule is built so the C schedule is the one made. */
+    ctx->aesXts->aes.use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
+    ctx->aesXts->tweak.use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
+#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
+    ctx->aesXts->aes_decrypt.use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
+#endif
+#endif
+
     err = wc_AesXtsSetKeyNoInit(ctx->aesXts, in_key, key_len,
                                 AES_ENCRYPTION_AND_DECRYPTION);
 
@@ -2296,12 +2311,6 @@ static int km_AesXtsSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
         return -EINVAL;
     }
 
-    /* It's possible to set ctx->aesXts->{tweak,aes,aes_decrypt}.use_aesni to
-     * WC_FLAG_DONT_USE_VECTOR_OPS here, for WC_LINUXKM_C_FALLBACK_IN_SHIMS in
-     * AES-XTS, but we can use the WC_C_DYNAMIC_FALLBACK mechanism
-     * unconditionally because there's no AES-XTS in Cert 4718.
-     */
-
     #ifdef WOLFKM_DEBUG_AES
     pr_info("info: exiting km_AesXtsSetKey: %d\n", key_len);
     #endif /* WOLFKM_DEBUG_AES */
@@ -2321,7 +2330,8 @@ static int km_AesXtsSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
     typeof(wc_AesXtsEncryptUpdate_fips) wc_AesXtsEncryptUpdate;
 #endif
 
-#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && !defined(WC_LINUXKM_SVR_NO_BATCHING)
+#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \
+    !defined(WC_LINUXKM_SVR_NO_BATCHING) && !defined(WC_LINUXKM_XTS_NO_AESNI)
     #ifndef WC_LINUXKM_XTS_SVR_BATCH
         #define WC_LINUXKM_XTS_SVR_BATCH (16 * 4096)
     #endif

linuxkm/lkcapi_glue.c

@@ -97,7 +97,13 @@
     #define LKCAPI_HAVE_ARCH_ACCEL
 #endif
 
-#if defined(LKCAPI_HAVE_ARCH_ACCEL) &&                   \
+/* v7 pins one lane per algorithm, so the shims keep no second key schedule.
+ * Tried and failed to make a refused save reach one: skcipher from hardirq is
+ * refused (crypto/skcipher.c:449), softirq always has SIMD (fpu/core.c:76). */
+#if defined(HAVE_FIPS) && FIPS_VERSION3_GE(7,0,0) && \
+    !defined(WOLFSSL_FIPS_DEV) && !defined(WOLFSSL_FIPS_DEV_NO_POST)
+    #undef WC_LINUXKM_C_FALLBACK_IN_SHIMS
+#elif defined(LKCAPI_HAVE_ARCH_ACCEL) &&                 \
     (!defined(WC_C_DYNAMIC_FALLBACK) ||                  \
      (defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0))) && \
     !defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS)
@@ -106,6 +112,16 @@
     #undef WC_LINUXKM_C_FALLBACK_IN_SHIMS
 #endif
 
+/* With one lane and no fallback, the save has to be available in every context
+ * the kernel may call us from.  The module's own XSAVE/FXSAVE area supplies it
+ * in hardirq and NMI as well (linuxkm/x86_vector_register_glue.c). */
+#if defined(HAVE_FIPS) && FIPS_VERSION3_GE(7,0,0) &&                     \
+    !defined(WOLFSSL_FIPS_DEV) && !defined(WOLFSSL_FIPS_DEV_NO_POST) &&  \
+    defined(CONFIG_X86) && defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \
+    !defined(WC_SVR_USE_NATIVE_REG_BUFS)
+    #error FIPS v7 LKCAPI needs WC_SVR_USE_NATIVE_REG_BUFS: one lane, no fallback.
+#endif
+
 #if defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && !defined(CAN_SAVE_VECTOR_REGISTERS)
     #error WC_LINUXKM_C_FALLBACK_IN_SHIMS is defined but CAN_SAVE_VECTOR_REGISTERS is missing.
 #endif

linuxkm/module_hooks.c

@@ -1262,61 +1262,13 @@ static int wolfssl_init(void)
 #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING
     {
         long long unsigned int svr_disallowed_count = wc_svr_disallowed_count_current();
-        long long unsigned int svr_disallowed_snapshot;
         if (svr_disallowed_count > 0) {
             pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after wc_RunAllCast_fips().\n", svr_disallowed_count);
             (void)libwolfssl_cleanup();
             return -ECANCELED;
         }
-
-    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
-    {
-        unsigned long stack_usage;
-        wc_linuxkm_stack_hwm_prepare(0xee);
-    #endif
-
-        ret = DISABLE_VECTOR_REGISTERS();
-        if (ret != 0) {
-            pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for wc_RunAllCast_fips() returned %d.\n", ret);
-            (void)libwolfssl_cleanup();
-            return -ECANCELED;
-        }
-
-        /* See the snapshot rationale in the wolfCrypt_IntegrityTest_fips()
-         * block above. */
-        svr_disallowed_snapshot = wc_svr_disallowed_count_current();
-
-        ret = wc_RunAllCast_fips();
-
-        REENABLE_VECTOR_REGISTERS();
-
-    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
-        stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee);
-        pr_info("STACK INFO: rel usage by wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS(): %lu\n", stack_usage);
-        /* shush up false stack HWM reading by kernel: */
-        wc_linuxkm_stack_hwm_prepare(0);
-    }
-    #endif
-
-        svr_disallowed_count = wc_svr_disallowed_count_current();
-        if (svr_disallowed_count <= svr_disallowed_snapshot) {
-            pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS() (snapshot %llu): inhibited-save instrumentation was not exercised.\n", svr_disallowed_count, svr_disallowed_snapshot);
-            (void)libwolfssl_cleanup();
-            return -ECANCELED;
-        }
-
-        if (ret != 0) {
-            pr_err("ERROR: wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS() returned %d.\n", ret);
-            (void)libwolfssl_cleanup();
-            return -ECANCELED;
-        }
-
-        ret = wolfCrypt_GetStatus_fips();
-        if (ret != 0) {
-            pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret));
-            (void)libwolfssl_cleanup();
-            return -ECANCELED;
-        }
+        /* The CASTs are not re-run with the registers disabled: CPUID picks
+         * one lane per algorithm, so a refused save is an error there. */
     }
 
 #endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */

linuxkm/x86_vector_register_glue.c

@@ -599,8 +599,8 @@ WARN_UNUSED_RESULT int wc_save_vector_registers_x86(enum wc_svr_flags flags)
      * Note that this is not actually an abnormal condition -- e.g. with
      * LINUXKM_DRBG_GET_RANDOM_BYTES, get_random_u32() and the like called from
      * hard IRQ handlers can land here, and we return success if
-     * WC_SVR_USE_NATIVE_REG_BUFS, else WC_ACCEL_INHIBIT_E for graceful fallback
-     * to C.
+     * WC_SVR_USE_NATIVE_REG_BUFS, else WC_ACCEL_INHIBIT_E.  Callers whose lane
+     * is pinned report that error rather than computing the answer in C.
      */
     if ((cur_preempt_count & (NMI_MASK | HARDIRQ_MASK)) != 0) {
 #ifdef WC_SVR_USE_NATIVE_REG_BUFS

tests/api/test_mldsa.c

@@ -30073,10 +30073,10 @@ int test_mldsa_encode_w1_large_values(void)
 
 #if defined(DEBUG_VECTOR_REGISTER_ACCESS) && \
     defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING)
-    /* Pin dispatch to the C path: under SVR2 fuzzing the two calls can
-     * otherwise take different (AVX2 vs C) implementations, which are only
-     * specified - and only equal - on the valid input domain. */
-    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(WC_NO_ERR_TRACE(SYSLIB_FAILED_E));
+    /* Let every save succeed for this test.  A refused save is an error that
+     * writes nothing, so two refused calls would compare equal while encoding
+     * nothing; pinning saves ON keeps one lane for both calls instead. */
+    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0);
 #endif
 
     /* ---- 6-bit encoding (mldsa_encode_w1_88 path) ---- */
@@ -30093,8 +30093,8 @@ int test_mldsa_encode_w1_large_values(void)
 
             XMEMSET(enc_a, 0, sizeof(enc_a));
             XMEMSET(enc_b, 0, sizeof(enc_b));
-            wc_mldsa_encode_w1_88(w1, enc_a);
-            wc_mldsa_encode_w1_88(w1, enc_b);
+            ExpectIntEQ(wc_mldsa_encode_w1_88(w1, enc_a), 0);
+            ExpectIntEQ(wc_mldsa_encode_w1_88(w1, enc_b), 0);
 
             /* Determinism: same input must produce same output */
             ExpectIntEQ(XMEMCMP(enc_a, enc_b, sizeof(enc_a)), 0);
@@ -30106,8 +30106,8 @@ int test_mldsa_encode_w1_large_values(void)
         }
         XMEMSET(enc_a, 0, sizeof(enc_a));
         XMEMSET(enc_b, 0, sizeof(enc_b));
-        wc_mldsa_encode_w1_88(w1, enc_a);
-        wc_mldsa_encode_w1_88(w1, enc_b);
+        ExpectIntEQ(wc_mldsa_encode_w1_88(w1, enc_a), 0);
+        ExpectIntEQ(wc_mldsa_encode_w1_88(w1, enc_b), 0);
         ExpectIntEQ(XMEMCMP(enc_a, enc_b, sizeof(enc_a)), 0);
 
         /* Ascending pattern: each element differs */
@@ -30116,8 +30116,8 @@ int test_mldsa_encode_w1_large_values(void)
         }
         XMEMSET(enc_a, 0, sizeof(enc_a));
         XMEMSET(enc_b, 0, sizeof(enc_b));
-        wc_mldsa_encode_w1_88(w1, enc_a);
-        wc_mldsa_encode_w1_88(w1, enc_b);
+        ExpectIntEQ(wc_mldsa_encode_w1_88(w1, enc_a), 0);
+        ExpectIntEQ(wc_mldsa_encode_w1_88(w1, enc_b), 0);
         ExpectIntEQ(XMEMCMP(enc_a, enc_b, sizeof(enc_a)), 0);
     }
 #endif /* !WOLFSSL_NO_ML_DSA_44 */
@@ -30136,8 +30136,8 @@ int test_mldsa_encode_w1_large_values(void)
 
             XMEMSET(enc_a, 0, sizeof(enc_a));
             XMEMSET(enc_b, 0, sizeof(enc_b));
-            wc_mldsa_encode_w1_32(w1, enc_a);
-            wc_mldsa_encode_w1_32(w1, enc_b);
+            ExpectIntEQ(wc_mldsa_encode_w1_32(w1, enc_a), 0);
+            ExpectIntEQ(wc_mldsa_encode_w1_32(w1, enc_b), 0);
 
             ExpectIntEQ(XMEMCMP(enc_a, enc_b, sizeof(enc_a)), 0);
         }
@@ -30148,8 +30148,8 @@ int test_mldsa_encode_w1_large_values(void)
         }
         XMEMSET(enc_a, 0, sizeof(enc_a));
         XMEMSET(enc_b, 0, sizeof(enc_b));
-        wc_mldsa_encode_w1_32(w1, enc_a);
-        wc_mldsa_encode_w1_32(w1, enc_b);
+        ExpectIntEQ(wc_mldsa_encode_w1_32(w1, enc_a), 0);
+        ExpectIntEQ(wc_mldsa_encode_w1_32(w1, enc_b), 0);
         ExpectIntEQ(XMEMCMP(enc_a, enc_b, sizeof(enc_a)), 0);
 
         /* Ascending pattern */
@@ -30158,8 +30158,8 @@ int test_mldsa_encode_w1_large_values(void)
         }
         XMEMSET(enc_a, 0, sizeof(enc_a));
         XMEMSET(enc_b, 0, sizeof(enc_b));
-        wc_mldsa_encode_w1_32(w1, enc_a);
-        wc_mldsa_encode_w1_32(w1, enc_b);
+        ExpectIntEQ(wc_mldsa_encode_w1_32(w1, enc_a), 0);
+        ExpectIntEQ(wc_mldsa_encode_w1_32(w1, enc_b), 0);
         ExpectIntEQ(XMEMCMP(enc_a, enc_b, sizeof(enc_a)), 0);
     }
 #endif /* !WOLFSSL_NO_ML_DSA_65 || !WOLFSSL_NO_ML_DSA_87 */