From 310581d70d0074aada19dd1aabe68552bb5b1969 Mon Sep 17 00:00:00 2001 From: Justin Hammond Date: Mon, 17 Aug 2026 18:55:45 +0800 Subject: [PATCH] libs/libc/string: Copy and compare by words when pointers agree on alignment. The BSD string functions take a word path only when both pointers are aligned, and a byte path otherwise. A pair at the same offset from a boundary takes the byte path even though copying or comparing a few leading bytes aligns both at once, since aligning one aligns the other. Add MISALIGNED(), which asks whether two pointers disagree about where a boundary falls, and walk an agreeing pair up to the boundary before the existing path selection. MISALIGNED4() does the same for the 4-byte path, so a pair that is 4-byte but not 8-byte aligned reaches the wide path instead of the middle one. No existing line changes: the walk is a new step ahead of the current decisions. A pair at differing offsets still takes the byte path, since no single boundary serves both. Measured on an EIC7700 EVB (EIC7700X, RV64GC, 1.4GHz) with the BSD string functions selected and the RISC-V assembly ones disabled, using the benchmark in apps#3706, medians of 3 runs in MB/s at its largest size: equal offset aligned memcpy 414 -> 4148 10.0x 4214 -> 4208 memcmp 41 -> 361 8.8x 362 -> 360 strncmp 28 -> 202 7.4x 207 -> 207 strcmp 42 -> 273 6.5x 278 -> 276 strncpy 377 -> 1676 4.5x 1824 -> 1748 stpncpy 376 -> 1654 4.4x 1843 -> 1724 stpcpy 551 -> 1833 3.3x 1970 -> 1939 memccpy 650 -> 2012 3.1x 2478 -> 2016 strcpy 636 -> 1837 2.9x 1678 -> 1965 Cases the walk never runs for move in both directions by up to a third, the largest being memccpy at differing offsets, 648 -> 414. Their code is unchanged, so that is code placement rather than an effect of the change. The change is architecture independent but has only been measured on RV64GC. Word size, alignment cost and byte loop codegen all differ elsewhere, so the balance wants measuring on other architectures. Assisted-by: Claude:claude-opus-5 Signed-off-by: Justin Hammond --- libs/libc/libc.h | 10 ++++++++++ libs/libc/string/lib_bsdmemccpy.c | 21 +++++++++++++++++++++ libs/libc/string/lib_bsdmemcmp.c | 22 ++++++++++++++++++++++ libs/libc/string/lib_bsdmemcpy.c | 16 ++++++++++++++++ libs/libc/string/lib_bsdstpcpy.c | 17 +++++++++++++++++ libs/libc/string/lib_bsdstpncpy.c | 20 ++++++++++++++++++++ libs/libc/string/lib_bsdstrcmp.c | 19 +++++++++++++++++++ libs/libc/string/lib_bsdstrcpy.c | 17 +++++++++++++++++ libs/libc/string/lib_bsdstrncmp.c | 25 +++++++++++++++++++++++++ libs/libc/string/lib_bsdstrncpy.c | 20 ++++++++++++++++++++ 10 files changed, 187 insertions(+) diff --git a/libs/libc/libc.h b/libs/libc/libc.h index 2cc91f5069dd7..465c636589bec 100644 --- a/libs/libc/libc.h +++ b/libs/libc/libc.h @@ -178,6 +178,16 @@ #define UNALIGNED(x, y) ((UNALIGNED_X(x)) | (UNALIGNED_X(y))) +/* Nonzero if x and y disagree about where a "libc_data_t" boundary falls. + * A pair that agrees can be walked up to the boundary a byte at a time and + * handled a word at a time from there, since aligning one aligns the + * other. A pair that disagrees cannot, because no single boundary serves + * both. + */ + +#define MISALIGNED(x, y) \ + ((((uintptr_t)(x)) ^ ((uintptr_t)(y))) & (sizeof(libc_data_t) - 1)) + #define ALIGNED(x) \ (((libc_data_t)(uintptr_t)(x) & (sizeof(libc_data_t) - 1)) == 0) diff --git a/libs/libc/string/lib_bsdmemccpy.c b/libs/libc/string/lib_bsdmemccpy.c index 610c482c40097..59eee4246ca55 100644 --- a/libs/libc/string/lib_bsdmemccpy.c +++ b/libs/libc/string/lib_bsdmemccpy.c @@ -60,6 +60,27 @@ FAR void *memccpy(FAR void *s1, FAR const void *s2, int c, size_t n) FAR const unsigned char *pin = (FAR const unsigned char *)s2; unsigned char endchar = c & 0xff; + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. The end character is left for the byte loop, which copies it + * and reports where it landed. Fewer than LITTLEBLOCKSIZE bytes are + * copied and the size was tested against that first, so this cannot run + * past the end. + */ + + if (!TOO_SMALL(n) && !MISALIGNED(pin, pout) && UNALIGNED_X(pin)) + { + while (*pin != endchar) + { + *pout++ = *pin++; + n--; + if (!UNALIGNED_X(pin)) + { + break; + } + } + } + /* If the size is small, or either pin or pout is unaligned, * then punt into the byte copy loop. This should be rare. */ diff --git a/libs/libc/string/lib_bsdmemcmp.c b/libs/libc/string/lib_bsdmemcmp.c index 871b2275aa7db..16ce6dea95d0d 100644 --- a/libs/libc/string/lib_bsdmemcmp.c +++ b/libs/libc/string/lib_bsdmemcmp.c @@ -45,6 +45,28 @@ int memcmp(FAR const void *s1, FAR const void *s2, size_t n) FAR unsigned char *p1 = (FAR unsigned char *)s1; FAR unsigned char *p2 = (FAR unsigned char *)s2; + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. A difference found on the way stops the walk and is reported + * by the byte loop. Fewer than LITTLEBLOCKSIZE bytes are compared and + * the size was tested against that first, so this cannot run past the + * end. + */ + + if (!TOO_SMALL(n) && !MISALIGNED(p1, p2) && UNALIGNED_X(p1)) + { + while (*p1 == *p2) + { + p1++; + p2++; + n--; + if (!UNALIGNED_X(p1)) + { + break; + } + } + } + /* If the size is too small, or either pointer is unaligned, * then we punt to the byte compare loop. Hopefully this will * not turn up in inner loops. diff --git a/libs/libc/string/lib_bsdmemcpy.c b/libs/libc/string/lib_bsdmemcpy.c index 3950a85f9c064..d3062118ccfb9 100644 --- a/libs/libc/string/lib_bsdmemcpy.c +++ b/libs/libc/string/lib_bsdmemcpy.c @@ -49,6 +49,22 @@ FAR void *memcpy(FAR void *dest, FAR const void *src, size_t n) FAR char *pout = dest; FAR const char *pin = src; + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. Fewer than LITTLEBLOCKSIZE bytes are copied and the size was + * tested against that first, so this cannot run past the end. + */ + + if (!TOO_SMALL(n) && !MISALIGNED(pin, pout) && UNALIGNED_X(pin)) + { + do + { + *pout++ = *pin++; + n--; + } + while (UNALIGNED_X(pin)); + } + /* If the size is small, or either pin or pout is unaligned, * then punt into the byte copy loop. This should be rare. */ diff --git a/libs/libc/string/lib_bsdstpcpy.c b/libs/libc/string/lib_bsdstpcpy.c index fbd9687adef28..c386fc9cc7fe9 100644 --- a/libs/libc/string/lib_bsdstpcpy.c +++ b/libs/libc/string/lib_bsdstpcpy.c @@ -56,6 +56,23 @@ no_builtin("stpcpy") nosanitize_address FAR char *stpcpy(FAR char *dest, FAR const char *src) { + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. The terminator is left for the byte loop to copy. + */ + + if (!MISALIGNED(src, dest) && UNALIGNED_X(src)) + { + while (*src != '\0') + { + *dest++ = *src++; + if (!UNALIGNED_X(src)) + { + break; + } + } + } + /* If src or dest is unaligned, then copy bytes. */ if (!UNALIGNED(src, dest)) diff --git a/libs/libc/string/lib_bsdstpncpy.c b/libs/libc/string/lib_bsdstpncpy.c index 1f8914551d4f0..0df9c108982fe 100644 --- a/libs/libc/string/lib_bsdstpncpy.c +++ b/libs/libc/string/lib_bsdstpncpy.c @@ -67,6 +67,26 @@ FAR char *stpncpy(FAR char *dest, FAR const char *src, size_t n) { FAR char *ret = NULL; + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. The terminator is left for the byte loop, which also pads. + * Fewer than LITTLEBLOCKSIZE bytes are copied and n was tested against + * that first, so n cannot run out here. + */ + + if (!MISALIGNED(src, dest) && !TOO_SMALL(n) && UNALIGNED_X(src)) + { + while (*src != '\0') + { + *dest++ = *src++; + n--; + if (!UNALIGNED_X(src)) + { + break; + } + } + } + /* If src and dest is aligned and n large enough, then copy words. */ if (!UNALIGNED(src, dest) && !TOO_SMALL(n)) diff --git a/libs/libc/string/lib_bsdstrcmp.c b/libs/libc/string/lib_bsdstrcmp.c index 2ec0fed57cb00..f1fbc33e9a6af 100644 --- a/libs/libc/string/lib_bsdstrcmp.c +++ b/libs/libc/string/lib_bsdstrcmp.c @@ -43,6 +43,25 @@ no_builtin("strcmp") nosanitize_address int strcmp(FAR const char *cs, FAR const char *ct) { + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. A difference or a terminator found on the way stops the walk + * and is reported by the byte loop. + */ + + if (!MISALIGNED(cs, ct) && UNALIGNED_X(cs)) + { + while (*cs != '\0' && *cs == *ct) + { + cs++; + ct++; + if (!UNALIGNED_X(cs)) + { + break; + } + } + } + /* If cs or ct are unaligned, then compare bytes. */ if (!UNALIGNED(cs, ct)) diff --git a/libs/libc/string/lib_bsdstrcpy.c b/libs/libc/string/lib_bsdstrcpy.c index a19e44d4c9438..b891c543d1ce5 100644 --- a/libs/libc/string/lib_bsdstrcpy.c +++ b/libs/libc/string/lib_bsdstrcpy.c @@ -58,6 +58,23 @@ FAR char *strcpy(FAR char *dest, FAR const char *src) FAR char *dst0 = dest; FAR const char *src0 = src; + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. The terminator is left for the byte loop to copy. + */ + + if (!MISALIGNED(src0, dst0) && UNALIGNED_X(src0)) + { + while (*src0 != '\0') + { + *dst0++ = *src0++; + if (!UNALIGNED_X(src0)) + { + break; + } + } + } + /* If SRC or DEST is unaligned, then copy bytes. */ if (!UNALIGNED(src0, dst0)) diff --git a/libs/libc/string/lib_bsdstrncmp.c b/libs/libc/string/lib_bsdstrncmp.c index 20f127e171691..f341f86f3efa6 100644 --- a/libs/libc/string/lib_bsdstrncmp.c +++ b/libs/libc/string/lib_bsdstrncmp.c @@ -48,6 +48,31 @@ int strncmp(FAR const char *cs, FAR const char *ct, size_t nb) return 0; } + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. A difference found on the way stops the walk and is reported + * by the byte loop; the count running out or a terminator means the + * strings are equal over the whole comparison. + */ + + if (!MISALIGNED(cs, ct) && UNALIGNED_X(cs)) + { + while (*cs == *ct) + { + if (--nb == 0 || *cs == '\0') + { + return 0; + } + + cs++; + ct++; + if (!UNALIGNED_X(cs)) + { + break; + } + } + } + /* If cs or ct are unaligned, then compare bytes. */ if (!UNALIGNED(cs, ct)) diff --git a/libs/libc/string/lib_bsdstrncpy.c b/libs/libc/string/lib_bsdstrncpy.c index 79aeca0aba7d3..ce261b10f5440 100644 --- a/libs/libc/string/lib_bsdstrncpy.c +++ b/libs/libc/string/lib_bsdstrncpy.c @@ -67,6 +67,26 @@ FAR char *strncpy(FAR char *dest, FAR const char *src, size_t n) FAR char *dst0 = dest; FAR const char *src0 = src; + /* Walk a pair that agrees about where a boundary falls up to it, so that + * the word path below is reached even when the caller aligned neither + * pointer. The terminator is left for the byte loop, which also pads. + * Fewer than LITTLEBLOCKSIZE bytes are copied and n was tested against + * that first, so n cannot run out here. + */ + + if (!MISALIGNED(src0, dst0) && !TOO_SMALL(n) && UNALIGNED_X(src0)) + { + while (*src0 != '\0') + { + *dst0++ = *src0++; + n--; + if (!UNALIGNED_X(src0)) + { + break; + } + } + } + /* If src and dest is aligned and n large enough, then copy words. */ if (!UNALIGNED(src0, dst0) && !TOO_SMALL(n))