From 6f8100b3ec3916be165b501b3981f65dc3e36442 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sat, 26 Sep 2026 09:55:57 -0400 Subject: [PATCH 1/2] Add UTF-16 toWellFormed/isWellFormed with AVX-512, AVX2, SSE and NEON kernels. UTF16.ToWellFormed replaces lone surrogates by U+FFFD (JavaScript's toWellFormed); UTF16.IsWellFormed and GetPointerToFirstInvalidChar validate. The string overload returns well-formed input as is. The kernels are based on simdutf's utf16fix (Clausecker and Lemire, Fixing ill-formed UTF-16 strings with SIMD instructions, Software: Practice and Experience, 2026). Rather than loading a lookback block, they carry the high-surrogate bit across blocks, skip surrogate-free blocks with a min-based pre-check, align output stores, and handle the end of the input with masked loads (AVX-512) or overlapping windows. Other systems fall back on IndexOfAnyInRange. Tests compare every kernel, in place and out of place, against a rune-based reference. Benchmarks compare against IndexOfAnyInRange, EnumerateRunes and Encoding.Unicode. --- README.md | 104 +- benchmark/UTF16_benchmark.cs | 262 +++++ docs/articles/getting-started.md | 33 + src/UTF16.cs | 1596 ++++++++++++++++++++++++++++++ test/UTF16WellFormedTests.cs | 399 ++++++++ 5 files changed, 2393 insertions(+), 1 deletion(-) create mode 100644 benchmark/UTF16_benchmark.cs create mode 100644 src/UTF16.cs create mode 100644 test/UTF16WellFormedTests.cs diff --git a/README.md b/README.md index ec86c42..bec8326 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # SimdUnicode [![.NET](https://github.com/simdutf/SimdUnicode/actions/workflows/dotnet.yml/badge.svg)](https://github.com/simdutf/SimdUnicode/actions/workflows/dotnet.yml) -This is a fast C# library to validate UTF-8 strings. +This is a fast C# library to validate UTF-8 strings and to make UTF-16 strings well formed. ## Motivation @@ -230,6 +230,108 @@ faster than the standard library. | Russian-Lipsum | 2.3 | 0.7 | 3.3 x | +## Well-formed UTF-16 strings + +.NET strings may contain lone surrogates. `SimdUnicode.UTF16.ToWellFormed` replaces each +lone surrogate by the replacement character U+FFFD, like JavaScript's +`String.prototype.toWellFormed()`, and `SimdUnicode.UTF16.IsWellFormed` checks whether +there is any (like `isWellFormed()`). + +```cs +string s = UTF16.ToWellFormed("ab\uD800cd"); // "ab\uFFFDcd" +// Well-formed strings are returned as is, without allocation. +bool ok = UTF16.IsWellFormed(span); +// Buffer to buffer, or in place (same pointer for input and output). +UTF16.ToWellFormed(char* input, int length, char* output); +// Returns a pointer to the first lone surrogate, or input + length. +char* p = UTF16.GetPointerToFirstInvalidChar(char* input, int length); +``` + +We provide AVX-512, AVX2, SSE (SSE4.1) and ARM64 (NEON) kernels, selected at runtime, based on simdutf's +algorithms described in the following article: + +- Robert Clausecker, Daniel Lemire, Fixing ill-formed UTF-16 strings with SIMD instructions, Software: Practice and Experience, 2026 + +Other systems fall back on the runtime's vectorized `IndexOfAnyInRange`. + +We compare against the best approach we know that uses only public .NET APIs: return the input +when it is well formed, otherwise copy it and fix the errors found with the vectorized +`IndexOfAnyInRange('\uD800', '\uDFFF')`. It is fast on text without surrogates, +but it stops at every surrogate pair, so it is slow on text such as emojis. The idiomatic +`string.Concat(s.EnumerateRunes())` runs at 0.2 GB/s to 0.4 GB/s, and a round trip through +`Encoding.Unicode` at 1.3 GB/s to 7 GB/s. + +To reproduce: `dotnet run -c Release --filter "*UTF16WellFormed*"` in the `benchmark` directory. +All results are in GB/s of UTF-16 input, for well-formed inputs. "validate" is `IsWellFormed` +(and `ToWellFormed(string)`, which returns well-formed strings as is); "buffer" writes the +output to a separate buffer; "short strings" cuts the input into strings of 1 to 64 code units +and calls `ToWellFormed(string)` on each. + +Intel Xeon Gold 6548N (.NET 10), AVX-512: + +| data set | validate: SimdUnicode | validate: IndexOfAnyInRange | buffer: SimdUnicode | buffer: copy + IndexOfAnyInRange | plain copy | short strings: SimdUnicode | short strings: IndexOfAnyInRange | +|:--|--:|--:|--:|--:|--:|--:|--:| +| Twitter.json | 69 | 31 | 26 | 15 | 28 | 9.2 | 7.1 | +| Arabic-Lipsum | 68 | 33 | 43 | 20 | 46 | 17 | 17 | +| Chinese-Lipsum | 115 | 39 | 43 | 20 | 46 | 17 | 16 | +| Emoji-Lipsum | 53 | 0.39 | 40 | 0.39 | 44 | 3.1 | 0.43 | +| Hindi-Lipsum | 68 | 33 | 43 | 21 | 45 | 17 | 17 | +| Japanese-Lipsum | 115 | 39 | 43 | 20 | 46 | 17 | 16 | +| Korean-Lipsum | 68 | 33 | 42 | 20 | 46 | 17 | 17 | +| Latin-Lipsum | 69 | 33 | 43 | 20 | 46 | 17 | 17 | +| Russian-Lipsum | 69 | 33 | 43 | 19 | 46 | 17 | 17 | + +Same machine with AVX-512 disabled (`DOTNET_EnableAVX512=0`), AVX2 kernel (Haswell level): + +| data set | validate: SimdUnicode | validate: IndexOfAnyInRange | buffer: SimdUnicode | buffer: copy + IndexOfAnyInRange | plain copy | short strings: SimdUnicode | short strings: IndexOfAnyInRange | +|:--|--:|--:|--:|--:|--:|--:|--:| +| Twitter.json | 58 | 41 | 27 | 17 | 28 | 6.5 | 6.4 | +| Arabic-Lipsum | 58 | 42 | 43 | 23 | 47 | 17 | 17 | +| Chinese-Lipsum | 80 | 50 | 43 | 23 | 46 | 17 | 17 | +| Emoji-Lipsum | 35 | 0.50 | 28 | 0.49 | 45 | 3.4 | 0.53 | +| Hindi-Lipsum | 58 | 42 | 43 | 24 | 45 | 17 | 17 | +| Japanese-Lipsum | 77 | 50 | 43 | 24 | 46 | 17 | 17 | +| Korean-Lipsum | 58 | 42 | 43 | 23 | 46 | 17 | 17 | +| Latin-Lipsum | 58 | 42 | 43 | 24 | 46 | 17 | 17 | +| Russian-Lipsum | 58 | 42 | 43 | 22 | 47 | 17 | 17 | + +Same machine with AVX disabled (`DOTNET_EnableAVX=0`), SSE kernel (Westmere level): + +| data set | validate: SimdUnicode | validate: IndexOfAnyInRange | buffer: SimdUnicode | buffer: copy + IndexOfAnyInRange | plain copy | short strings: SimdUnicode | short strings: IndexOfAnyInRange | +|:--|--:|--:|--:|--:|--:|--:|--:| +| Twitter.json | 50 | 28 | 26 | 15 | 26 | 6.1 | 6.1 | +| Arabic-Lipsum | 51 | 28 | 45 | 18 | 46 | 15 | 13 | +| Chinese-Lipsum | 63 | 28 | 43 | 18 | 46 | 15 | 13 | +| Emoji-Lipsum | 26 | 0.56 | 22 | 0.56 | 45 | 3.3 | 0.55 | +| Hindi-Lipsum | 50 | 28 | 43 | 17 | 45 | 15 | 13 | +| Japanese-Lipsum | 64 | 28 | 44 | 18 | 46 | 15 | 13 | +| Korean-Lipsum | 50 | 28 | 43 | 18 | 46 | 16 | 14 | +| Latin-Lipsum | 51 | 28 | 45 | 18 | 47 | 15 | 13 | +| Russian-Lipsum | 51 | 28 | 45 | 18 | 47 | 15 | 13 | + +Apple M4 Max (.NET 10), NEON: + +| data set | validate: SimdUnicode | validate: IndexOfAnyInRange | buffer: SimdUnicode | buffer: copy + IndexOfAnyInRange | plain copy | short strings: SimdUnicode | short strings: IndexOfAnyInRange | +|:--|--:|--:|--:|--:|--:|--:|--:| +| Twitter.json | 106 | 62 | 71 | 35 | 84 | 23 | 21 | +| Arabic-Lipsum | 135 | 64 | 64 | 31 | 109 | 22 | 22 | +| Chinese-Lipsum | 134 | 64 | 66 | 40 | 75 | 22 | 22 | +| Emoji-Lipsum | 52 | 2.0 | 52 | 1.6 | 63 | 5.0 | 0.80 | +| Hindi-Lipsum | 135 | 64 | 73 | 38 | 85 | 23 | 23 | +| Japanese-Lipsum | 135 | 64 | 68 | 40 | 82 | 23 | 23 | +| Korean-Lipsum | 135 | 65 | 52 | 40 | 76 | 22 | 23 | +| Latin-Lipsum | 102 | 63 | 80 | 32 | 82 | 23 | 23 | +| Russian-Lipsum | 135 | 65 | 68 | 27 | 106 | 22 | 23 | + +- Validation is 1.4 to 3 times faster than `IndexOfAnyInRange`, and 25 to 140 times faster + on emoji-heavy text. +- Buffer to buffer, on x64 we are within 10% of the speed of a plain copy (except on emojis), + and 1.3 to 2.5 times faster than copying and then scanning with `IndexOfAnyInRange` on all systems + (30 to 110 times faster on emojis). On the M4 Max, we do not reach the speed of a plain copy. +- On short strings, we are on par (within 5%) or faster, and 6 to 8 times faster on emojis. +- When the string must be fixed (100 lone surrogates per million code units), both approaches + are dominated by the allocation of the new string, except on emojis. + ## Building the library ``` diff --git a/benchmark/UTF16_benchmark.cs b/benchmark/UTF16_benchmark.cs new file mode 100644 index 0000000..4e695f4 --- /dev/null +++ b/benchmark/UTF16_benchmark.cs @@ -0,0 +1,262 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using SimdUnicode; + +namespace SimdUnicodeBenchmarks +{ + // Speed in GB/s of UTF-16 input (2 bytes per code unit). + public class Utf16Speed : IColumn + { + public string GetValue(Summary summary, BenchmarkCase benchmarkCase) + { + if (summary is null || benchmarkCase is null || benchmarkCase.Parameters is null) + { + return "N/A"; + } + var ourReport = summary.Reports.First(x => x.BenchmarkCase.Equals(benchmarkCase)); + if (ourReport is null || ourReport.ResultStatistics is null) + { + return "N/A"; + } + var fileName = (string)benchmarkCase.Parameters["FileName"]; + var errors = (int)benchmarkCase.Parameters["ErrorsPerMillion"]; + long bytes = 2L * UTF16WellFormedBenchmark.Load(fileName, errors).Length; + return $"{(bytes / ourReport.ResultStatistics.Mean):#####.00}"; + } + + public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase); + public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; + public bool IsAvailable(Summary summary) => true; + + public string Id { get; } = nameof(Utf16Speed); + public string ColumnName { get; } = "Speed (GB/s)"; + public bool AlwaysShow { get; } = true; + public ColumnCategory Category { get; } = ColumnCategory.Custom; + public int PriorityInCategory { get; } + public bool IsNumeric { get; } + public UnitType UnitType { get; } = UnitType.Dimensionless; + public string Legend { get; } = "The speed in gigabytes per second (UTF-16 input)"; + } + + // Replacing lone surrogates by U+FFFD (JavaScript's toWellFormed). + // Run with: dotnet run -c Release --filter "*UTF16WellFormed*" + [SimpleJob(launchCount: 1, warmupCount: 3, iterationCount: 5)] + [Config(typeof(Config))] + public class UTF16WellFormedBenchmark + { +#pragma warning disable CA1812 + private sealed class Config : ManualConfig + { + public Config() + { + AddColumn(new Utf16Speed()); + } + } + + [Params(@"data/twitter.json", + @"data/Arabic-Lipsum.utf8.txt", + @"data/Chinese-Lipsum.utf8.txt", + @"data/Emoji-Lipsum.utf8.txt", + @"data/Hebrew-Lipsum.utf8.txt", + @"data/Hindi-Lipsum.utf8.txt", + @"data/Japanese-Lipsum.utf8.txt", + @"data/Korean-Lipsum.utf8.txt", + @"data/Latin-Lipsum.utf8.txt", + @"data/Russian-Lipsum.utf8.txt")] +#pragma warning disable CA1051 + public string FileName = ""; + + // 0: well-formed input (the common case). Otherwise, about this many code units + // per million are overwritten with lone surrogates. + [Params(0, 100)] + public int ErrorsPerMillion; + + private string input = ""; + private char[] output = Array.Empty(); + // The same content, cut into short strings of 1 to 64 code units. + private string[] shortStrings = Array.Empty(); + + public static string Load(string fileName, int errorsPerMillion) + { + char[] chars = Encoding.UTF8.GetString(File.ReadAllBytes(fileName)).ToCharArray(); + if (errorsPerMillion > 0) + { + var rand = new Random(1234); + long count = (long)chars.Length * errorsPerMillion / 1_000_000 + 1; + for (long k = 0; k < count; k++) + { + int pos = rand.Next(chars.Length); + chars[pos] = (char)(rand.Next(2) == 0 ? 0xD800 + rand.Next(0x400) : 0xDC00 + rand.Next(0x400)); + } + } + return new string(chars); + } + + [GlobalSetup] + public void Setup() + { + input = Load(FileName, ErrorsPerMillion); + output = new char[input.Length]; + var rand = new Random(4321); + var pieces = new System.Collections.Generic.List(); + for (int i = 0; i < input.Length;) + { + int len = Math.Min(rand.Next(1, 65), input.Length - i); + pieces.Add(input.Substring(i, len)); + i += len; + } + shortStrings = pieces.ToArray(); + string expected = RunesToWellFormed(input); + if (UTF16.ToWellFormed(input) != expected || IndexOfAnyToWellFormed(input) != expected) + { + throw new InvalidOperationException("Mismatch between implementations."); + } + } + + // The idiomatic version: decode rune by rune. + private static string RunesToWellFormed(string s) => string.Concat(s.EnumerateRunes()); + + // The best version we know that uses only public .NET APIs: return the input + // when possible and skip non-surrogates with the vectorized IndexOfAnyInRange. + private static string IndexOfAnyToWellFormed(string s) + { + int first = NextError(s, 0); + if (first < 0) + { + return s; + } + return string.Create(s.Length, (s, first), static (dst, state) => + { + var (src, i) = state; + src.AsSpan().CopyTo(dst); + do + { + dst[i] = '\uFFFD'; + i = NextError(src, i + 1); + } while (i >= 0); + }); + } + + private static int NextError(ReadOnlySpan s, int start) + { + int i = start; + while (true) + { + int k = s.Slice(i).IndexOfAnyInRange('\uD800', '\uDFFF'); + if (k < 0) + { + return -1; + } + i += k; + if (char.IsHighSurrogate(s[i]) && i + 1 < s.Length && char.IsLowSurrogate(s[i + 1])) + { + i += 2; + } + else + { + return i; + } + } + } + + [Benchmark] + [BenchmarkCategory("string")] + public string StringRunes() => RunesToWellFormed(input); + + [Benchmark] + [BenchmarkCategory("string")] + public string StringIndexOfAnyInRange() => IndexOfAnyToWellFormed(input); + + // Encoding.Unicode replaces lone surrogates by U+FFFD when encoding. + [Benchmark] + [BenchmarkCategory("string")] + public string StringEncodingRoundTrip() => Encoding.Unicode.GetString(Encoding.Unicode.GetBytes(input)); + + [Benchmark] + [BenchmarkCategory("string")] + public string StringSimdUnicode() => UTF16.ToWellFormed(input); + + // Short strings (1 to 64 code units), one call per string. + [Benchmark] + [BenchmarkCategory("short")] + public int ShortIndexOfAnyInRange() + { + int count = 0; + foreach (string s in shortStrings) + { + count += IndexOfAnyToWellFormed(s).Length; + } + return count; + } + + [Benchmark] + [BenchmarkCategory("short")] + public int ShortSimdUnicode() + { + int count = 0; + foreach (string s in shortStrings) + { + count += UTF16.ToWellFormed(s).Length; + } + return count; + } + + // Buffer to buffer: always writes the whole output. + // A plain copy, as a reference for the best we can hope for. + [Benchmark] + [BenchmarkCategory("buffer")] + public void BufferCopyOnly() => input.AsSpan().CopyTo(output); + + [Benchmark] + [BenchmarkCategory("buffer")] + public unsafe void BufferScalar() + { + fixed (char* pIn = input) + fixed (char* pOut = output) + { + UTF16.ToWellFormedScalar(pIn, input.Length, pOut); + } + } + + [Benchmark] + [BenchmarkCategory("buffer")] + public void BufferCopyThenIndexOfAnyInRange() + { + input.AsSpan().CopyTo(output); + Span dst = output; + int i = NextError(dst, 0); + while (i >= 0) + { + dst[i] = '\uFFFD'; + i = NextError(dst, i + 1); + } + } + + [Benchmark] + [BenchmarkCategory("buffer")] + public unsafe void BufferSimdUnicode() + { + fixed (char* pIn = input) + fixed (char* pOut = output) + { + UTF16.ToWellFormed(pIn, input.Length, pOut); + } + } + + // Validation only (isWellFormed). + [Benchmark] + [BenchmarkCategory("validate")] + public bool ValidateIndexOfAnyInRange() => NextError(input, 0) < 0; + + [Benchmark] + [BenchmarkCategory("validate")] + public bool ValidateSimdUnicode() => UTF16.IsWellFormed(input); + } +} diff --git a/docs/articles/getting-started.md b/docs/articles/getting-started.md index 5782fc6..3a5e150 100644 --- a/docs/articles/getting-started.md +++ b/docs/articles/getting-started.md @@ -104,3 +104,36 @@ unsafe ``` Continue to [How it works](how-it-works.md) or jump to the [API reference](xref:SimdUnicode.UTF8). + +## Making UTF-16 strings well formed + +.NET strings are sequences of UTF-16 code units and may contain *lone surrogates* +(a high surrogate not followed by a low surrogate, or a low surrogate not preceded by a +high surrogate). [`UTF16.ToWellFormed`](xref:SimdUnicode.UTF16) replaces each lone surrogate +by the replacement character U+FFFD, like JavaScript's `String.prototype.toWellFormed()`. + +```csharp +using SimdUnicode; + +string fixedString = UTF16.ToWellFormed("ab\uD800cd"); // "ab\uFFFDcd" + +// Already well-formed strings are returned as is, without allocation. +string s = "héllo 😀"; +bool same = ReferenceEquals(UTF16.ToWellFormed(s), s); // true + +// Validation only (like JavaScript's isWellFormed()). +bool ok = UTF16.IsWellFormed(s.AsSpan()); + +// Buffer to buffer, or in place when source and destination are the same. +char[] buffer = "x\uDC00y".ToCharArray(); +UTF16.ToWellFormed(buffer, buffer); +``` + +There is also a pointer API: `UTF16.ToWellFormed(char* input, int length, char* output)` and +`UTF16.GetPointerToFirstInvalidChar(char* input, int length)`, which returns a pointer to the +first lone surrogate, or to the end of the buffer when the input is well formed. + +On AVX-512, AVX2, SSE4.1 and ARM64 (NEON) hardware, we use algorithms based on simdutf's, described in +Robert Clausecker, Daniel Lemire, *Fixing ill-formed UTF-16 strings with SIMD instructions*, +Software: Practice and Experience, 2026. Other systems fall back on the runtime's +vectorized `IndexOfAnyInRange`. diff --git a/src/UTF16.cs b/src/UTF16.cs new file mode 100644 index 0000000..24ed2eb --- /dev/null +++ b/src/UTF16.cs @@ -0,0 +1,1596 @@ +using System; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Numerics; + +// Making UTF-16 strings well formed: every lone surrogate is replaced by U+FFFD, +// which matches JavaScript's String.prototype.toWellFormed(). +// +// The AVX-512, AVX2, SSE and ARM64 kernels are based on simdutf's utf16fix functions, described in +// +// Robert Clausecker, Daniel Lemire, Fixing ill-formed UTF-16 strings with SIMD +// instructions, Software: Practice and Experience, 2026. +// +// See https://github.com/simdutf/simdutf/blob/master/src/icelake/icelake_utf16fix.cpp +// Unlike simdutf, we carry the high-surrogate bitmask across blocks instead of loading +// a lookback block, and we skip blocks of 128 code units without surrogates quickly. +namespace SimdUnicode +{ + public static class UTF16 + { + /// The replacement character U+FFFD. + public const char ReplacementCharacter = '\uFFFD'; + + /// + /// Given an input buffer of UTF-16 code units, + /// returns a pointer to the first lone (unpaired) surrogate. + /// + /// + /// Returns a pointer to the end of if the buffer is well-formed. + /// + public unsafe static char* GetPointerToFirstInvalidChar(char* pInputBuffer, int inputLength) + { + if (AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian) + { + return GetPointerToFirstInvalidCharArm64(pInputBuffer, inputLength); + } + if (Vector512.IsHardwareAccelerated && Avx512BW.IsSupported) + { + return GetPointerToFirstInvalidCharAvx512(pInputBuffer, inputLength); + } + if (Avx2.IsSupported) + { + return GetPointerToFirstInvalidCharAvx2(pInputBuffer, inputLength); + } + if (Sse41.IsSupported) + { + return GetPointerToFirstInvalidCharSse(pInputBuffer, inputLength); + } + return GetPointerToFirstInvalidCharFallback(pInputBuffer, inputLength); + } + + /// + /// Returns true if contains no lone surrogate. + /// + public unsafe static bool IsWellFormed(ReadOnlySpan input) + { + fixed (char* p = input) + { + return GetPointerToFirstInvalidChar(p, input.Length) == p + input.Length; + } + } + + /// + /// Copies UTF-16 code units from to + /// , replacing every lone surrogate by U+FFFD. + /// The two buffers must either be identical (in-place operation) or not overlap. + /// + public unsafe static void ToWellFormed(char* pInputBuffer, int inputLength, char* pOutputBuffer) + { + if (AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian) + { + ToWellFormedArm64(pInputBuffer, inputLength, pOutputBuffer); + return; + } + if (Vector512.IsHardwareAccelerated && Avx512BW.IsSupported) + { + ToWellFormedAvx512(pInputBuffer, inputLength, pOutputBuffer); + return; + } + if (Avx2.IsSupported) + { + ToWellFormedAvx2(pInputBuffer, inputLength, pOutputBuffer); + return; + } + if (Sse41.IsSupported) + { + ToWellFormedSse(pInputBuffer, inputLength, pOutputBuffer); + return; + } + ToWellFormedFallback(pInputBuffer, inputLength, pOutputBuffer); + } + + /// + /// Copies to , replacing every lone surrogate by U+FFFD. + /// The spans must either start at the same address (in-place operation) or not overlap. + /// + public unsafe static void ToWellFormed(ReadOnlySpan source, Span destination) + { + if (destination.Length < source.Length) + { + throw new ArgumentException("Destination is too short.", nameof(destination)); + } + fixed (char* pIn = source) + fixed (char* pOut = destination) + { + if (pIn != pOut && source.Overlaps(destination)) + { + throw new ArgumentException("Source and destination must be identical or must not overlap.", nameof(destination)); + } + ToWellFormed(pIn, source.Length, pOut); + } + } + + /// + /// Returns a well-formed copy of where every lone surrogate is replaced by U+FFFD. + /// If is already well formed, it is returned as is (no allocation). + /// + public unsafe static string ToWellFormed(string input) + { + ArgumentNullException.ThrowIfNull(input); + int first; + fixed (char* p = input) + { + first = (int)(GetPointerToFirstInvalidChar(p, input.Length) - p); + } + if (first == input.Length) + { + return input; + } + return string.Create(input.Length, (input, first), static (dst, state) => + { + (string src, int first) = state; + src.AsSpan().CopyTo(dst); + // Nothing before 'first' needs fixing, and the code unit before 'first' + // is not a high surrogate, so we can fix the tail on its own. + fixed (char* p = dst) + { + ToWellFormed(p + first, dst.Length - first, p + first); + } + }); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsHighSurrogate(char c) => (c & 0xFC00) == 0xD800; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsLowSurrogate(char c) => (c & 0xFC00) == 0xDC00; + + /// + /// Straightforward scalar version, a port of simdutf's scalar::utf16::to_well_formed_utf16. + /// + public unsafe static void ToWellFormedScalar(char* pInputBuffer, int inputLength, char* pOutputBuffer) + { + ToWellFormedScalarFrom((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer, 0, false); + } + + // Processes input[i..n), where highSurrogatePrev tells whether input[i-1] is a high surrogate + // (output[i-1] is then overwritten if it is a lone surrogate). + private unsafe static void ToWellFormedScalarFrom(ushort* pInputBuffer, int inputLength, ushort* pOutputBuffer, int i, bool highSurrogatePrev) + { + for (; i < inputLength; i++) + { + char c = (char)pInputBuffer[i]; + bool highSurrogate = IsHighSurrogate(c); + bool lowSurrogate = IsLowSurrogate(c); + if (highSurrogatePrev && !lowSurrogate) + { + pOutputBuffer[i - 1] = ReplacementCharacter; + } + pOutputBuffer[i] = (!highSurrogatePrev && lowSurrogate) ? ReplacementCharacter : c; + highSurrogatePrev = highSurrogate; + } + // The string may not end with a high surrogate. + if (highSurrogatePrev) + { + pOutputBuffer[i - 1] = ReplacementCharacter; + } + } + + public unsafe static char* GetPointerToFirstInvalidCharScalar(char* pInputBuffer, int inputLength) + { + return GetPointerToFirstInvalidCharScalarFrom(pInputBuffer, inputLength, 0, false); + } + + // Scans pInputBuffer[i..n), where highSurrogatePrev tells whether pInputBuffer[i-1] is a high surrogate. + private unsafe static char* GetPointerToFirstInvalidCharScalarFrom(char* pInputBuffer, int inputLength, int i, bool highSurrogatePrev) + { + if (highSurrogatePrev) + { + if (i == inputLength || !IsLowSurrogate(pInputBuffer[i])) + { + return pInputBuffer + i - 1; + } + i++; + } + for (; i < inputLength; i++) + { + char c = pInputBuffer[i]; + if (IsLowSurrogate(c)) + { + return pInputBuffer + i; + } + if (IsHighSurrogate(c)) + { + if (i + 1 == inputLength || !IsLowSurrogate(pInputBuffer[i + 1])) + { + return pInputBuffer + i; + } + i++; + } + } + return pInputBuffer + inputLength; + } + + // Without AVX-512, we rely on the runtime's vectorized IndexOfAnyInRange to skip + // everything that is not a surrogate. + private unsafe static char* GetPointerToFirstInvalidCharFallback(char* pInputBuffer, int inputLength) + { + ReadOnlySpan s = new ReadOnlySpan(pInputBuffer, inputLength); + int i = 0; + while (true) + { + int k = s.Slice(i).IndexOfAnyInRange('\uD800', '\uDFFF'); + if (k < 0) + { + return pInputBuffer + inputLength; + } + i += k; + if (IsHighSurrogate(s[i]) && i + 1 < s.Length && IsLowSurrogate(s[i + 1])) + { + i += 2; + } + else + { + return pInputBuffer + i; + } + } + } + + private unsafe static void ToWellFormedFallback(char* pInputBuffer, int inputLength, char* pOutputBuffer) + { + if (pInputBuffer != pOutputBuffer) + { + Buffer.MemoryCopy(pInputBuffer, pOutputBuffer, (long)inputLength * sizeof(char), (long)inputLength * sizeof(char)); + } + // Replacing a lone surrogate never changes whether another surrogate is paired, + // so we can keep scanning the (partially fixed) output. + char* end = pOutputBuffer + inputLength; + char* p = GetPointerToFirstInvalidCharFallback(pOutputBuffer, inputLength); + while (p != end) + { + *p++ = ReplacementCharacter; + p = GetPointerToFirstInvalidCharFallback(p, (int)(end - p)); + } + } + + // ARM64 NEON. + // + // We gather the most significant bytes of 16 code units into one vector (uzp2), so + // that one comparison classifies 16 code units. The 'lookback' (is the previous + // code unit a high surrogate?) is obtained by shifting the high-surrogate vector by + // one lane (ext), carrying the last lane over from the previous block. (simdutf + // loads a second, lookback block instead.) A block is well formed exactly where + // lookback == low. + + // Most significant bytes of the 16 code units in v0 and v1 (little endian). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 Arm64HighBytes(Vector128 v0, Vector128 v1) + { + return AdvSimd.Arm64.UnzipOdd(v0.AsByte(), v1.AsByte()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Arm64Classify(Vector128 highBytes, out Vector128 isHigh, out Vector128 isLow) + { + Vector128 masked = AdvSimd.And(highBytes, Vector128.Create((byte)0xFC)); + isHigh = AdvSimd.CompareEqual(masked, Vector128.Create((byte)0xD8)); + isLow = AdvSimd.CompareEqual(masked, Vector128.Create((byte)0xDC)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool Arm64AnyNonZero(Vector128 v) + { + return AdvSimd.Arm64.MaxPairwise(v, v).AsUInt64().ToScalar() != 0; + } + + // True if any of the 64 code units (given by their most significant bytes) is a + // surrogate: adding 0x28 maps 0xD8 to 0xDF to 0 to 7. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool Arm64AnySurrogate(Vector128 h0, Vector128 h1, Vector128 h2, Vector128 h3) + { + Vector128 offset = Vector128.Create((byte)0x28); + Vector128 min = AdvSimd.Min(AdvSimd.Min(AdvSimd.Add(h0, offset), AdvSimd.Add(h1, offset)), + AdvSimd.Min(AdvSimd.Add(h2, offset), AdvSimd.Add(h3, offset))); + return AdvSimd.Arm64.MinAcross(min).ToScalar() < 8; + } + + // One bit per lane (lanes set to 0xFF or 0), 64 lanes. From simdutf. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Arm64Bits(Vector128 v0, Vector128 v1, Vector128 v2, Vector128 v3) + { + Vector128 bitMask = Vector128.Create((byte)0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80); + Vector128 sum0 = AdvSimd.Arm64.AddPairwise(AdvSimd.And(v0, bitMask), AdvSimd.And(v1, bitMask)); + Vector128 sum1 = AdvSimd.Arm64.AddPairwise(AdvSimd.And(v2, bitMask), AdvSimd.And(v3, bitMask)); + sum0 = AdvSimd.Arm64.AddPairwise(sum0, sum1); + sum0 = AdvSimd.Arm64.AddPairwise(sum0, sum0); + return sum0.AsUInt64().ToScalar(); + } + + // Replaces the lone surrogates flagged in 'illseq' (lookback XOR low). Bit j refers + // to the pair (output[j-1], output[j]): the lone surrogate is output[j-1] when + // output[j-1] is a high surrogate (bit j of lookback), output[j] otherwise. + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe static void Arm64FixBits(ushort* output, ulong illseq, ulong lookback) + { + while (illseq != 0) + { + int j = BitOperations.TrailingZeroCount(illseq); + output[j - (int)((lookback >> j) & 1)] = ReplacementCharacter; + illseq &= illseq - 1; + } + } + + // Generic flags: the JIT specializes generic methods over struct type arguments, + // so 'TInPlace.Value' is a constant in each instantiation. + private interface IFlag + { + static abstract bool Value { get; } + } + + private struct TrueFlag : IFlag + { + public static bool Value => true; + } + + private struct FalseFlag : IFlag + { + public static bool Value => false; + } + + public unsafe static void ToWellFormedArm64(char* pInputBuffer, int inputLength, char* pOutputBuffer) + { + if (pInputBuffer == pOutputBuffer) + { + Arm64FixAll((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer); + } + else + { + Arm64FixAll((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer); + } + } + + // Lookback and illegal-sequence vectors (one byte per code unit) for the 16 code + // units at p, where p[-1] is readable. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static Vector128 Arm64Window16(ushort* p, out Vector128 lookback) + { + Arm64Classify(Arm64HighBytes(AdvSimd.LoadVector128(p - 1), AdvSimd.LoadVector128(p + 7)), out lookback, out _); + Arm64Classify(Arm64HighBytes(AdvSimd.LoadVector128(p), AdvSimd.LoadVector128(p + 8)), out _, out Vector128 isLow); + return AdvSimd.Xor(lookback, isLow); + } + + // Same for the 8 code units at p (in the first 8 lanes, the others are zero); + // p[-1] is read only if hasPrevious. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static Vector128 Arm64Window8(ushort* p, bool hasPrevious, out Vector128 lookback) + { + Vector128 block = AdvSimd.LoadVector128(p); + Vector128 lb = hasPrevious ? AdvSimd.LoadVector128(p - 1) : AdvSimd.ExtractVector128(Vector128.Zero, block, 7); + Vector128 mask = Vector128.Create((ushort)0xFC00); + Vector128 lbIsHigh = AdvSimd.CompareEqual(AdvSimd.And(lb, mask), Vector128.Create((ushort)0xD800)); + Vector128 isLow = AdvSimd.CompareEqual(AdvSimd.And(block, mask), Vector128.Create((ushort)0xDC00)); + lookback = AdvSimd.ExtractNarrowingLower(lbIsHigh).ToVector128(); + return AdvSimd.ExtractNarrowingLower(AdvSimd.Xor(lbIsHigh, isLow)).ToVector128(); + } + + // Replaces the lone surrogates of a window, if any. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static void Arm64FixWindow(ushort* output, Vector128 illseq, Vector128 lookback) + { + if (Arm64AnyNonZero(illseq)) + { + Vector128 zero = Vector128.Zero; + Arm64FixBits(output, Arm64Bits(illseq, zero, zero, zero), Arm64Bits(lookback, zero, zero, zero)); + } + } + + // Offset of the first lone surrogate of a window, or int.MinValue if there is none. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Arm64WindowError(Vector128 illseq, Vector128 lookback) + { + if (!Arm64AnyNonZero(illseq)) + { + return int.MinValue; + } + Vector128 zero = Vector128.Zero; + return Arm64FirstError(Arm64Bits(illseq, zero, zero, zero), Arm64Bits(lookback, zero, zero, zero)); + } + + // Not inlined: its own compilation unit, so that the JIT can inline all the helpers. + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe static void Arm64FixAll(ushort* input, int n, ushort* output) where TInPlace : struct, IFlag + { + ushort* p = input; + ushort* q = output; + ushort* end = input + n; + // Lane 15 tells whether the code unit before p is a high surrogate. + Vector128 prevIsHigh = Vector128.Zero; + for (; end - p >= 64; p += 64, q += 64) + { + Vector128 v0 = AdvSimd.LoadVector128(p); + Vector128 v1 = AdvSimd.LoadVector128(p + 8); + Vector128 v2 = AdvSimd.LoadVector128(p + 16); + Vector128 v3 = AdvSimd.LoadVector128(p + 24); + Vector128 v4 = AdvSimd.LoadVector128(p + 32); + Vector128 v5 = AdvSimd.LoadVector128(p + 40); + Vector128 v6 = AdvSimd.LoadVector128(p + 48); + Vector128 v7 = AdvSimd.LoadVector128(p + 56); + if (!TInPlace.Value) + { + AdvSimd.Store(q, v0); + AdvSimd.Store(q + 8, v1); + AdvSimd.Store(q + 16, v2); + AdvSimd.Store(q + 24, v3); + AdvSimd.Store(q + 32, v4); + AdvSimd.Store(q + 40, v5); + AdvSimd.Store(q + 48, v6); + AdvSimd.Store(q + 56, v7); + } + Vector128 h0 = Arm64HighBytes(v0, v1); + Vector128 h1 = Arm64HighBytes(v2, v3); + Vector128 h2 = Arm64HighBytes(v4, v5); + Vector128 h3 = Arm64HighBytes(v6, v7); + // Most text has no surrogate at all. + if (prevIsHigh.GetElement(15) == 0 && !Arm64AnySurrogate(h0, h1, h2, h3)) + { + continue; + } + Arm64Classify(h0, out Vector128 isHigh0, out Vector128 isLow0); + Arm64Classify(h1, out Vector128 isHigh1, out Vector128 isLow1); + Arm64Classify(h2, out Vector128 isHigh2, out Vector128 isLow2); + Arm64Classify(h3, out Vector128 isHigh3, out Vector128 isLow3); + Vector128 lb0 = AdvSimd.ExtractVector128(prevIsHigh, isHigh0, 15); + Vector128 lb1 = AdvSimd.ExtractVector128(isHigh0, isHigh1, 15); + Vector128 lb2 = AdvSimd.ExtractVector128(isHigh1, isHigh2, 15); + Vector128 lb3 = AdvSimd.ExtractVector128(isHigh2, isHigh3, 15); + prevIsHigh = isHigh3; + Vector128 ill0 = AdvSimd.Xor(lb0, isLow0); + Vector128 ill1 = AdvSimd.Xor(lb1, isLow1); + Vector128 ill2 = AdvSimd.Xor(lb2, isLow2); + Vector128 ill3 = AdvSimd.Xor(lb3, isLow3); + if (Arm64AnyNonZero(AdvSimd.Or(AdvSimd.Or(ill0, ill1), AdvSimd.Or(ill2, ill3)))) + { + Arm64FixBits(q, Arm64Bits(ill0, ill1, ill2, ill3), Arm64Bits(lb0, lb1, lb2, lb3)); + } + } + for (; end - p >= 16; p += 16, q += 16) + { + Vector128 v0 = AdvSimd.LoadVector128(p); + Vector128 v1 = AdvSimd.LoadVector128(p + 8); + if (!TInPlace.Value) + { + AdvSimd.Store(q, v0); + AdvSimd.Store(q + 8, v1); + } + Arm64Classify(Arm64HighBytes(v0, v1), out Vector128 isHigh, out Vector128 isLow); + Vector128 lb = AdvSimd.ExtractVector128(prevIsHigh, isHigh, 15); + prevIsHigh = isHigh; + Vector128 ill = AdvSimd.Xor(lb, isLow); + if (Arm64AnyNonZero(ill)) + { + Vector128 zero = Vector128.Zero; + Arm64FixBits(q, Arm64Bits(ill, zero, zero, zero), Arm64Bits(lb, zero, zero, zero)); + } + } + if (p != end) + { + // The last 1 to 15 code units, using overlapping windows. Code units that were + // already written are recomputed identically. (In place, a replaced code unit + // was a lone surrogate: replacing it does not change the pairing of the others.) + if (n > 16) + { + ushort* w = end - 16; + if (!TInPlace.Value) + { + AdvSimd.Store(output + n - 16, AdvSimd.LoadVector128(w)); + AdvSimd.Store(output + n - 8, AdvSimd.LoadVector128(w + 8)); + } + Arm64FixWindow(output + n - 16, Arm64Window16(w, out Vector128 lookback), lookback); + } + else if (n >= 8) + { + Vector128 first = AdvSimd.LoadVector128(input); + Vector128 last = AdvSimd.LoadVector128(input + n - 8); + if (!TInPlace.Value) + { + AdvSimd.Store(output, first); + AdvSimd.Store(output + n - 8, last); + } + Arm64FixWindow(output, Arm64Window8(input, false, out Vector128 lookback), lookback); + if (n > 8) + { + Arm64FixWindow(output + n - 8, Arm64Window8(input + n - 8, true, out lookback), lookback); + } + } + else + { + ToWellFormedScalarFrom(input, n, output, 0, false); + return; + } + } + // The string may not end with a high surrogate. + if (n > 0 && IsHighSurrogate((char)output[n - 1])) + { + output[n - 1] = ReplacementCharacter; + } + } + + // For inputs of fewer than 64 code units: true if there is no surrogate at all, + // which is the common case. We use overlapping loads. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static bool Arm64ShortHasNoSurrogate(ushort* input, int n) + { + if (n >= 8) + { + Vector128 offset = Vector128.Create((ushort)0x2800); + Vector128 min = AdvSimd.Add(AdvSimd.LoadVector128(input + n - 8), offset); + for (int i = 0; i + 8 < n; i += 8) + { + min = AdvSimd.Min(min, AdvSimd.Add(AdvSimd.LoadVector128(input + i), offset)); + } + return AdvSimd.Arm64.MinAcross(min).ToScalar() >= 0x800; + } + if (n >= 4) + { + Vector64 offset = Vector64.Create((ushort)0x2800); + Vector64 min = AdvSimd.Min(AdvSimd.Add(AdvSimd.LoadVector64(input), offset), + AdvSimd.Add(AdvSimd.LoadVector64(input + n - 4), offset)); + return AdvSimd.Arm64.MinAcross(min).ToScalar() >= 0x800; + } + for (int i = 0; i < n; i++) + { + if ((ushort)(input[i] - 0xD800) < 0x800) + { + return false; + } + } + return true; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public unsafe static char* GetPointerToFirstInvalidCharArm64(char* pInputBuffer, int inputLength) + { + ushort* input = (ushort*)pInputBuffer; + int n = inputLength; + if (n < 64 && Arm64ShortHasNoSurrogate(input, n)) + { + return pInputBuffer + n; + } + ushort* p = input; + ushort* end = input + n; + Vector128 prevIsHigh = Vector128.Zero; + for (; end - p >= 64; p += 64) + { + Vector128 h0 = Arm64HighBytes(AdvSimd.LoadVector128(p), AdvSimd.LoadVector128(p + 8)); + Vector128 h1 = Arm64HighBytes(AdvSimd.LoadVector128(p + 16), AdvSimd.LoadVector128(p + 24)); + Vector128 h2 = Arm64HighBytes(AdvSimd.LoadVector128(p + 32), AdvSimd.LoadVector128(p + 40)); + Vector128 h3 = Arm64HighBytes(AdvSimd.LoadVector128(p + 48), AdvSimd.LoadVector128(p + 56)); + if (prevIsHigh.GetElement(15) == 0 && !Arm64AnySurrogate(h0, h1, h2, h3)) + { + continue; + } + Arm64Classify(h0, out Vector128 isHigh0, out Vector128 isLow0); + Arm64Classify(h1, out Vector128 isHigh1, out Vector128 isLow1); + Arm64Classify(h2, out Vector128 isHigh2, out Vector128 isLow2); + Arm64Classify(h3, out Vector128 isHigh3, out Vector128 isLow3); + Vector128 lb0 = AdvSimd.ExtractVector128(prevIsHigh, isHigh0, 15); + Vector128 lb1 = AdvSimd.ExtractVector128(isHigh0, isHigh1, 15); + Vector128 lb2 = AdvSimd.ExtractVector128(isHigh1, isHigh2, 15); + Vector128 lb3 = AdvSimd.ExtractVector128(isHigh2, isHigh3, 15); + prevIsHigh = isHigh3; + Vector128 ill0 = AdvSimd.Xor(lb0, isLow0); + Vector128 ill1 = AdvSimd.Xor(lb1, isLow1); + Vector128 ill2 = AdvSimd.Xor(lb2, isLow2); + Vector128 ill3 = AdvSimd.Xor(lb3, isLow3); + if (Arm64AnyNonZero(AdvSimd.Or(AdvSimd.Or(ill0, ill1), AdvSimd.Or(ill2, ill3)))) + { + return (char*)p + Arm64FirstError(Arm64Bits(ill0, ill1, ill2, ill3), Arm64Bits(lb0, lb1, lb2, lb3)); + } + } + for (; end - p >= 16; p += 16) + { + Arm64Classify(Arm64HighBytes(AdvSimd.LoadVector128(p), AdvSimd.LoadVector128(p + 8)), + out Vector128 isHigh, out Vector128 isLow); + Vector128 lb = AdvSimd.ExtractVector128(prevIsHigh, isHigh, 15); + prevIsHigh = isHigh; + Vector128 ill = AdvSimd.Xor(lb, isLow); + if (Arm64AnyNonZero(ill)) + { + Vector128 zero = Vector128.Zero; + return (char*)p + Arm64FirstError(Arm64Bits(ill, zero, zero, zero), Arm64Bits(lb, zero, zero, zero)); + } + } + if (p != end) + { + // The last 1 to 15 code units, using overlapping windows: the pairs that + // were already checked are well formed. + if (n > 16) + { + int j = Arm64WindowError(Arm64Window16(end - 16, out Vector128 lookback), lookback); + if (j != int.MinValue) + { + return pInputBuffer + n - 16 + j; + } + } + else if (n >= 8) + { + int j = Arm64WindowError(Arm64Window8(input, false, out Vector128 lookback), lookback); + if (j != int.MinValue) + { + return pInputBuffer + j; + } + if (n > 8) + { + j = Arm64WindowError(Arm64Window8(input + n - 8, true, out lookback), lookback); + if (j != int.MinValue) + { + return pInputBuffer + n - 8 + j; + } + } + } + else + { + return GetPointerToFirstInvalidCharScalarFrom(pInputBuffer, n, 0, false); + } + } + return (n > 0 && IsHighSurrogate(pInputBuffer[n - 1])) ? pInputBuffer + n - 1 : pInputBuffer + n; + } + + // Offset of the first lone surrogate: see Arm64FixBits. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Arm64FirstError(ulong illseq, ulong lookback) + { + int j = BitOperations.TrailingZeroCount(illseq); + return j - (int)((lookback >> j) & 1); + } + + // AVX2 (Haswell and better). + // + // Same approach as the AVX-512 kernel, but AVX2 has no mask registers: vpmovmskb + // gives us 2 bits per code unit, so a 64-bit word covers 32 code units and the + // high-surrogate bitmask is shifted by 2 (carrying 2 bits between words). We fix + // errors with a scalar loop over the bitmask (as simdutf does on ARM), and handle + // the end of the input with overlapping windows since AVX2 has no 16-bit masked loads. + + // Two bits per code unit (2j and 2j+1 for block[j]) for high and low surrogates. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Avx2Classify(Vector256 block, out uint high, out uint low) + { + Vector256 masked = block & Vector256.Create((ushort)0xFC00); + high = (uint)Avx2.MoveMask(Vector256.Equals(masked, Vector256.Create((ushort)0xD800)).AsByte()); + low = (uint)Avx2.MoveMask(Vector256.Equals(masked, Vector256.Create((ushort)0xDC00)).AsByte()); + } + + // True if any of the 64 code units is a surrogate: adding 0xA800 maps the surrogates + // (0xD800 to 0xDFFF) to the smallest signed 16-bit values (-32768 to -30721). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool Avx2AnySurrogate(Vector256 b0, Vector256 b1, Vector256 b2, Vector256 b3) + { + Vector256 offset = Vector256.Create(unchecked((short)0xA800)); + Vector256 min = Vector256.Min(Vector256.Min(b0.AsInt16() + offset, b1.AsInt16() + offset), + Vector256.Min(b2.AsInt16() + offset, b3.AsInt16() + offset)); + return Vector256.LessThanAny(min, Vector256.Create((short)-30720)); + } + + // Replaces the lone surrogates given illseq = lookback ^ low (2 bits per code unit). + // The pair (output[j-1], output[j]) is ill formed; output[j-1] is the lone surrogate + // when it is a high surrogate (bit 2j of lookback), output[j] otherwise. + // Inlined: a call inside the main loop makes the JIT spill registers. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static void Avx2FixBits(ushort* output, ulong illseq, ulong lookback) + { + illseq &= 0x5555555555555555UL; + while (illseq != 0) + { + int bit = BitOperations.TrailingZeroCount(illseq); + output[(bit >> 1) - (int)((lookback >> bit) & 1)] = ReplacementCharacter; + illseq &= illseq - 1; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Avx2FirstError(ulong illseq, ulong lookback) + { + int bit = BitOperations.TrailingZeroCount(illseq); + return (bit >> 1) - (int)((lookback >> bit) & 1); + } + + // Illegal-sequence and lookback vectors for the 16 code units at p, where p[-1] is readable. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static Vector256 Avx2Window16(ushort* p, out Vector256 lookback) + { + Vector256 mask = Vector256.Create((ushort)0xFC00); + lookback = Vector256.Equals(Vector256.Load(p - 1) & mask, Vector256.Create((ushort)0xD800)); + Vector256 isLow = Vector256.Equals(Vector256.Load(p) & mask, Vector256.Create((ushort)0xDC00)); + return lookback ^ isLow; + } + + // Same for the 8 code units at p; p[-1] is read only if hasPrevious. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static Vector128 Sse2Window8(ushort* p, bool hasPrevious, out Vector128 lookback) + { + Vector128 block = Vector128.Load(p); + Vector128 lb = hasPrevious ? Vector128.Load(p - 1) : Sse2.ShiftLeftLogical128BitLane(block, 2); + Vector128 mask = Vector128.Create((ushort)0xFC00); + lookback = Vector128.Equals(lb & mask, Vector128.Create((ushort)0xD800)); + Vector128 isLow = Vector128.Equals(block & mask, Vector128.Create((ushort)0xDC00)); + return lookback ^ isLow; + } + + // For inputs of fewer than 64 code units: true if there is no surrogate at all. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static bool Avx2ShortHasNoSurrogate(ushort* input, int n) + { + if (n >= 16) + { + Vector256 offset = Vector256.Create(unchecked((short)0xA800)); + Vector256 min = Vector256.Load((short*)input + n - 16) + offset; + for (int i = 0; i + 16 < n; i += 16) + { + min = Vector256.Min(min, Vector256.Load((short*)input + i) + offset); + } + return !Vector256.LessThanAny(min, Vector256.Create((short)-30720)); + } + return SseShortHasNoSurrogate(input, n); + } + + // For inputs of fewer than 64 code units: true if there is no surrogate at all. + // SSE2 only: overlapping 128-bit loads. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static bool SseShortHasNoSurrogate(ushort* input, int n) + { + if (n >= 8) + { + Vector128 offset = Vector128.Create(unchecked((short)0xA800)); + Vector128 min = Vector128.Load((short*)input + n - 8) + offset; + for (int i = 0; i + 8 < n; i += 8) + { + min = Vector128.Min(min, Vector128.Load((short*)input + i) + offset); + } + return !Vector128.LessThanAny(min, Vector128.Create((short)-30720)); + } + if (n >= 4) + { + Vector128 offset = Vector128.Create(unchecked((short)0xA800)); + Vector128 a = Sse2.X64.ConvertScalarToVector128UInt64(*(ulong*)input).AsInt16(); + Vector128 b = Sse2.X64.ConvertScalarToVector128UInt64(*(ulong*)(input + n - 4)).AsInt16(); + // The upper lanes are zero, and 0 + 0xA800 is not below -30720. + Vector128 min = Vector128.Min(a + offset, b + offset); + return !Vector128.LessThanAny(min, Vector128.Create((short)-30720)); + } + for (int i = 0; i < n; i++) + { + if ((ushort)(input[i] - 0xD800) < 0x800) + { + return false; + } + } + return true; + } + + public unsafe static void ToWellFormedAvx2(char* pInputBuffer, int inputLength, char* pOutputBuffer) + { + if (pInputBuffer == pOutputBuffer) + { + Avx2FixAll((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer); + } + else + { + Avx2FixAll((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer); + } + } + + // Processes the 64 code units at p. The state holds the carry (bits 0-1: 0b11 if the + // code unit before p is a high surrogate) and bit 2 (the previous block had surrogates, + // e.g., emojis: we then skip the pre-check). Returned by value to keep it in a register. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static ulong Avx2Fix64(ushort* p, ushort* q, ulong state) where TInPlace : struct, IFlag + { + Vector256 v0 = Vector256.Load(p); + Vector256 v1 = Vector256.Load(p + 16); + Vector256 v2 = Vector256.Load(p + 32); + Vector256 v3 = Vector256.Load(p + 48); + if (!TInPlace.Value) + { + v0.Store(q); + v1.Store(q + 16); + v2.Store(q + 32); + v3.Store(q + 48); + } + // Most text has no surrogate at all. + if (state == 0 && !Avx2AnySurrogate(v0, v1, v2, v3)) + { + return 0; + } + Avx2Classify(v0, out uint high0, out uint low0); + Avx2Classify(v1, out uint high1, out uint low1); + Avx2Classify(v2, out uint high2, out uint low2); + Avx2Classify(v3, out uint high3, out uint low3); + ulong highA = high0 | ((ulong)high1 << 32); + ulong lowA = low0 | ((ulong)low1 << 32); + ulong highB = high2 | ((ulong)high3 << 32); + ulong lowB = low2 | ((ulong)low3 << 32); + ulong lbA = (highA << 2) | (state & 3); + ulong lbB = (highB << 2) | (highA >> 62); + if (((lbA ^ lowA) | (lbB ^ lowB)) != 0) + { + Avx2FixBits(q, lbA ^ lowA, lbA); + Avx2FixBits(q + 32, lbB ^ lowB, lbB); + } + return (highB >> 62) | ((highA | lowA | highB | lowB) != 0 ? 4UL : 0UL); + } + + // Not inlined: its own compilation unit, so that the JIT can inline all the helpers. + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe static void Avx2FixAll(ushort* input, int n, ushort* output) where TInPlace : struct, IFlag + { + ushort* p = input; + ushort* q = output; + ushort* end = input + n; + ulong state = 0; + if (!TInPlace.Value && n >= 128) + { + // Misaligned stores are expensive: after the first block, we step back so that + // the output is 32-byte aligned. The overlapping code units are stored and + // checked again, with the carry recomputed from the input. + state = Avx2Fix64(p, q, state); + p += 64; + q += 64; + nuint misalignment = (nuint)q & 31; + if (misalignment != 0 && (misalignment & 1) == 0) + { + p -= misalignment / 2; + q -= misalignment / 2; + state = IsHighSurrogate((char)p[-1]) ? 3UL : 0UL; + } + } + for (; p + 64 <= end; p += 64, q += 64) + { + state = Avx2Fix64(p, q, state); + } + ulong carry = state & 3; + for (; p + 16 <= end; p += 16, q += 16) + { + Vector256 v = Vector256.Load(p); + if (!TInPlace.Value) + { + v.Store(q); + } + Avx2Classify(v, out uint high, out uint low); + ulong lb = ((ulong)high << 2) | carry; + carry = high >> 30; + if ((uint)lb != low) + { + Avx2FixBits(q, (uint)lb ^ low, lb); + } + } + if (p != end) + { + // The last 1 to 15 code units, using overlapping windows. Code units that were + // already written are recomputed identically. (In place, a replaced code unit + // was a lone surrogate: replacing it does not change the pairing of the others.) + if (n > 16) + { + Vector256 last = Vector256.Load(end - 16); + if (!TInPlace.Value) + { + last.Store(output + n - 16); + } + Vector256 ill = Avx2Window16(end - 16, out Vector256 lookback); + if (ill != Vector256.Zero) + { + Avx2FixBits(output + n - 16, (uint)Avx2.MoveMask(ill.AsByte()), (uint)Avx2.MoveMask(lookback.AsByte())); + } + } + else if (n >= 8) + { + Vector128 first = Vector128.Load(input); + Vector128 last = Vector128.Load(input + n - 8); + if (!TInPlace.Value) + { + first.Store(output); + last.Store(output + n - 8); + } + Vector128 ill = Sse2Window8(input, false, out Vector128 lookback); + if (ill != Vector128.Zero) + { + Avx2FixBits(output, (uint)Sse2.MoveMask(ill.AsByte()), (uint)Sse2.MoveMask(lookback.AsByte())); + } + if (n > 8) + { + ill = Sse2Window8(input + n - 8, true, out lookback); + if (ill != Vector128.Zero) + { + Avx2FixBits(output + n - 8, (uint)Sse2.MoveMask(ill.AsByte()), (uint)Sse2.MoveMask(lookback.AsByte())); + } + } + } + else + { + ToWellFormedScalarFrom(input, n, output, 0, false); + return; + } + } + // The string may not end with a high surrogate. + if (n > 0 && IsHighSurrogate((char)output[n - 1])) + { + output[n - 1] = ReplacementCharacter; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public unsafe static char* GetPointerToFirstInvalidCharAvx2(char* pInputBuffer, int inputLength) + { + ushort* input = (ushort*)pInputBuffer; + int n = inputLength; + if (n < 64 && Avx2ShortHasNoSurrogate(input, n)) + { + return pInputBuffer + n; + } + ushort* p = input; + ushort* end = input + n; + ulong carry = 0; + bool surrogates = false; + for (; p + 64 <= end; p += 64) + { + Vector256 v0 = Vector256.Load(p); + Vector256 v1 = Vector256.Load(p + 16); + Vector256 v2 = Vector256.Load(p + 32); + Vector256 v3 = Vector256.Load(p + 48); + if (!surrogates && carry == 0 && !Avx2AnySurrogate(v0, v1, v2, v3)) + { + continue; + } + Avx2Classify(v0, out uint high0, out uint low0); + Avx2Classify(v1, out uint high1, out uint low1); + ulong high = high0 | ((ulong)high1 << 32); + ulong low = low0 | ((ulong)low1 << 32); + surrogates = (high | low) != 0; + ulong lb = (high << 2) | carry; + if (lb != low) + { + return (char*)p + Avx2FirstError(lb ^ low, lb); + } + carry = high >> 62; + Avx2Classify(v2, out high0, out low0); + Avx2Classify(v3, out high1, out low1); + high = high0 | ((ulong)high1 << 32); + low = low0 | ((ulong)low1 << 32); + lb = (high << 2) | carry; + if (lb != low) + { + return (char*)p + 32 + Avx2FirstError(lb ^ low, lb); + } + carry = high >> 62; + } + for (; p + 16 <= end; p += 16) + { + Avx2Classify(Vector256.Load(p), out uint high, out uint low); + ulong lb = ((ulong)high << 2) | carry; + if ((uint)lb != low) + { + return (char*)p + Avx2FirstError((uint)lb ^ low, lb); + } + carry = high >> 30; + } + if (p != end) + { + // The last 1 to 15 code units, using overlapping windows: the pairs that + // were already checked are well formed. + if (n > 16) + { + Vector256 ill = Avx2Window16(end - 16, out Vector256 lookback); + if (ill != Vector256.Zero) + { + return pInputBuffer + n - 16 + Avx2FirstError((uint)Avx2.MoveMask(ill.AsByte()), (uint)Avx2.MoveMask(lookback.AsByte())); + } + } + else if (n >= 8) + { + Vector128 ill = Sse2Window8(input, false, out Vector128 lookback); + if (ill != Vector128.Zero) + { + return pInputBuffer + Avx2FirstError((uint)Sse2.MoveMask(ill.AsByte()), (uint)Sse2.MoveMask(lookback.AsByte())); + } + if (n > 8) + { + ill = Sse2Window8(input + n - 8, true, out lookback); + if (ill != Vector128.Zero) + { + return pInputBuffer + n - 8 + Avx2FirstError((uint)Sse2.MoveMask(ill.AsByte()), (uint)Sse2.MoveMask(lookback.AsByte())); + } + } + } + else + { + return GetPointerToFirstInvalidCharScalarFrom(pInputBuffer, n, 0, false); + } + } + return (n > 0 && IsHighSurrogate(pInputBuffer[n - 1])) ? pInputBuffer + n - 1 : pInputBuffer + n; + } + + // SSE (Westmere and better, no AVX). + // + // Same approach as the AVX-512 kernel with 128-bit registers: we pack the comparison + // results of two registers (packsswb) so that one pmovmskb gives one bit per code + // unit for 16 code units. A 64-bit word covers 64 code units, the high-surrogate + // bitmask is shifted by one, carrying one bit between words. Errors are fixed with a + // scalar loop over the bitmask; the end of the input uses overlapping windows. + + // One bit per code unit for the 16 code units in v0 and v1. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SseClassify(Vector128 v0, Vector128 v1, out uint high, out uint low) + { + Vector128 fc = Vector128.Create((ushort)0xFC00); + Vector128 d8 = Vector128.Create((ushort)0xD800); + Vector128 dc = Vector128.Create((ushort)0xDC00); + Vector128 m0 = v0 & fc; + Vector128 m1 = v1 & fc; + high = (uint)Sse2.MoveMask(Sse2.PackSignedSaturate(Vector128.Equals(m0, d8).AsInt16(), Vector128.Equals(m1, d8).AsInt16())); + low = (uint)Sse2.MoveMask(Sse2.PackSignedSaturate(Vector128.Equals(m0, dc).AsInt16(), Vector128.Equals(m1, dc).AsInt16())); + } + + // True if any of the 64 code units is a surrogate (see Avx2AnySurrogate). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool SseAnySurrogate(Vector128 v0, Vector128 v1, Vector128 v2, Vector128 v3, + Vector128 v4, Vector128 v5, Vector128 v6, Vector128 v7) + { + Vector128 offset = Vector128.Create(unchecked((short)0xA800)); + Vector128 m0 = Vector128.Min(v0.AsInt16() + offset, v1.AsInt16() + offset); + Vector128 m1 = Vector128.Min(v2.AsInt16() + offset, v3.AsInt16() + offset); + Vector128 m2 = Vector128.Min(v4.AsInt16() + offset, v5.AsInt16() + offset); + Vector128 m3 = Vector128.Min(v6.AsInt16() + offset, v7.AsInt16() + offset); + Vector128 min = Vector128.Min(Vector128.Min(m0, m1), Vector128.Min(m2, m3)); + return Vector128.LessThanAny(min, Vector128.Create((short)-30720)); + } + + // Replaces the lone surrogates given illseq = lookback ^ low, one bit per code unit + // (see Arm64FixBits). Inlined: a call inside the main loop makes the JIT spill registers. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static void SseFixBits(ushort* output, ulong illseq, ulong lookback) + { + while (illseq != 0) + { + int j = BitOperations.TrailingZeroCount(illseq); + output[j - (int)((lookback >> j) & 1)] = ReplacementCharacter; + illseq &= illseq - 1; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int SseFirstError(ulong illseq, ulong lookback) + { + int j = BitOperations.TrailingZeroCount(illseq); + return j - (int)((lookback >> j) & 1); + } + + // High and low bits for 64 code units. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SseClassify64(Vector128 v0, Vector128 v1, Vector128 v2, Vector128 v3, + Vector128 v4, Vector128 v5, Vector128 v6, Vector128 v7, + out ulong high, out ulong low) + { + SseClassify(v0, v1, out uint h0, out uint l0); + SseClassify(v2, v3, out uint h1, out uint l1); + SseClassify(v4, v5, out uint h2, out uint l2); + SseClassify(v6, v7, out uint h3, out uint l3); + high = h0 | ((ulong)h1 << 16) | ((ulong)h2 << 32) | ((ulong)h3 << 48); + low = l0 | ((ulong)l1 << 16) | ((ulong)l2 << 32) | ((ulong)l3 << 48); + } + + public unsafe static void ToWellFormedSse(char* pInputBuffer, int inputLength, char* pOutputBuffer) + { + if (pInputBuffer == pOutputBuffer) + { + SseFixAll((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer); + } + else + { + SseFixAll((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer); + } + } + + // Processes the 64 code units at p. The state holds the carry (bit 0: the code unit + // before p is a high surrogate) and bit 1 (the previous block had surrogates: we then + // skip the pre-check). We return the new state by value: a 'ref' parameter would keep + // it out of registers. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static ulong SseFix64(ushort* p, ushort* q, ulong state) where TInPlace : struct, IFlag + { + Vector128 v0 = Vector128.Load(p); + Vector128 v1 = Vector128.Load(p + 8); + Vector128 v2 = Vector128.Load(p + 16); + Vector128 v3 = Vector128.Load(p + 24); + Vector128 v4 = Vector128.Load(p + 32); + Vector128 v5 = Vector128.Load(p + 40); + Vector128 v6 = Vector128.Load(p + 48); + Vector128 v7 = Vector128.Load(p + 56); + if (!TInPlace.Value) + { + v0.Store(q); + v1.Store(q + 8); + v2.Store(q + 16); + v3.Store(q + 24); + v4.Store(q + 32); + v5.Store(q + 40); + v6.Store(q + 48); + v7.Store(q + 56); + } + // Most text has no surrogate at all. + if (state == 0 && !SseAnySurrogate(v0, v1, v2, v3, v4, v5, v6, v7)) + { + return 0; + } + SseClassify64(v0, v1, v2, v3, v4, v5, v6, v7, out ulong high, out ulong low); + ulong lb = (high << 1) | (state & 1); + if (lb != low) + { + SseFixBits(q, lb ^ low, lb); + } + return (high >> 63) | ((high | low) != 0 ? 2UL : 0UL); + } + + // Not inlined: as its own compilation unit, the JIT has the budget to inline all the + // helpers above (otherwise, they become calls in the main loop). + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe static void SseFixAll(ushort* input, int n, ushort* output) where TInPlace : struct, IFlag + { + ushort* p = input; + ushort* q = output; + ushort* end = input + n; + ulong state = 0; + if (!TInPlace.Value && n >= 128) + { + // Misaligned 16-byte stores are expensive: after the first block, we step + // back so that the output is 16-byte aligned. The overlapping code units are + // stored and checked again, with the carry recomputed from the input. + state = SseFix64(p, q, state); + p += 64; + q += 64; + nuint misalignment = (nuint)q & 15; + if (misalignment != 0 && (misalignment & 1) == 0) + { + p -= misalignment / 2; + q -= misalignment / 2; + state = IsHighSurrogate((char)p[-1]) ? 1UL : 0UL; + } + } + for (; p + 64 <= end; p += 64, q += 64) + { + state = SseFix64(p, q, state); + } + ulong carry = state & 1; + for (; p + 16 <= end; p += 16, q += 16) + { + Vector128 v0 = Vector128.Load(p); + Vector128 v1 = Vector128.Load(p + 8); + if (!TInPlace.Value) + { + v0.Store(q); + v1.Store(q + 8); + } + SseClassify(v0, v1, out uint high, out uint low); + // 16 bits per block: bit 16 of lb belongs to the next block. + ulong lb = ((ulong)high << 1) | carry; + carry = high >> 15; + ulong illseq = (lb ^ low) & 0xFFFF; + if (illseq != 0) + { + SseFixBits(q, illseq, lb); + } + } + if (p != end) + { + // The last 1 to 15 code units, using overlapping 8-wide windows. Code units that + // were already written are recomputed identically. (In place, a replaced code unit + // was a lone surrogate: replacing it does not change the pairing of the others.) + if (n < 8) + { + ToWellFormedScalarFrom(input, n, output, 0, false); + return; + } + if (end - p > 8 || p == input) + { + // 9 to 15 code units left, or an input of 8 to 15 code units: one window at p + // (which may be the start of the input). + Vector128 first = Vector128.Load(p); + if (!TInPlace.Value) + { + first.Store(q); + } + Vector128 ill = Sse2Window8(p, p != input, out Vector128 lookback); + if (ill != Vector128.Zero) + { + SseFixBits(q, Sse2Bits8(ill), Sse2Bits8(lookback)); + } + } + if (n > 8) + { + Vector128 last = Vector128.Load(end - 8); + if (!TInPlace.Value) + { + last.Store(output + n - 8); + } + Vector128 ill = Sse2Window8(end - 8, true, out Vector128 lookback); + if (ill != Vector128.Zero) + { + SseFixBits(output + n - 8, Sse2Bits8(ill), Sse2Bits8(lookback)); + } + } + } + // The string may not end with a high surrogate. + if (n > 0 && IsHighSurrogate((char)output[n - 1])) + { + output[n - 1] = ReplacementCharacter; + } + } + + // One bit per 16-bit lane. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Sse2Bits8(Vector128 v) + { + return (uint)Sse2.MoveMask(Sse2.PackSignedSaturate(v.AsInt16(), Vector128.Zero)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public unsafe static char* GetPointerToFirstInvalidCharSse(char* pInputBuffer, int inputLength) + { + ushort* input = (ushort*)pInputBuffer; + int n = inputLength; + if (n < 64 && SseShortHasNoSurrogate(input, n)) + { + return pInputBuffer + n; + } + ushort* p = input; + ushort* end = input + n; + ulong carry = 0; + bool surrogates = false; + for (; p + 64 <= end; p += 64) + { + Vector128 v0 = Vector128.Load(p); + Vector128 v1 = Vector128.Load(p + 8); + Vector128 v2 = Vector128.Load(p + 16); + Vector128 v3 = Vector128.Load(p + 24); + Vector128 v4 = Vector128.Load(p + 32); + Vector128 v5 = Vector128.Load(p + 40); + Vector128 v6 = Vector128.Load(p + 48); + Vector128 v7 = Vector128.Load(p + 56); + if (!surrogates && carry == 0 && !SseAnySurrogate(v0, v1, v2, v3, v4, v5, v6, v7)) + { + continue; + } + SseClassify64(v0, v1, v2, v3, v4, v5, v6, v7, out ulong high, out ulong low); + surrogates = (high | low) != 0; + ulong lb = (high << 1) | carry; + if (lb != low) + { + return (char*)p + SseFirstError(lb ^ low, lb); + } + carry = high >> 63; + } + for (; p + 16 <= end; p += 16) + { + SseClassify(Vector128.Load(p), Vector128.Load(p + 8), out uint high, out uint low); + // 16 bits per block: bit 16 of lb belongs to the next block. + ulong lb = ((ulong)high << 1) | carry; + ulong illseq = (lb ^ low) & 0xFFFF; + if (illseq != 0) + { + return (char*)p + SseFirstError(illseq, lb); + } + carry = high >> 15; + } + if (p != end) + { + if (n < 8) + { + return GetPointerToFirstInvalidCharScalarFrom(pInputBuffer, n, 0, false); + } + if (end - p > 8 || p == input) + { + Vector128 ill = Sse2Window8(p, p != input, out Vector128 lookback); + if (ill != Vector128.Zero) + { + return (char*)p + SseFirstError(Sse2Bits8(ill), Sse2Bits8(lookback)); + } + } + if (n > 8) + { + Vector128 ill = Sse2Window8(end - 8, true, out Vector128 lookback); + if (ill != Vector128.Zero) + { + return pInputBuffer + n - 8 + SseFirstError(Sse2Bits8(ill), Sse2Bits8(lookback)); + } + } + } + return (n > 0 && IsHighSurrogate(pInputBuffer[n - 1])) ? pInputBuffer + n - 1 : pInputBuffer + n; + } + + // AVX-512 (Ice Lake and better, AMD Zen 4 and better). + // + // A low surrogate must be preceded by a high surrogate and a high surrogate must be + // followed by a low surrogate. With one bit per code unit, the input is well formed + // exactly where (high << 1) == low, where the bit shifted out of a block is carried + // into the next one. A string may not end with a high surrogate: we treat the end + // of the string as a code unit that is not a low surrogate. + // + // simdutf compares each block with a 'lookback' block loaded one code unit earlier; + // shifting bitmasks instead lets us load each code unit only once. + + // Bit j is set when block[j] is a high (resp. low) surrogate. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Avx512Classify(Vector512 block, out uint high, out uint low) + { + Vector512 masked = block & Vector512.Create((ushort)0xFC00); + high = (uint)Vector512.Equals(masked, Vector512.Create((ushort)0xD800)).ExtractMostSignificantBits(); + low = (uint)Vector512.Equals(masked, Vector512.Create((ushort)0xDC00)).ExtractMostSignificantBits(); + } + + // True if any of the 128 code units is a surrogate (0xD800 to 0xDFFF): adding 0x2800 + // maps the surrogates to 0 to 0x7FF, so we only need one comparison on the minimum. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool Avx512AnySurrogate(Vector512 b0, Vector512 b1, Vector512 b2, Vector512 b3) + { + Vector512 offset = Vector512.Create((ushort)0x2800); + Vector512 min = Vector512.Min(Vector512.Min(b0 + offset, b1 + offset), Vector512.Min(b2 + offset, b3 + offset)); + return Vector512.LessThanAny(min, Vector512.Create((ushort)0x800)); + } + + // Lanes 0 to count - 1 set. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 Avx512FirstLanes(int count) + { + return Vector512.LessThan(Vector512.Indices, Vector512.Create((ushort)count)); + } + + // Lane j set when bit j of 'bits' is set. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 Avx512MaskFromBits(uint bits) + { + // Lanes 0-15 get the low 16 bits, lanes 16-31 the high 16 bits (vpermw). We avoid + // Vector512.Create(Vector256, Vector256), which the JIT compiles to calls. + Vector512 v = Avx512BW.PermuteVar32x16(Vector512.Create(bits).AsUInt16(), + Vector512.Create((ushort)0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); + Vector512 bit = Vector512.Create( + (ushort)0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, + 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, + 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, + 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000); + return Vector512.Equals(v & bit, bit); + } + + // Writes the (up to) 32 code units of 'block' to 'output', replacing the lone + // surrogates, given lbHigh = (high << 1) | carry. Bit 0 of the lone high + // surrogates refers to output[-1], which was already written. + // Inlined: a call inside the main loop would force the JIT to spill vector registers. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static void Avx512FixSlow(Vector512 block, ushort* output, ulong lbHigh, uint low, Vector512 storeMask) + { + ulong loneHigh = lbHigh & ~(ulong)low; + uint loneLow = low & ~(uint)lbHigh; + if ((loneHigh & 1) != 0) + { + output[-1] = ReplacementCharacter; + } + uint bad = loneLow | (uint)(loneHigh >> 1); + Vector512 fixedBlock = Vector512.ConditionalSelect(Avx512MaskFromBits(bad), Vector512.Create((ushort)ReplacementCharacter), block); + Avx512BW.MaskStore(output, storeMask, fixedBlock); + } + + // 32 code units, not at the end of the input. Returns the new carry. + // (We avoid 'ref' parameters: they keep the carry out of registers.) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static ulong Avx512Fix32(Vector512 block, ushort* output, bool inPlace, ulong carry) + { + Avx512Classify(block, out uint high, out uint low); + ulong lbHigh = ((ulong)high << 1) | carry; + if ((uint)lbHigh == low) + { + if (!inPlace) + { + block.Store(output); + } + } + else + { + Avx512FixSlow(block, output, (uint)lbHigh, low, Vector512.AllBitsSet); + } + return high >> 31; + } + + // 64 code units, not at the end of the input. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static ulong Avx512Fix64(Vector512 block1, Vector512 block2, ushort* output, bool inPlace, ulong carry) + { + Avx512Classify(block1, out uint high1, out uint low1); + Avx512Classify(block2, out uint high2, out uint low2); + ulong high = high1 | ((ulong)high2 << 32); + ulong low = low1 | ((ulong)low2 << 32); + if (((high << 1) | carry) == low) + { + if (!inPlace) + { + block1.Store(output); + block2.Store(output + 32); + } + return high >> 63; + } + carry = Avx512Fix32(block1, output, inPlace, carry); + return Avx512Fix32(block2, output + 32, inPlace, carry); + } + + // The last 0 to 31 code units. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static void Avx512FixTail(ushort* input, int count, ushort* output, bool inPlace, ulong carry) + { + Vector512 mask = Avx512FirstLanes(count); + Vector512 block = Avx512BW.MaskLoad(input, mask, Vector512.Zero); + Avx512Classify(block, out uint high, out uint low); + // Bit 'count' of lbHigh is set if the input ends with a high surrogate: + // it is then a lone surrogate, since bit 'count' of low is zero. + ulong lbHigh = ((ulong)high << 1) | carry; + if (lbHigh == low) + { + if (!inPlace) + { + Avx512BW.MaskStore(output, mask, block); + } + return; + } + Avx512FixSlow(block, output, lbHigh, low, mask); + } + + public unsafe static void ToWellFormedAvx512(char* pInputBuffer, int inputLength, char* pOutputBuffer) + { + if (inputLength <= 0) + { + return; + } + // The loops are duplicated so that the JIT specializes them. + if (pInputBuffer == pOutputBuffer) + { + Avx512FixAll((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer); + } + else + { + Avx512FixAll((ushort*)pInputBuffer, inputLength, (ushort*)pOutputBuffer); + } + } + + // Not inlined: as its own compilation unit, the JIT has the budget to inline all the + // helpers (otherwise, some of them become calls in the main loop). + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe static void Avx512FixAll(ushort* input, int n, ushort* output) where TInPlace : struct, IFlag + { + bool inPlace = TInPlace.Value; + ulong carry = 0; + int i = 0; + if (!inPlace && n >= 256) + { + // Misaligned stores are expensive: after the first 64 code units, we step back + // so that the output is 64-byte aligned. The overlapping code units are stored + // and checked again, with the carry recomputed from the input. + carry = Avx512Fix64(Vector512.Load(input), Vector512.Load(input + 32), output, inPlace, carry); + i = 64; + nuint misalignment = (nuint)(output + i) & 63; + if (misalignment != 0 && (misalignment & 1) == 0) + { + i -= (int)(misalignment / 2); + carry = IsHighSurrogate((char)input[i - 1]) ? 1UL : 0UL; + } + } + for (; i + 128 <= n; i += 128) + { + Vector512 block0 = Vector512.Load(input + i); + Vector512 block1 = Vector512.Load(input + i + 32); + Vector512 block2 = Vector512.Load(input + i + 64); + Vector512 block3 = Vector512.Load(input + i + 96); + // Most text has no surrogate at all. + if (carry == 0 && !Avx512AnySurrogate(block0, block1, block2, block3)) + { + if (!inPlace) + { + block0.Store(output + i); + block1.Store(output + i + 32); + block2.Store(output + i + 64); + block3.Store(output + i + 96); + } + continue; + } + carry = Avx512Fix64(block0, block1, output + i, inPlace, carry); + carry = Avx512Fix64(block2, block3, output + i + 64, inPlace, carry); + } + if (i + 64 <= n) + { + carry = Avx512Fix64(Vector512.Load(input + i), Vector512.Load(input + i + 32), output + i, inPlace, carry); + i += 64; + } + if (i + 32 <= n) + { + carry = Avx512Fix32(Vector512.Load(input + i), output + i, inPlace, carry); + i += 32; + } + Avx512FixTail(input + i, n - i, output + i, inPlace, carry); + } + + // Offset of the first lone surrogate given lbHigh = (high << 1) | carry, when + // lbHigh != low: either the code unit at j - 1 is a high surrogate without a low + // surrogate after it, or the code unit at j is a low surrogate without a high + // surrogate before it. The result is -1 when the lone surrogate precedes the block. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Avx512FirstError(ulong lbHigh, ulong low) + { + int j = BitOperations.TrailingZeroCount(lbHigh ^ low); + return j - (int)((lbHigh >> j) & 1); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public unsafe static char* GetPointerToFirstInvalidCharAvx512(char* pInputBuffer, int inputLength) + { + ushort* input = (ushort*)pInputBuffer; + int n = inputLength; + ulong carry = 0; + ulong high, low, lbHigh; + int i = 0; + for (; i + 128 <= n; i += 128) + { + Vector512 block0 = Vector512.Load(input + i); + Vector512 block1 = Vector512.Load(input + i + 32); + Vector512 block2 = Vector512.Load(input + i + 64); + Vector512 block3 = Vector512.Load(input + i + 96); + if (carry == 0 && !Avx512AnySurrogate(block0, block1, block2, block3)) + { + continue; + } + Avx512Classify(block0, out uint high0, out uint low0); + Avx512Classify(block1, out uint high1, out uint low1); + high = high0 | ((ulong)high1 << 32); + low = low0 | ((ulong)low1 << 32); + lbHigh = (high << 1) | carry; + if (lbHigh != low) + { + return pInputBuffer + i + Avx512FirstError(lbHigh, low); + } + carry = high >> 63; + Avx512Classify(block2, out uint high2, out uint low2); + Avx512Classify(block3, out uint high3, out uint low3); + high = high2 | ((ulong)high3 << 32); + low = low2 | ((ulong)low3 << 32); + lbHigh = (high << 1) | carry; + if (lbHigh != low) + { + return pInputBuffer + i + 64 + Avx512FirstError(lbHigh, low); + } + carry = high >> 63; + } + for (; i + 32 <= n; i += 32) + { + Avx512Classify(Vector512.Load(input + i), out uint high0, out uint low0); + lbHigh = ((ulong)high0 << 1) | carry; + if ((uint)lbHigh != low0) + { + return pInputBuffer + i + Avx512FirstError((uint)lbHigh, low0); + } + carry = high0 >> 31; + } + { + // The last 0 to 31 code units. Bit n - i of lbHigh is set if the input + // ends with a high surrogate. + Vector512 block = Avx512BW.MaskLoad(input + i, Avx512FirstLanes(n - i), Vector512.Zero); + Avx512Classify(block, out uint high0, out uint low0); + lbHigh = ((ulong)high0 << 1) | carry; + if (lbHigh != low0) + { + return pInputBuffer + i + Avx512FirstError(lbHigh, low0); + } + } + return pInputBuffer + n; + } + } +} diff --git a/test/UTF16WellFormedTests.cs b/test/UTF16WellFormedTests.cs new file mode 100644 index 0000000..5062965 --- /dev/null +++ b/test/UTF16WellFormedTests.cs @@ -0,0 +1,399 @@ +namespace tests; + +using System.Text; +using SimdUnicode; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +// Tests for UTF16.ToWellFormed and UTF16.GetPointerToFirstInvalidChar, which follow +// JavaScript's String.prototype.toWellFormed() / isWellFormed() semantics. +public unsafe class UTF16WellFormedTests +{ + public unsafe delegate void ToWellFormedFunction(char* input, int length, char* output); + public unsafe delegate char* FirstInvalidFunction(char* input, int length); + + private static bool HasAvx512 => Vector512.IsHardwareAccelerated && Avx512BW.IsSupported; + + private sealed class FactOnAvx512Attribute : FactAttribute + { + public FactOnAvx512Attribute() + { + if (!HasAvx512) + { + Skip = "Test is skipped due to not meeting system requirements (AVX-512)."; + } + } + } + + private sealed class FactOnAvx2Attribute : FactAttribute + { + public FactOnAvx2Attribute() + { + if (!Avx2.IsSupported) + { + Skip = "Test is skipped due to not meeting system requirements (AVX2)."; + } + } + } + + private sealed class FactOnSse41Attribute : FactAttribute + { + public FactOnSse41Attribute() + { + if (!Sse41.IsSupported) + { + Skip = "Test is skipped due to not meeting system requirements (SSE4.1)."; + } + } + } + + private sealed class FactOnArm64Attribute : FactAttribute + { + public FactOnArm64Attribute() + { + if (!(System.Runtime.Intrinsics.Arm.AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian)) + { + Skip = "Test is skipped due to not meeting system requirements (ARM64)."; + } + } + } + + // Independent reference: decode rune by rune, as the runtime does. + private static string Reference(string s) + { + var sb = new StringBuilder(s.Length); + foreach (Rune r in s.EnumerateRunes()) + { + sb.Append(r.ToString()); + } + return sb.ToString(); + } + + private static int ReferenceFirstInvalid(string s) + { + int i = 0; + foreach (Rune r in s.EnumerateRunes()) + { + if (r == Rune.ReplacementChar && s[i] != '\uFFFD') + { + return i; + } + i += r.Utf16SequenceLength; + } + return s.Length; + } + + // Random strings mixing ASCII, other BMP characters, U+FFFD, valid pairs and lone surrogates. + private static string RandomString(Random rand, int length, int errorWeight) + { + var sb = new StringBuilder(length); + while (sb.Length < length) + { + int kind = rand.Next(8 + errorWeight); + switch (kind) + { + case 0: + case 1: + sb.Append((char)rand.Next(0x80)); + break; + case 2: + sb.Append((char)rand.Next(0x80, 0xD800)); + break; + case 3: + sb.Append((char)rand.Next(0xE000, 0x10000)); + break; + case 4: + sb.Append('\uFFFD'); + break; + case 5: + case 6: + case 7: + sb.Append((char)rand.Next(0xD800, 0xDC00)); + sb.Append((char)rand.Next(0xDC00, 0xE000)); + break; + default: + sb.Append((char)rand.Next(0xD800, 0xE000)); // lone (or accidentally paired) surrogate + break; + } + } + return sb.ToString(0, length); + } + + private static readonly string[] HardCoded = { + "", + "a", + "\uD800", + "\uDC00", + "􏿿", + "\uDC00\uD800", + "\uD800𐀀", + "𐀀\uDC00", + "ab😀cd", + "ab\uD83Dcd", + "ab\uDE00cd", + "😀😀\uD83D", + "\uDE00😀😀", + new string('\uD800', 100), + new string('\uDC00', 100), + string.Concat(Enumerable.Repeat("𐀀", 100)), + string.Concat(Enumerable.Repeat("\uDC00\uD800", 100)), + new string('x', 31) + "\uD800", + new string('x', 32) + "\uD800", + new string('x', 33) + "\uD800", + new string('x', 31) + "𐀀", + new string('x', 32) + "𐀀" + new string('x', 32), + "\uDC00" + new string('x', 40), + "\uD800" + new string('x', 40), + "𐀀" + new string('x', 40), + }; + + private static string Run(ToWellFormedFunction f, string s, bool inPlace) + { + // Pad on both sides with high surrogates to catch out-of-bounds reads and writes. + char[] buffer = new char[s.Length + 2]; + buffer[0] = '\uD800'; + buffer[^1] = '\uDC00'; + char[] output = new char[s.Length + 2]; + output[0] = 'A'; + output[^1] = 'Z'; + s.CopyTo(0, buffer, 1, s.Length); + fixed (char* pIn = buffer) + fixed (char* pOut = output) + { + if (inPlace) + { + f(pIn + 1, s.Length, pIn + 1); + Assert.Equal('\uD800', buffer[0]); + Assert.Equal('\uDC00', buffer[^1]); + return new string(buffer, 1, s.Length); + } + f(pIn + 1, s.Length, pOut + 1); + Assert.Equal('A', output[0]); + Assert.Equal('Z', output[^1]); + Assert.Equal(s, new string(buffer, 1, s.Length)); // input untouched + return new string(output, 1, s.Length); + } + } + + private static int RunFirst(FirstInvalidFunction f, string s) + { + char[] buffer = new char[s.Length + 2]; + buffer[0] = '\uD800'; + buffer[^1] = '\uDC00'; + s.CopyTo(0, buffer, 1, s.Length); + fixed (char* p = buffer) + { + return (int)(f(p + 1, s.Length) - (p + 1)); + } + } + + private static void Check(ToWellFormedFunction f, FirstInvalidFunction first, string s) + { + string expected = Reference(s); + Assert.Equal(expected, Run(f, s, false)); + Assert.Equal(expected, Run(f, s, true)); + Assert.Equal(ReferenceFirstInvalid(s), RunFirst(first, s)); + } + + private static void HardCodedTest(ToWellFormedFunction f, FirstInvalidFunction first) + { + foreach (string s in HardCoded) + { + Check(f, first, s); + } + } + + private static void RandomTest(ToWellFormedFunction f, FirstInvalidFunction first) + { + var rand = new Random(1234); + for (int length = 0; length <= 200; length++) + { + for (int trial = 0; trial < 20; trial++) + { + Check(f, first, RandomString(rand, length, trial % 4)); + } + } + foreach (int length in new[] { 511, 512, 513, 1000, 4096, 10007 }) + { + for (int trial = 0; trial < 20; trial++) + { + Check(f, first, RandomString(rand, length, trial % 3)); + } + } + } + + // Well-formed input with a single lone surrogate at every possible position. + private static void SingleErrorTest(ToWellFormedFunction f, FirstInvalidFunction first) + { + var rand = new Random(4321); + foreach (int length in new[] { 1, 2, 17, 31, 32, 33, 34, 63, 64, 65, 66, 97, 130 }) + { + string valid = RandomString(rand, length, 0); + // RandomString may cut a pair in half at the end. + valid = Reference(valid); + for (int pos = 0; pos <= length; pos++) + { + foreach (char bad in new[] { '\uD800', '\uDBFF', '\uDC00', '\uDFFF' }) + { + Check(f, first, valid.Insert(pos, bad.ToString())); + } + } + } + } + + // Mostly surrogate-free input (the SIMD fast path) with a single valid pair, lone + // surrogate or pair followed by a lone surrogate at every position. + private static void SparseTest(ToWellFormedFunction f, FirstInvalidFunction first) + { + var rand = new Random(777); + foreach (int length in Enumerable.Range(0, 71).Concat(new[] { 300, 520 })) + { + var sb = new StringBuilder(length); + for (int k = 0; k < length; k++) + { + sb.Append(rand.Next(2) == 0 ? (char)rand.Next(0x80) : (char)rand.Next(0xE000, 0x10000)); + } + string valid = sb.ToString(); + foreach (string insert in new[] { "😀", "\uD83D", "\uDE00", "😀\uDE00", "\uD83D😀" }) + { + for (int pos = 0; pos <= length; pos++) + { + Check(f, first, valid.Insert(pos, insert)); + } + } + } + } + + [Fact] + [Trait("Category", "scalar")] + public void SparseScalar() => SparseTest(UTF16.ToWellFormedScalar, UTF16.GetPointerToFirstInvalidCharScalar); + + [FactOnAvx512] + [Trait("Category", "avx512")] + public void SparseAvx512() => SparseTest(UTF16.ToWellFormedAvx512, UTF16.GetPointerToFirstInvalidCharAvx512); + + [Fact] + public void SparseDefault() => SparseTest(UTF16.ToWellFormed, UTF16.GetPointerToFirstInvalidChar); + + [Fact] + [Trait("Category", "scalar")] + public void HardCodedScalar() => HardCodedTest(UTF16.ToWellFormedScalar, UTF16.GetPointerToFirstInvalidCharScalar); + + [Fact] + [Trait("Category", "scalar")] + public void RandomScalar() => RandomTest(UTF16.ToWellFormedScalar, UTF16.GetPointerToFirstInvalidCharScalar); + + [Fact] + [Trait("Category", "scalar")] + public void SingleErrorScalar() => SingleErrorTest(UTF16.ToWellFormedScalar, UTF16.GetPointerToFirstInvalidCharScalar); + + [FactOnAvx512] + [Trait("Category", "avx512")] + public void HardCodedAvx512() => HardCodedTest(UTF16.ToWellFormedAvx512, UTF16.GetPointerToFirstInvalidCharAvx512); + + [FactOnAvx512] + [Trait("Category", "avx512")] + public void RandomAvx512() => RandomTest(UTF16.ToWellFormedAvx512, UTF16.GetPointerToFirstInvalidCharAvx512); + + [FactOnAvx512] + [Trait("Category", "avx512")] + public void SingleErrorAvx512() => SingleErrorTest(UTF16.ToWellFormedAvx512, UTF16.GetPointerToFirstInvalidCharAvx512); + + [FactOnSse41] + [Trait("Category", "sse")] + public void HardCodedSse() => HardCodedTest(UTF16.ToWellFormedSse, UTF16.GetPointerToFirstInvalidCharSse); + + [FactOnSse41] + [Trait("Category", "sse")] + public void RandomSse() => RandomTest(UTF16.ToWellFormedSse, UTF16.GetPointerToFirstInvalidCharSse); + + [FactOnSse41] + [Trait("Category", "sse")] + public void SingleErrorSse() => SingleErrorTest(UTF16.ToWellFormedSse, UTF16.GetPointerToFirstInvalidCharSse); + + [FactOnSse41] + [Trait("Category", "sse")] + public void SparseSse() => SparseTest(UTF16.ToWellFormedSse, UTF16.GetPointerToFirstInvalidCharSse); + + [FactOnAvx2] + [Trait("Category", "avx")] + public void HardCodedAvx2() => HardCodedTest(UTF16.ToWellFormedAvx2, UTF16.GetPointerToFirstInvalidCharAvx2); + + [FactOnAvx2] + [Trait("Category", "avx")] + public void RandomAvx2() => RandomTest(UTF16.ToWellFormedAvx2, UTF16.GetPointerToFirstInvalidCharAvx2); + + [FactOnAvx2] + [Trait("Category", "avx")] + public void SingleErrorAvx2() => SingleErrorTest(UTF16.ToWellFormedAvx2, UTF16.GetPointerToFirstInvalidCharAvx2); + + [FactOnAvx2] + [Trait("Category", "avx")] + public void SparseAvx2() => SparseTest(UTF16.ToWellFormedAvx2, UTF16.GetPointerToFirstInvalidCharAvx2); + + [FactOnArm64] + [Trait("Category", "arm64")] + public void HardCodedArm64() => HardCodedTest(UTF16.ToWellFormedArm64, UTF16.GetPointerToFirstInvalidCharArm64); + + [FactOnArm64] + [Trait("Category", "arm64")] + public void RandomArm64() => RandomTest(UTF16.ToWellFormedArm64, UTF16.GetPointerToFirstInvalidCharArm64); + + [FactOnArm64] + [Trait("Category", "arm64")] + public void SingleErrorArm64() => SingleErrorTest(UTF16.ToWellFormedArm64, UTF16.GetPointerToFirstInvalidCharArm64); + + [FactOnArm64] + [Trait("Category", "arm64")] + public void SparseArm64() => SparseTest(UTF16.ToWellFormedArm64, UTF16.GetPointerToFirstInvalidCharArm64); + + // The dispatching entry points (AVX-512 or the IndexOfAnyInRange fallback). + [Fact] + public void HardCodedDefault() => HardCodedTest(UTF16.ToWellFormed, UTF16.GetPointerToFirstInvalidChar); + + [Fact] + public void RandomDefault() => RandomTest(UTF16.ToWellFormed, UTF16.GetPointerToFirstInvalidChar); + + [Fact] + public void SingleErrorDefault() => SingleErrorTest(UTF16.ToWellFormed, UTF16.GetPointerToFirstInvalidChar); + + [Fact] + public void StringApi() + { + var rand = new Random(99); + for (int length = 0; length <= 300; length += 7) + { + for (int trial = 0; trial < 10; trial++) + { + string s = RandomString(rand, length, trial % 3); + string expected = Reference(s); + string actual = UTF16.ToWellFormed(s); + Assert.Equal(expected, actual); + Assert.Equal(expected == s, UTF16.IsWellFormed(s)); + if (UTF16.IsWellFormed(s)) + { + Assert.Same(s, actual); // no allocation for well-formed strings + } + } + } + Assert.Throws(() => UTF16.ToWellFormed((string)null!)); + } + + [Fact] + public void SpanApi() + { + string s = "ab\uD800cd\uDC00😀"; + char[] dst = new char[s.Length]; + UTF16.ToWellFormed(s.AsSpan(), dst); + Assert.Equal(Reference(s), new string(dst)); + + char[] inplace = s.ToCharArray(); + UTF16.ToWellFormed(inplace, inplace); + Assert.Equal(Reference(s), new string(inplace)); + + Assert.Throws(() => UTF16.ToWellFormed(s.AsSpan(), new char[s.Length - 1])); + char[] overlap = new char[s.Length + 1]; + Assert.Throws(() => UTF16.ToWellFormed(overlap.AsSpan(0, s.Length), overlap.AsSpan(1))); + } +} From 9e7c4c914daa9ca5263a7274718edcbbde031f21 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sat, 26 Sep 2026 12:04:37 -0400 Subject: [PATCH 2/2] Clarify IsWellFormed in the README; document and test the IndexOfAnyInRange fallback. The fallback runs only when no SIMD kernel applies (no ARM64 NEON, AVX-512, AVX2 or SSE4.1), so the tests now call it directly on every system. --- README.md | 4 ++-- src/UTF16.cs | 13 +++++++++---- test/UTF16WellFormedTests.cs | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bec8326..7077455 100644 --- a/README.md +++ b/README.md @@ -234,8 +234,8 @@ faster than the standard library. .NET strings may contain lone surrogates. `SimdUnicode.UTF16.ToWellFormed` replaces each lone surrogate by the replacement character U+FFFD, like JavaScript's -`String.prototype.toWellFormed()`, and `SimdUnicode.UTF16.IsWellFormed` checks whether -there is any (like `isWellFormed()`). +`String.prototype.toWellFormed()`, and `SimdUnicode.UTF16.IsWellFormed` returns true when +there is no lone surrogate (like `isWellFormed()`). ```cs string s = UTF16.ToWellFormed("ab\uD800cd"); // "ab\uFFFDcd" diff --git a/src/UTF16.cs b/src/UTF16.cs index 24ed2eb..3fdef27 100644 --- a/src/UTF16.cs +++ b/src/UTF16.cs @@ -215,9 +215,11 @@ private unsafe static void ToWellFormedScalarFrom(ushort* pInputBuffer, int inpu return pInputBuffer + inputLength; } - // Without AVX-512, we rely on the runtime's vectorized IndexOfAnyInRange to skip - // everything that is not a surrogate. - private unsafe static char* GetPointerToFirstInvalidCharFallback(char* pInputBuffer, int inputLength) + /// + /// Portable version used when no SIMD kernel applies (no ARM64 NEON, AVX-512, AVX2 or SSE4.1): + /// we rely on the runtime's vectorized IndexOfAnyInRange to skip everything that is not a surrogate. + /// + public unsafe static char* GetPointerToFirstInvalidCharFallback(char* pInputBuffer, int inputLength) { ReadOnlySpan s = new ReadOnlySpan(pInputBuffer, inputLength); int i = 0; @@ -240,7 +242,10 @@ private unsafe static void ToWellFormedScalarFrom(ushort* pInputBuffer, int inpu } } - private unsafe static void ToWellFormedFallback(char* pInputBuffer, int inputLength, char* pOutputBuffer) + /// + /// Portable version used when no SIMD kernel applies (see ). + /// + public unsafe static void ToWellFormedFallback(char* pInputBuffer, int inputLength, char* pOutputBuffer) { if (pInputBuffer != pOutputBuffer) { diff --git a/test/UTF16WellFormedTests.cs b/test/UTF16WellFormedTests.cs index 5062965..29ef603 100644 --- a/test/UTF16WellFormedTests.cs +++ b/test/UTF16WellFormedTests.cs @@ -348,6 +348,24 @@ private static void SparseTest(ToWellFormedFunction f, FirstInvalidFunction firs [Trait("Category", "arm64")] public void SparseArm64() => SparseTest(UTF16.ToWellFormedArm64, UTF16.GetPointerToFirstInvalidCharArm64); + // The IndexOfAnyInRange fallback, used only when no SIMD kernel applies: we call it + // directly so that it is tested on every system. + [Fact] + [Trait("Category", "scalar")] + public void HardCodedFallback() => HardCodedTest(UTF16.ToWellFormedFallback, UTF16.GetPointerToFirstInvalidCharFallback); + + [Fact] + [Trait("Category", "scalar")] + public void RandomFallback() => RandomTest(UTF16.ToWellFormedFallback, UTF16.GetPointerToFirstInvalidCharFallback); + + [Fact] + [Trait("Category", "scalar")] + public void SingleErrorFallback() => SingleErrorTest(UTF16.ToWellFormedFallback, UTF16.GetPointerToFirstInvalidCharFallback); + + [Fact] + [Trait("Category", "scalar")] + public void SparseFallback() => SparseTest(UTF16.ToWellFormedFallback, UTF16.GetPointerToFirstInvalidCharFallback); + // The dispatching entry points (AVX-512 or the IndexOfAnyInRange fallback). [Fact] public void HardCodedDefault() => HardCodedTest(UTF16.ToWellFormed, UTF16.GetPointerToFirstInvalidChar);