fix: preserve whitespace-only cells in the XLS reader

src/Data.Common/Utils/CsvWhitespaceGuard.cs

@@ -0,0 +1,17 @@
+namespace Data.Common.Utils;
+
+/// <summary>
+/// Protocol constant shared between a CSV-producing provider (e.g. the XLS provider, which converts a
+/// sheet to CSV) and the CSV reader.
+///
+/// CSV parsing goes through <c>DataFrame.LoadCsv</c>, which is backed by
+/// <c>Microsoft.VisualBasic.FileIO.TextFieldParser</c> whose <c>TrimWhiteSpace</c> is <c>true</c> and is
+/// not configurable. That trims a whitespace-only cell down to an empty string. A provider that must
+/// preserve such whitespace wraps the value in this guard on both ends (so the field's edges are
+/// non-whitespace and survive parsing); the reader strips the guard afterwards. A Unicode private-use
+/// character is used so it will not collide with real content.
+/// </summary>
+public static class CsvWhitespaceGuard
+{
+    public const char Sentinel = (char)0xE000;
+}

src/Data.Csv/Utils/CsvVirtualDataTable.cs

@@ -1,4 +1,5 @@
 using Microsoft.Data.Analysis;
+using Data.Common.Utils;
 using Data.Common.Utils.ConnectionString;
 using SqlBuildingBlocks.POCOs;
 using System.Text;
@@ -17,6 +18,7 @@ public class CsvVirtualDataTable : VirtualDataTable, IDisposable
     private readonly int _guessTypeRows;
     private readonly FloatingPointDataType _preferredFloatingPointDataType;
     private readonly Func<IEnumerable<string>, Type> _guessTypeFunction;
+    private readonly bool _stripWhitespaceGuard;
 
     // Hold on to the underlying stream so we can keep paging through it.
     private readonly CsvTransformStream _transformStream;
@@ -49,13 +51,15 @@ public CsvVirtualDataTable(
             int guessTypeRows,
             FloatingPointDataType preferredFloatingPointDataType,
             Func<IEnumerable<string>, Type> guessTypeFunction,
-            char separator
+            char separator,
+            bool stripWhitespaceGuard = false
         )
         : base(tableName)
     {
         _pageSize = pageSize;
         _guessTypeRows = guessTypeRows > 0 ? guessTypeRows : throw new ArgumentOutOfRangeException(nameof(guessTypeRows), $"Guess type row must be greater than 0.  GuessRows: {guessTypeRows}");
         _preferredFloatingPointDataType = preferredFloatingPointDataType;
+        _stripWhitespaceGuard = stripWhitespaceGuard;
         _guessTypeFunction = guessTypeFunction;
 
         // Instead of using "using", store the reader and transform stream for later use.
@@ -165,13 +169,35 @@ private IEnumerable<DataRow> ToDataRows(DataFrame dataFrame)
             DataRow newRow = NewRow();
             for (int colIndex = 0; colIndex < Columns.Count; colIndex++)
             {
-                var value = dataFrame.Columns[colIndex][rowIndex] ?? DBNull.Value;
+                var value = StripWhitespaceGuard(dataFrame.Columns[colIndex][rowIndex]) ?? DBNull.Value;
                 newRow[colIndex] = value;
             }
             yield return newRow;
         }
     }
 
+    // Removes the whitespace guard added by a producer (see CsvWhitespaceGuard). Only runs when this
+    // reader was told the source can contain guards (the XLS provider) — for other providers a value that
+    // merely looks guard-shaped is real data and must be returned untouched.
+    private object StripWhitespaceGuard(object value)
+    {
+        if (_stripWhitespaceGuard && value is string text && IsGuardedWhitespace(text))
+        {
+            return text.Substring(1, text.Length - 2);
+        }
+
+        return value;
+    }
+
+    // True when the value is exactly the guard sentinel + whitespace-only content + the guard sentinel —
+    // i.e. only what the producer would have emitted for a whitespace-only cell.
+    private static bool IsGuardedWhitespace(string value) =>
+        value != null
+        && value.Length >= 3
+        && value[0] == CsvWhitespaceGuard.Sentinel
+        && value[value.Length - 1] == CsvWhitespaceGuard.Sentinel
+        && string.IsNullOrWhiteSpace(value.Substring(1, value.Length - 2));
+
     public void Dispose()
     {
         if (!_disposed)

src/Data.Xls/Utils/CsvUtils.cs

@@ -1,3 +1,5 @@
+using Data.Common.Utils;
+
 namespace Data.Xls.Utils;
 
 public static class CsvUtils
@@ -6,13 +8,23 @@ public static IEnumerable<string> EscapeCsvValues(this IEnumerable<string> value
     {
         foreach (var value in values)
         {
-            if (value.Contains(",") || value.Contains("\""))
+            // A whitespace-only cell would be trimmed to an empty string by DataFrame's CSV reader
+            // (TextFieldParser.TrimWhiteSpace). Wrap it in a guard character on both ends so the field's
+            // edges are non-whitespace and survive parsing; CsvVirtualDataTable strips the guard afterwards.
+            //
+            // Scope: only *fully* whitespace-only cells are guarded. Leading/trailing whitespace on
+            // otherwise-non-empty text (e.g. " x ") is still trimmed by the reader and is not preserved.
+            var guarded = value.Length > 0 && string.IsNullOrWhiteSpace(value)
+                ? CsvWhitespaceGuard.Sentinel + value + CsvWhitespaceGuard.Sentinel
+                : value;
+
+            if (guarded.Contains(",") || guarded.Contains("\""))
             {
-                yield return $"\"{value.Replace("\"", "\"\"")}\"";
+                yield return $"\"{guarded.Replace("\"", "\"\"")}\"";
             }
             else
             {
-                yield return value;
+                yield return guarded;
             }
         }
     }

src/Data.Xls/XlsIO/Read/XlsReader.cs

@@ -57,7 +57,7 @@ private VirtualDataTable PrepareDataTable(StreamReader streamReader, string tabl
         char separator = ',';  //Since the XlsSheetStream composed the stream as comma separated, we are ensured that the separator is a comma and don't need to detect it.
         CsvVirtualDataTable virtualDataTable = new(streamReader, tableName, pageSize, xlsConnection.GuessTypeRows,
                                                    fileConnection.PreferredFloatingPointDataType, xlsConnection.GuessTypeFunction,
-                                                   separator);
+                                                   separator, stripWhitespaceGuard: true);
 
         return virtualDataTable;
     }

tests/Data.Csv.Tests/CsvVirtualDataTableTests.cs

@@ -0,0 +1,36 @@
+using System.IO;
+using System.Linq;
+using System.Text;
+using Data.Common.Utils;
+using Data.Common.Utils.ConnectionString;
+using Data.Csv.Utils;
+using Xunit;
+
+namespace Data.Csv.Tests;
+
+public class CsvVirtualDataTableTests
+{
+    [Fact]
+    public void GuardShapedValue_WhenGuardStrippingDisabled_IsReturnedUnchanged()
+    {
+        // A value that merely looks like a whitespace guard (sentinel + whitespace + sentinel) must NOT
+        // be altered by a reader whose source does not use guards. Guard stripping is opt-in via
+        // stripWhitespaceGuard (used only by the XLS provider, which writes the guards); every other
+        // provider must leave such data exactly as-is.
+        var guard = CsvWhitespaceGuard.Sentinel;
+        var guardShaped = $"{guard}  {guard}";
+        var csv = $"Value,Marker\n{guardShaped},guarded\nnormal,control\n";
+
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(csv));
+        using var reader = new StreamReader(stream);
+        using var table = new CsvVirtualDataTable(
+            reader, "guardShaped", pageSize: 4096, guessTypeRows: 1000,
+            FloatingPointDataType.Double, TypeGuesser.GuessType, separator: ',',
+            stripWhitespaceGuard: false);
+
+        var rows = table.Rows!.ToList();
+        var guardedRow = rows.Single(row => (string)row["Marker"] == "guarded");
+
+        Assert.Equal(guardShaped, (string)guardedRow["Value"]);
+    }
+}

tests/Data.Tests.Common/DataReaderTests.cs

@@ -239,7 +239,80 @@ public static void Reader_ShouldReadEmptyCells<TFileParameter>(
 
         connection.Close();
     }
-
+
+    // NOTE: whitespace-only preservation is currently an XLS-provider behavior (only the XLS write path
+    // guards whitespace cells). The table name is parameterized so the fixture/sheet is not assumed.
+    public static void Reader_ShouldPreserveWhitespaceOnlyCell<TFileParameter>(
+        Func<FileConnection<TFileParameter>> createFileConnection, string tableName)
+        where TFileParameter : FileParameter<TFileParameter>, new()
+    {
+        // Arrange
+        var connection = createFileConnection();
+        var command = connection.CreateCommand($"SELECT * FROM [{tableName}]");
+
+        // Act & Assert
+        connection.Open();
+        var found = false;
+        using (var reader = command.ExecuteReader())
+        {
+            while (reader.Read())
+            {
+                if (reader["Marker"].ToString() != "spaces")
+                {
+                    continue;
+                }
+
+                // A cell containing only whitespace must be read back with the whitespace intact,
+                // not trimmed away.
+                found = true;
+                Assert.IsType<string>(reader["Value"]);
+                Assert.Equal("  ", reader["Value"]);
+            }
+        }
+        connection.Close();
+
+        Assert.True(found, "The 'spaces' row was not returned by the reader.");
+    }
+
+    // Preserving a whitespace-only cell as text has a consequence for a column that also contains
+    // numbers: that column is read as text, because the cell cannot be both whitespace and a number.
+    // The whitespace is preserved and the numeric cells come back in their string form.
+    public static void Reader_WhitespaceCellInNumericLikeColumn_IsPreservedAsText<TFileParameter>(
+        Func<FileConnection<TFileParameter>> createFileConnection, string tableName)
+        where TFileParameter : FileParameter<TFileParameter>, new()
+    {
+        // Arrange
+        var connection = createFileConnection();
+        var command = connection.CreateCommand($"SELECT * FROM [{tableName}]");
+
+        // Act & Assert
+        connection.Open();
+        var seenMarkers = new List<string>();
+        using (var reader = command.ExecuteReader())
+        {
+            while (reader.Read())
+            {
+                var marker = reader["Marker"].ToString() ?? string.Empty;
+                seenMarkers.Add(marker);
+                switch (marker)
+                {
+                    case "num1":
+                        Assert.Equal("10", reader["Amount"]);
+                        break;
+                    case "num2":
+                        Assert.Equal("20", reader["Amount"]);
+                        break;
+                    case "spaces":
+                        Assert.Equal("  ", reader["Amount"]);
+                        break;
+                }
+            }
+        }
+        connection.Close();
+
+        Assert.Equal(new[] { "num1", "spaces", "num2" }.OrderBy(m => m), seenMarkers.OrderBy(m => m));
+    }
+
     public static void Reader_ShouldReadFormulasAsString<TFileParameter>(Func<FileConnection<TFileParameter>> createFileConnection)
         where TFileParameter : FileParameter<TFileParameter>, new()
     {

tests/Data.Tests.Common/Utils/DatabaseFullPaths.cs

@@ -20,6 +20,8 @@ public DatabaseFullPaths(string extension)
     public string WithFormula => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"WithFormula.{extension}");
     public string EmptyCells => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"emptyCells.{extension}");
     public string CellsWithComma => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"cellsWithComma.{extension}");
+    public string WithSpaceCell => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"withSpaceCell.{extension}");
+    public string WithSpaceCellMixed => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"withSpaceCellMixed.{extension}");
     public string eComFileDataBase => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"ecommerce.{extension}");
     public string eComFolderDataBase => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"eCom");
     public string FolderEmptyWithTables => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, "EmptyDatabase");

tests/Data.Xls.Tests/Data.Xls.Tests.csproj

@@ -58,5 +58,11 @@
     <None Update="Sources\emptyCells.xlsx">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="Sources\withSpaceCell.xlsx">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Sources\withSpaceCellMixed.xlsx">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 </Project>

tests/Data.Xls.Tests/FileAsDatabase/XlsDataReaderTests.cs

@@ -64,6 +64,20 @@ public void Reader_ShouldReadEmptyCells()
         DataReaderTests.Reader_ShouldReadEmptyCells(() => new XlsConnection(ConnectionStrings.Instance.EmptyCellsAsDB));
     }
 
+    [Fact]
+    public void Reader_ShouldPreserveWhitespaceOnlyCell()
+    {
+        DataReaderTests.Reader_ShouldPreserveWhitespaceOnlyCell(() =>
+            new XlsConnection(ConnectionStrings.Instance.WithSpaceCellAsDB), "Sheet1");
+    }
+
+    [Fact]
+    public void Reader_WhitespaceCellInNumericLikeColumn_IsPreservedAsText()
+    {
+        DataReaderTests.Reader_WhitespaceCellInNumericLikeColumn_IsPreservedAsText(() =>
+            new XlsConnection(ConnectionStrings.Instance.WithSpaceCellMixedAsDB), "Sheet1");
+    }
+
     [Fact]
     public void Reader_Limit_ShouldReturnOnlyFirstRow()
     {

tests/Data.Xls.Tests/Utils/ConnectionStrings.cs

@@ -1,10 +1,15 @@
-using Data.Tests.Common.Utils;
+using Data.Common.Utils.ConnectionString;
+using Data.Tests.Common.Utils;
 
 namespace Data.Xls.Tests;
 
 public class ConnectionStrings : ConnectionStringsBase
 {
     public override string Extension => "xlsx";
 
+    public FileConnectionString WithSpaceCellAsDB => new FileConnectionString { DataSource = Database.WithSpaceCell };
+
+    public FileConnectionString WithSpaceCellMixedAsDB => new FileConnectionString { DataSource = Database.WithSpaceCellMixed };
+
     public new static ConnectionStrings Instance => new ConnectionStrings();
 }