diff --git a/src/Data.Common/Utils/CsvWhitespaceGuard.cs b/src/Data.Common/Utils/CsvWhitespaceGuard.cs new file mode 100644 index 0000000..923fb50 --- /dev/null +++ b/src/Data.Common/Utils/CsvWhitespaceGuard.cs @@ -0,0 +1,17 @@ +namespace Data.Common.Utils; + +/// +/// Protocol constant shared between a CSV-producing provider (e.g. the XLS provider, which converts a +/// sheet to CSV) and the CSV reader. +/// +/// CSV parsing goes through DataFrame.LoadCsv, which is backed by +/// Microsoft.VisualBasic.FileIO.TextFieldParser whose TrimWhiteSpace is true and is +/// not configurable. That trims a whitespace-only cell down to an empty string. A provider that must +/// preserve such whitespace wraps the value in this guard on both ends (so the field's edges are +/// non-whitespace and survive parsing); the reader strips the guard afterwards. A Unicode private-use +/// character is used so it will not collide with real content. +/// +public static class CsvWhitespaceGuard +{ + public const char Sentinel = (char)0xE000; +} diff --git a/src/Data.Csv/Utils/CsvVirtualDataTable.cs b/src/Data.Csv/Utils/CsvVirtualDataTable.cs index 4b6915a..5278ff7 100644 --- a/src/Data.Csv/Utils/CsvVirtualDataTable.cs +++ b/src/Data.Csv/Utils/CsvVirtualDataTable.cs @@ -1,4 +1,5 @@ using Microsoft.Data.Analysis; +using Data.Common.Utils; using Data.Common.Utils.ConnectionString; using SqlBuildingBlocks.POCOs; using System.Text; @@ -17,6 +18,7 @@ public class CsvVirtualDataTable : VirtualDataTable, IDisposable private readonly int _guessTypeRows; private readonly FloatingPointDataType _preferredFloatingPointDataType; private readonly Func, Type> _guessTypeFunction; + private readonly bool _stripWhitespaceGuard; // Hold on to the underlying stream so we can keep paging through it. private readonly CsvTransformStream _transformStream; @@ -49,13 +51,15 @@ public CsvVirtualDataTable( int guessTypeRows, FloatingPointDataType preferredFloatingPointDataType, Func, Type> guessTypeFunction, - char separator + char separator, + bool stripWhitespaceGuard = false ) : base(tableName) { _pageSize = pageSize; _guessTypeRows = guessTypeRows > 0 ? guessTypeRows : throw new ArgumentOutOfRangeException(nameof(guessTypeRows), $"Guess type row must be greater than 0. GuessRows: {guessTypeRows}"); _preferredFloatingPointDataType = preferredFloatingPointDataType; + _stripWhitespaceGuard = stripWhitespaceGuard; _guessTypeFunction = guessTypeFunction; // Instead of using "using", store the reader and transform stream for later use. @@ -165,13 +169,35 @@ private IEnumerable ToDataRows(DataFrame dataFrame) DataRow newRow = NewRow(); for (int colIndex = 0; colIndex < Columns.Count; colIndex++) { - var value = dataFrame.Columns[colIndex][rowIndex] ?? DBNull.Value; + var value = StripWhitespaceGuard(dataFrame.Columns[colIndex][rowIndex]) ?? DBNull.Value; newRow[colIndex] = value; } yield return newRow; } } + // Removes the whitespace guard added by a producer (see CsvWhitespaceGuard). Only runs when this + // reader was told the source can contain guards (the XLS provider) — for other providers a value that + // merely looks guard-shaped is real data and must be returned untouched. + private object StripWhitespaceGuard(object value) + { + if (_stripWhitespaceGuard && value is string text && IsGuardedWhitespace(text)) + { + return text.Substring(1, text.Length - 2); + } + + return value; + } + + // True when the value is exactly the guard sentinel + whitespace-only content + the guard sentinel — + // i.e. only what the producer would have emitted for a whitespace-only cell. + private static bool IsGuardedWhitespace(string value) => + value != null + && value.Length >= 3 + && value[0] == CsvWhitespaceGuard.Sentinel + && value[value.Length - 1] == CsvWhitespaceGuard.Sentinel + && string.IsNullOrWhiteSpace(value.Substring(1, value.Length - 2)); + public void Dispose() { if (!_disposed) diff --git a/src/Data.Xls/Utils/CsvUtils.cs b/src/Data.Xls/Utils/CsvUtils.cs index 1a1b517..d305b5a 100644 --- a/src/Data.Xls/Utils/CsvUtils.cs +++ b/src/Data.Xls/Utils/CsvUtils.cs @@ -1,3 +1,5 @@ +using Data.Common.Utils; + namespace Data.Xls.Utils; public static class CsvUtils @@ -6,13 +8,23 @@ public static IEnumerable EscapeCsvValues(this IEnumerable value { foreach (var value in values) { - if (value.Contains(",") || value.Contains("\"")) + // A whitespace-only cell would be trimmed to an empty string by DataFrame's CSV reader + // (TextFieldParser.TrimWhiteSpace). Wrap it in a guard character on both ends so the field's + // edges are non-whitespace and survive parsing; CsvVirtualDataTable strips the guard afterwards. + // + // Scope: only *fully* whitespace-only cells are guarded. Leading/trailing whitespace on + // otherwise-non-empty text (e.g. " x ") is still trimmed by the reader and is not preserved. + var guarded = value.Length > 0 && string.IsNullOrWhiteSpace(value) + ? CsvWhitespaceGuard.Sentinel + value + CsvWhitespaceGuard.Sentinel + : value; + + if (guarded.Contains(",") || guarded.Contains("\"")) { - yield return $"\"{value.Replace("\"", "\"\"")}\""; + yield return $"\"{guarded.Replace("\"", "\"\"")}\""; } else { - yield return value; + yield return guarded; } } } diff --git a/src/Data.Xls/XlsIO/Read/XlsReader.cs b/src/Data.Xls/XlsIO/Read/XlsReader.cs index 8e70234..6f13f6e 100644 --- a/src/Data.Xls/XlsIO/Read/XlsReader.cs +++ b/src/Data.Xls/XlsIO/Read/XlsReader.cs @@ -57,7 +57,7 @@ private VirtualDataTable PrepareDataTable(StreamReader streamReader, string tabl char separator = ','; //Since the XlsSheetStream composed the stream as comma separated, we are ensured that the separator is a comma and don't need to detect it. CsvVirtualDataTable virtualDataTable = new(streamReader, tableName, pageSize, xlsConnection.GuessTypeRows, fileConnection.PreferredFloatingPointDataType, xlsConnection.GuessTypeFunction, - separator); + separator, stripWhitespaceGuard: true); return virtualDataTable; } diff --git a/tests/Data.Csv.Tests/CsvVirtualDataTableTests.cs b/tests/Data.Csv.Tests/CsvVirtualDataTableTests.cs new file mode 100644 index 0000000..4d7aede --- /dev/null +++ b/tests/Data.Csv.Tests/CsvVirtualDataTableTests.cs @@ -0,0 +1,36 @@ +using System.IO; +using System.Linq; +using System.Text; +using Data.Common.Utils; +using Data.Common.Utils.ConnectionString; +using Data.Csv.Utils; +using Xunit; + +namespace Data.Csv.Tests; + +public class CsvVirtualDataTableTests +{ + [Fact] + public void GuardShapedValue_WhenGuardStrippingDisabled_IsReturnedUnchanged() + { + // A value that merely looks like a whitespace guard (sentinel + whitespace + sentinel) must NOT + // be altered by a reader whose source does not use guards. Guard stripping is opt-in via + // stripWhitespaceGuard (used only by the XLS provider, which writes the guards); every other + // provider must leave such data exactly as-is. + var guard = CsvWhitespaceGuard.Sentinel; + var guardShaped = $"{guard} {guard}"; + var csv = $"Value,Marker\n{guardShaped},guarded\nnormal,control\n"; + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(csv)); + using var reader = new StreamReader(stream); + using var table = new CsvVirtualDataTable( + reader, "guardShaped", pageSize: 4096, guessTypeRows: 1000, + FloatingPointDataType.Double, TypeGuesser.GuessType, separator: ',', + stripWhitespaceGuard: false); + + var rows = table.Rows!.ToList(); + var guardedRow = rows.Single(row => (string)row["Marker"] == "guarded"); + + Assert.Equal(guardShaped, (string)guardedRow["Value"]); + } +} diff --git a/tests/Data.Tests.Common/DataReaderTests.cs b/tests/Data.Tests.Common/DataReaderTests.cs index 12d60ab..8a20b46 100644 --- a/tests/Data.Tests.Common/DataReaderTests.cs +++ b/tests/Data.Tests.Common/DataReaderTests.cs @@ -239,7 +239,80 @@ public static void Reader_ShouldReadEmptyCells( connection.Close(); } - + + // NOTE: whitespace-only preservation is currently an XLS-provider behavior (only the XLS write path + // guards whitespace cells). The table name is parameterized so the fixture/sheet is not assumed. + public static void Reader_ShouldPreserveWhitespaceOnlyCell( + Func> createFileConnection, string tableName) + where TFileParameter : FileParameter, new() + { + // Arrange + var connection = createFileConnection(); + var command = connection.CreateCommand($"SELECT * FROM [{tableName}]"); + + // Act & Assert + connection.Open(); + var found = false; + using (var reader = command.ExecuteReader()) + { + while (reader.Read()) + { + if (reader["Marker"].ToString() != "spaces") + { + continue; + } + + // A cell containing only whitespace must be read back with the whitespace intact, + // not trimmed away. + found = true; + Assert.IsType(reader["Value"]); + Assert.Equal(" ", reader["Value"]); + } + } + connection.Close(); + + Assert.True(found, "The 'spaces' row was not returned by the reader."); + } + + // Preserving a whitespace-only cell as text has a consequence for a column that also contains + // numbers: that column is read as text, because the cell cannot be both whitespace and a number. + // The whitespace is preserved and the numeric cells come back in their string form. + public static void Reader_WhitespaceCellInNumericLikeColumn_IsPreservedAsText( + Func> createFileConnection, string tableName) + where TFileParameter : FileParameter, new() + { + // Arrange + var connection = createFileConnection(); + var command = connection.CreateCommand($"SELECT * FROM [{tableName}]"); + + // Act & Assert + connection.Open(); + var seenMarkers = new List(); + using (var reader = command.ExecuteReader()) + { + while (reader.Read()) + { + var marker = reader["Marker"].ToString() ?? string.Empty; + seenMarkers.Add(marker); + switch (marker) + { + case "num1": + Assert.Equal("10", reader["Amount"]); + break; + case "num2": + Assert.Equal("20", reader["Amount"]); + break; + case "spaces": + Assert.Equal(" ", reader["Amount"]); + break; + } + } + } + connection.Close(); + + Assert.Equal(new[] { "num1", "spaces", "num2" }.OrderBy(m => m), seenMarkers.OrderBy(m => m)); + } + public static void Reader_ShouldReadFormulasAsString(Func> createFileConnection) where TFileParameter : FileParameter, new() { diff --git a/tests/Data.Tests.Common/Utils/DatabaseFullPaths.cs b/tests/Data.Tests.Common/Utils/DatabaseFullPaths.cs index b27778d..d3831be 100644 --- a/tests/Data.Tests.Common/Utils/DatabaseFullPaths.cs +++ b/tests/Data.Tests.Common/Utils/DatabaseFullPaths.cs @@ -20,6 +20,8 @@ public DatabaseFullPaths(string extension) public string WithFormula => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"WithFormula.{extension}"); public string EmptyCells => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"emptyCells.{extension}"); public string CellsWithComma => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"cellsWithComma.{extension}"); + public string WithSpaceCell => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"withSpaceCell.{extension}"); + public string WithSpaceCellMixed => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"withSpaceCellMixed.{extension}"); public string eComFileDataBase => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"ecommerce.{extension}"); public string eComFolderDataBase => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, $"eCom"); public string FolderEmptyWithTables => Path.Combine(FileConnectionStringTestsExtensions.SourcesFolder, "EmptyDatabase"); diff --git a/tests/Data.Xls.Tests/Data.Xls.Tests.csproj b/tests/Data.Xls.Tests/Data.Xls.Tests.csproj index 4eac125..fcf5f1e 100644 --- a/tests/Data.Xls.Tests/Data.Xls.Tests.csproj +++ b/tests/Data.Xls.Tests/Data.Xls.Tests.csproj @@ -58,5 +58,11 @@ Always + + Always + + + Always + diff --git a/tests/Data.Xls.Tests/FileAsDatabase/XlsDataReaderTests.cs b/tests/Data.Xls.Tests/FileAsDatabase/XlsDataReaderTests.cs index 30cba6d..74dcfdd 100644 --- a/tests/Data.Xls.Tests/FileAsDatabase/XlsDataReaderTests.cs +++ b/tests/Data.Xls.Tests/FileAsDatabase/XlsDataReaderTests.cs @@ -64,6 +64,20 @@ public void Reader_ShouldReadEmptyCells() DataReaderTests.Reader_ShouldReadEmptyCells(() => new XlsConnection(ConnectionStrings.Instance.EmptyCellsAsDB)); } + [Fact] + public void Reader_ShouldPreserveWhitespaceOnlyCell() + { + DataReaderTests.Reader_ShouldPreserveWhitespaceOnlyCell(() => + new XlsConnection(ConnectionStrings.Instance.WithSpaceCellAsDB), "Sheet1"); + } + + [Fact] + public void Reader_WhitespaceCellInNumericLikeColumn_IsPreservedAsText() + { + DataReaderTests.Reader_WhitespaceCellInNumericLikeColumn_IsPreservedAsText(() => + new XlsConnection(ConnectionStrings.Instance.WithSpaceCellMixedAsDB), "Sheet1"); + } + [Fact] public void Reader_Limit_ShouldReturnOnlyFirstRow() { diff --git a/tests/Data.Xls.Tests/Sources/withSpaceCell.xlsx b/tests/Data.Xls.Tests/Sources/withSpaceCell.xlsx new file mode 100644 index 0000000..d79c434 Binary files /dev/null and b/tests/Data.Xls.Tests/Sources/withSpaceCell.xlsx differ diff --git a/tests/Data.Xls.Tests/Sources/withSpaceCellMixed.xlsx b/tests/Data.Xls.Tests/Sources/withSpaceCellMixed.xlsx new file mode 100644 index 0000000..0d4b426 Binary files /dev/null and b/tests/Data.Xls.Tests/Sources/withSpaceCellMixed.xlsx differ diff --git a/tests/Data.Xls.Tests/Utils/ConnectionStrings.cs b/tests/Data.Xls.Tests/Utils/ConnectionStrings.cs index d30901f..a73b117 100644 --- a/tests/Data.Xls.Tests/Utils/ConnectionStrings.cs +++ b/tests/Data.Xls.Tests/Utils/ConnectionStrings.cs @@ -1,4 +1,5 @@ -using Data.Tests.Common.Utils; +using Data.Common.Utils.ConnectionString; +using Data.Tests.Common.Utils; namespace Data.Xls.Tests; @@ -6,5 +7,9 @@ public class ConnectionStrings : ConnectionStringsBase { public override string Extension => "xlsx"; + public FileConnectionString WithSpaceCellAsDB => new FileConnectionString { DataSource = Database.WithSpaceCell }; + + public FileConnectionString WithSpaceCellMixedAsDB => new FileConnectionString { DataSource = Database.WithSpaceCellMixed }; + public new static ConnectionStrings Instance => new ConnectionStrings(); } \ No newline at end of file