From 144be3bc27b82edcd5eb7585353bd805582ff094 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Wed, 5 Aug 2026 16:55:03 +0300 Subject: [PATCH 01/12] Fix data validation startpoint issue with multi byte characters --- .../Commands/DataValidationBenchmark.cs | 12 +- .../DataTests/StreamUtilitiesTests.cs | 143 +++-------- .../DataValidationTests/DataValidationTest.cs | 8 +- .../StreamSyntaxValidationTests.cs | 26 ++ .../PxFile/Data/PxFileStreamDataReader.cs | 18 +- Px.Utils/PxFile/Data/StreamUtilities.cs | 222 ++++++++++++++---- .../DataValidation/DataValidator.cs | 69 +++--- .../SyntaxValidationResult.cs | 16 +- .../SyntaxValidation/SyntaxValidator.cs | 18 +- 9 files changed, 306 insertions(+), 226 deletions(-) diff --git a/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs index 47c02ad9..5f3a9065 100644 --- a/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs @@ -22,12 +22,8 @@ internal sealed class DataValidationBenchmark : FileBenchmark internal override string Description => "Benchmarks the data validation capabilities of the DataValidator."; private long start; - private const string dataKeyword = "DATA"; - private Encoding encoding; - private const int readStartOffset = 3; - internal DataValidationBenchmark() { BenchmarkFunctions = [ValidateDataBenchmarks]; @@ -44,17 +40,17 @@ protected override async Task OneTimeBenchmarkSetupAsync() using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); PxFileMetadataReader reader = new(); encoding = reader.GetEncoding(stream); - start = StreamUtilities.FindKeywordPosition(stream, dataKeyword, PxFileConfiguration.Default); + start = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default); if (start == -1) { - throw new ArgumentException($"Could not find data keyword '{dataKeyword}'"); + throw new ArgumentException("Could not find the first data value after 'DATA='"); } } private void ValidateDataBenchmarks() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); - stream.Position = start + dataKeyword.Length + readStartOffset; // skip the '=' and linechange + stream.Position = start; DataValidator validator = new(expectedCols, expectedRows, 0); validator.Validate(stream, TestFilePath, encoding); } @@ -62,7 +58,7 @@ private void ValidateDataBenchmarks() private async Task ValidateDataBenchmarksAsync() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); - stream.Position = start + dataKeyword.Length + readStartOffset; // skip the '=' and linechange + stream.Position = start; DataValidator validator = new(expectedCols, expectedRows, 0); await validator.ValidateAsync(stream, TestFilePath, encoding); diff --git a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs index a8715c13..0da963f7 100644 --- a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs +++ b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs @@ -1,4 +1,4 @@ -using PxFileTests.Fixtures; +using PxFileTests.Fixtures; using Px.Utils.PxFile; using Px.Utils.PxFile.Data; using System.Text; @@ -13,141 +13,66 @@ public class StreamUtilitiesTests */ [TestMethod] - public void FindKeywordTestDataKeywordAtStartOfStreamReturnsZero() + [DataRow("1")] + [DataRow("-1")] + [DataRow(".5")] + [DataRow("0")] + [DataRow("0.0")] + [DataRow("0.5")] + [DataRow("123.456")] + [DataRow("\".\"")] + [DataRow("\"..\"")] + public void FindDataStartPositionDataWithBomMultibyteMetadataAndWhitespaceReturnsFirstValueOffset(string firstValue) { // Arrange - byte[] data = Encoding.UTF8.GetBytes("DATA="); + string content = "TITLE=\"DATA=79_20180101;\";\r\nVALUES=\"Ää, Öö\";\r\nDATA=\t \r\n" + firstValue + " 2;"; + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + byte[] data = [.. bom, .. contentBytes]; using Stream stream = new MemoryStream(data); + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..(content.LastIndexOf("DATA=", StringComparison.Ordinal) + 5 + "\t \r\n".Length)]); // Act - long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default); + long position = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 3); // Assert - Assert.AreEqual(0, position); + Assert.AreEqual(expectedPosition, position); + Assert.AreEqual(0, stream.Position); + Assert.AreEqual(firstValue[0], (char)data[(int)position]); } [TestMethod] - public void FindKeywordTestTwoDataKeywordsReturnsNegative1() + public async Task FindDataStartPositionAsyncDataSplitAcrossBuffersReturnsSameOffset() { // Arrange - byte[] data = Encoding.UTF8.GetBytes("DATADATA="); + string content = "TITLE=\"Åland\";\nDATA=\r\n\t\".\" 2;"; + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + byte[] data = [.. bom, .. contentBytes]; using Stream stream = new MemoryStream(data); + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..content.IndexOf('"', content.IndexOf("DATA=", StringComparison.Ordinal) + 5)]); // Act - long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default); + long synchronousPosition = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionAsync(stream, PxFileConfiguration.Default, 2, System.Threading.CancellationToken.None); // Assert - Assert.AreEqual(-1, position); + Assert.AreEqual(expectedPosition, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.AreEqual(0, stream.Position); } [TestMethod] - public void FindKeywordTestDataKeywordInTheMiddleOfStreamReturnsIndex() + public void FindDataStartPositionDataWithoutValueReturnsNegative1() { // Arrange - byte[] data = Encoding.UTF8.GetBytes("KEYWORD=\"foo\";\nDATA=123 345"); - using Stream stream = new MemoryStream(data); + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes("TITLE=\"foo\";\nDATA=\r\n\t ")); // Act - long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default); - - // Assert - Assert.AreEqual(15, position); - } - - [TestMethod] - public void FindKeywordTestDataKeywordAtTheEndOfStreamReturnsIndex() - { - // Arrange - byte[] data = Encoding.UTF8.GetBytes("DADADADATA="); - using Stream stream = new MemoryStream(data); - - // Act - long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default); + long position = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 2); // Assert Assert.AreEqual(-1, position); } - - [TestMethod] - public void FindKeywordTestDKeywordInTheMiddleOfStreamReturnsIndex() - { - // Arrange - byte[] data = Encoding.UTF8.GetBytes("FFFFFF;D=AAAA"); - using Stream stream = new MemoryStream(data); - - // Act - long position = StreamUtilities.FindKeywordPosition(stream, "D", PxFileConfiguration.Default); - - // Assert - Assert.AreEqual(7, position); - } - - [TestMethod] - public void FindKeywordTestDATAKeywordInTheMiddleOfUtfFixtureStreamReturnsIndex() - { - string keyword = "DATA"; - - // Arrange - byte[] data = Encoding.UTF8.GetBytes(MinimalPx.MINIMAL_UTF8_N); - using Stream stream = new MemoryStream(data); - - // Act - long position = StreamUtilities.FindKeywordPosition(stream, keyword, PxFileConfiguration.Default); - string result = Encoding.ASCII.GetString(data, (int)position, keyword.Length); - - // Assert - Assert.AreEqual(keyword, result); - } - - [TestMethod] - public void FindKeywordTestDATAKeywordInTheMiddleOfAsciiFixtureStreamReturnsIndex() - { - string keyword = "DATA"; - - // Arrange - byte[] data = Encoding.UTF8.GetBytes(MinimalPx.MINIMAL_ISO_8859_15_N); - using Stream stream = new MemoryStream(data); - - // Act - long position = StreamUtilities.FindKeywordPosition(stream, keyword, PxFileConfiguration.Default); - string result = Encoding.ASCII.GetString(data, (int)position, keyword.Length); - - // Assert - Assert.AreEqual(keyword, result); - } - - [TestMethod] - public void FindKeywordTestDATAKeywordInTheMiddleOfAsciiFixtureStreamShortBufferSplitsKeywordReturnsIndex() - { - string keyword = "DATA"; - - // Arrange - byte[] data = Encoding.UTF8.GetBytes(MinimalPx.MINIMAL_ISO_8859_15_N); - using Stream stream = new MemoryStream(data); - - // Act - long position = StreamUtilities.FindKeywordPosition(stream, keyword, PxFileConfiguration.Default, 3); - string result = Encoding.ASCII.GetString(data, (int)position, keyword.Length); - - // Assert - Assert.AreEqual(keyword, result); - } - - [TestMethod] - public void FindKeywordTestDATAKeywordInTheMiddleOfDataFinderFixtureStreamReturnsIndex() - { - string keyword = "DATA"; - - // Arrange - byte[] data = Encoding.UTF8.GetBytes(MinimalPx.MINIMAL_UTF8_N_FOR_DATA_FINDER); - using Stream stream = new MemoryStream(data); - - // Act - long position = StreamUtilities.FindKeywordPosition(stream, keyword, PxFileConfiguration.Default); - string result = Encoding.ASCII.GetString(data, (int)position -1, keyword.Length+2); - - // Assert - Assert.AreEqual('\n' + keyword + '=', result); - } } } diff --git a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs index 66846a97..5813b4f0 100644 --- a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs +++ b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs @@ -18,8 +18,8 @@ public class DataValidationTest [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS, 0, 0)] [DataRow(DataStreamContents.SIMPLE_INVALID_DATA, 7, 12)] - [DataRow(DataStreamContents.NO_DATA, 2, 6, false)] - [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 2, 6, false)] + [DataRow(DataStreamContents.NO_DATA, 1, 1, false)] + [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 1, 1, false)] [DataRow(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE, 0, 0)] [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE, 0, 0)] [DataRow(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE, 0, 0)] @@ -50,8 +50,8 @@ public void ValidateDataReturnsExpectedErrorCount( [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS, 0, 0)] [DataRow(DataStreamContents.SIMPLE_INVALID_DATA, 7, 12)] - [DataRow(DataStreamContents.NO_DATA, 2, 6, false)] - [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 2, 6, false)] + [DataRow(DataStreamContents.NO_DATA, 1, 1, false)] + [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 1, 1, false)] [DataRow(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE, 0, 0)] [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE, 0, 0)] [DataRow(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE, 0, 0)] diff --git a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs index 15f16d87..d2eee2e1 100644 --- a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs @@ -53,6 +53,32 @@ public void ValidatePxFileSyntaxCalledWithMininalUtf8ReturnsValidResult() Assert.HasCount(0, feedback); } + [TestMethod] + public async Task ValidateAndValidateAsyncDataWithBomAndMultibyteMetadataReturnSameRawByteOffset() + { + // Arrange + string content = "TITLE=\"Ää Öö\";\r\nDATA=\r\n\t2635 2;"; + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] data = [.. bom, .. Encoding.UTF8.GetBytes(content)]; + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..content.IndexOf("2635", StringComparison.Ordinal)]); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + SyntaxValidator synchronousValidator = new(); + SyntaxValidator asynchronousValidator = new(); + + // Act + SyntaxValidationResult synchronousResult = synchronousValidator.Validate(synchronousStream, "foo", Encoding.UTF8); + SyntaxValidationResult asynchronousResult = await asynchronousValidator.ValidateAsync(asynchronousStream, "foo", Encoding.UTF8, cancellationToken: System.Threading.CancellationToken.None); + + // Assert + Assert.AreEqual(expectedPosition, synchronousResult.DataStartStreamPosition); + Assert.AreEqual(synchronousResult.DataStartStreamPosition, asynchronousResult.DataStartStreamPosition); + Assert.AreEqual(1, synchronousResult.DataStartRow); + Assert.AreEqual(synchronousResult.DataStartRow, asynchronousResult.DataStartRow); + synchronousStream.Position = synchronousResult.DataStartStreamPosition; + Assert.AreEqual('2', (char)synchronousStream.ReadByte()); + } + [TestMethod] public void ValidateObjectsCalledWithMultipleEntriesInSingleLineReturnsWithWarnings() { diff --git a/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs b/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs index 9651e5b0..08ce2d41 100644 --- a/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs +++ b/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models.Data.DataValue; +using Px.Utils.Models.Data.DataValue; using Px.Utils.Models.Metadata; using Px.Utils.Models.Metadata.ExtensionMethods; @@ -36,7 +36,7 @@ public PxFileStreamDataReader(Stream stream, PxFileConfiguration? conf = null, i /// Constructor that allows specifying the position of the data section in the file. /// /// Px file stream - /// Position of the first data point in the file + /// Absolute raw byte offset of the first non-whitespace data value after DATA=. /// Px file syntax configuration public PxFileStreamDataReader(Stream stream, long dataStart, PxFileConfiguration? conf = null, int readBufferSize = 4096) { @@ -272,25 +272,23 @@ public void Dispose() private void SetReaderPositionIfZero() { if (_stream.Position != 0) return; - string dataKeyword = _conf.Tokens.KeyWords.Data; - long start = StreamUtilities.FindKeywordPosition(_stream, dataKeyword, _conf); + long start = StreamUtilities.FindDataStartPosition(_stream, _conf, _readBufferSize); if (start == -1) { - throw new ArgumentException($"Could not find data keyword '{dataKeyword}'"); + throw new ArgumentException($"Could not find the first data value after '{_conf.Tokens.KeyWords.Data}='"); } - _stream.Position = start + dataKeyword.Length + 1; // +1 to skip the '=' + _stream.Position = start; } private async Task SetReaderPositionIfZeroAsync(CancellationToken? cancellationToken = null) { if (_stream.Position != 0) return; - string dataKeyword = _conf.Tokens.KeyWords.Data; - long start = await StreamUtilities.FindKeywordPositionAsync(_stream, dataKeyword, _conf, cancellationToken); + long start = await StreamUtilities.FindDataStartPositionAsync(_stream, _conf, _readBufferSize, cancellationToken ?? CancellationToken.None); if (start == -1) { - throw new ArgumentException($"Could not find data keyword '{dataKeyword}'"); + throw new ArgumentException($"Could not find the first data value after '{_conf.Tokens.KeyWords.Data}='"); } - _stream.Position = start + dataKeyword.Length + 1; // +1 to skip the '=' + _stream.Position = start; } private void ReadItemsFromStreamByCoordinate(T[] buffer, int offset, IMatrixMap target, IMatrixMap complete, Func readItem, CancellationToken? token = null) diff --git a/Px.Utils/PxFile/Data/StreamUtilities.cs b/Px.Utils/PxFile/Data/StreamUtilities.cs index adcc9748..5c25e034 100644 --- a/Px.Utils/PxFile/Data/StreamUtilities.cs +++ b/Px.Utils/PxFile/Data/StreamUtilities.cs @@ -1,4 +1,5 @@ -using System.Text; +using System.Runtime.CompilerServices; +using System.Text; namespace Px.Utils.PxFile.Data { @@ -8,76 +9,197 @@ namespace Px.Utils.PxFile.Data public static class StreamUtilities { /// - /// Finds the position of the first occurrence of a specified keyword in a given stream. + /// Finds the absolute raw byte offset of the first non-whitespace data value after a top-level DATA entry. + /// The stream position is restored before this method returns. Returns -1 when the DATA entry or its first value cannot be found. /// - /// The stream to search in. - /// The keyword to search for. - /// A configuration object that contains symbols used in the px file syntax. + /// The seekable PX file stream to search from its origin. + /// A configuration object that contains the DATA keyword and PX syntax symbols. /// The size of the buffer to use when reading from the stream. Defaults to 4096. - /// The position of the keyword in the stream if found, otherwise -1. - public static long FindKeywordPosition(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize = 4096) + /// The absolute raw byte offset of the first data value, or -1. + public static long FindDataStartPosition(Stream stream, PxFileConfiguration conf, int bufferSize = 4096) { - return FindKeywordPostionImpl(stream, keyword, conf, bufferSize); + long originalPosition = stream.Position; + try + { + stream.Position = 0; + return FindDataStartPositionImpl(stream, conf, bufferSize); + } + finally + { + stream.Position = originalPosition; + } } /// - /// Asynchronously finds the position of the first occurrence of a specified keyword in a given stream. + /// Asynchronously finds the absolute raw byte offset of the first non-whitespace data value after a top-level DATA entry. + /// The stream position is restored before this method returns. Returns -1 when the DATA entry or its first value cannot be found. /// - /// The stream to search in. - /// The keyword to search for. - /// A configuration object that contains symbols used in the px file syntax. - /// A token that can be used to cancel the operation. Defaults to None. + /// The seekable PX file stream to search from its origin. + /// A configuration object that contains the DATA keyword and PX syntax symbols. + /// A token that can be used to cancel the operation. /// The size of the buffer to use when reading from the stream. Defaults to 4096. - /// The task result contains the position of the keyword in the stream if found, otherwise -1. - public async static Task FindKeywordPositionAsync(Stream stream, string keyword, PxFileConfiguration conf, CancellationToken? cToken = null, int bufferSize = 4096) + /// The absolute raw byte offset of the first data value, or -1. + public static async Task FindDataStartPositionAsync(Stream stream, PxFileConfiguration conf, int bufferSize = 4096, CancellationToken cancellationToken = default) { - return await Task.Factory.StartNew( - () => FindKeywordPostionImpl(stream, keyword, conf, bufferSize, cToken), cToken - ?? CancellationToken.None - ); + long originalPosition = stream.Position; + try + { + stream.Position = 0; + return await FindDataStartPositionImplAsync(stream, conf, bufferSize, cancellationToken); + } + finally + { + stream.Position = originalPosition; + } } - private static long FindKeywordPostionImpl(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize = 4096, CancellationToken? cancellationToken = null) + private static long FindDataStartPositionImpl(Stream stream, PxFileConfiguration conf, int bufferSize) { - char entrySeparator = conf.Symbols.EntrySeparator; - - byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); + byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); byte[] buffer = new byte[bufferSize]; + DataStartSearchState state = new(); - long read; - int keywordIndex = 0; - int lastKeyIndex = keywordBytes.Length - 1; - bool searchMode = true; - - do + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) { - cancellationToken?.ThrowIfCancellationRequested(); - read = stream.Read(buffer, 0, bufferSize); + long bufferStart = stream.Position - bytesRead; + if (TryFindDataStartPosition( + buffer.AsSpan(0, bytesRead), + bufferStart, + dataKeywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.KeywordSeparator, + (byte)conf.Symbols.Key.StringDelimeter, + ref state, + out long dataStartPosition)) + { + return dataStartPosition; + } + } + + return -1; + } - for (int i = 0; i < read; i++) + private static async Task FindDataStartPositionImplAsync(Stream stream, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) + { + byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); + byte[] buffer = new byte[bufferSize]; + DataStartSearchState state = new(); + + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) + { + long bufferStart = stream.Position - bytesRead; + if (TryFindDataStartPosition( + buffer.AsSpan(0, bytesRead), + bufferStart, + dataKeywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.KeywordSeparator, + (byte)conf.Symbols.Key.StringDelimeter, + ref state, + out long dataStartPosition)) { - if (searchMode && !CharacterConstants.WhitespaceCharacters.Contains((char)buffer[i])) - { - if (buffer[i] == keywordBytes[keywordIndex]) - { - if (keywordIndex == lastKeyIndex) return stream.Position - read + i - keyword.Length; - else keywordIndex++; - } - else - { - searchMode = false; - keywordIndex = 0; - } - } - else if (buffer[i] == entrySeparator) - { - searchMode = true; - } + return dataStartPosition; } } - while (read > 0); return -1; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryFindDataStartPosition( + ReadOnlySpan buffer, + long bufferStart, + ReadOnlySpan dataKeywordBytes, + byte entrySeparator, + byte keywordSeparator, + byte stringDelimiter, + ref DataStartSearchState state, + out long dataStartPosition) + { + for (int i = 0; i < buffer.Length; i++) + { + if (TryProcessDataStartByte(buffer[i], dataKeywordBytes, entrySeparator, keywordSeparator, stringDelimiter, ref state)) + { + dataStartPosition = bufferStart + i; + return true; + } + } + + dataStartPosition = -1; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryProcessDataStartByte( + byte currentByte, + ReadOnlySpan dataKeywordBytes, + byte entrySeparator, + byte keywordSeparator, + byte stringDelimiter, + ref DataStartSearchState state) + { + if (state.IsAfterDataKeyword) + { + return !IsWhitespace(currentByte); + } + if (state.IsInString) + { + state.IsInString = currentByte != stringDelimiter; + return false; + } + if (currentByte == stringDelimiter) + { + state.IsInString = true; + return false; + } + if (currentByte == entrySeparator) + { + state.ResetEntry(); + return false; + } + if (state.IsAtEntryStart && IsWhitespace(currentByte)) return false; + if (state.IsAtEntryStart && state.MatchedKeywordBytes < dataKeywordBytes.Length && currentByte == dataKeywordBytes[state.MatchedKeywordBytes]) + { + state.MatchedKeywordBytes++; + return false; + } + if (state.MatchedKeywordBytes == dataKeywordBytes.Length && currentByte == keywordSeparator) + { + state.IsAfterDataKeyword = true; + return false; + } + + state.IsAtEntryStart = false; + state.MatchedKeywordBytes = 0; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsWhitespace(byte value) + { + return value is CharacterConstants.SPACE or CharacterConstants.HORIZONTALTAB or CharacterConstants.CARRIAGERETURN or CharacterConstants.LINEFEED; + } + + private struct DataStartSearchState + { + public int MatchedKeywordBytes; + public bool IsAtEntryStart; + public bool IsInString; + public bool IsAfterDataKeyword; + + public DataStartSearchState() + { + IsAtEntryStart = true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ResetEntry() + { + MatchedKeywordBytes = 0; + IsAtEntryStart = true; + } + } } } diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index f726cec2..361b0ea6 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -1,6 +1,7 @@ using System.Runtime.CompilerServices; using System.Text; using Px.Utils.PxFile; +using Px.Utils.PxFile.Data; using Px.Utils.Validation.DatabaseValidation; namespace Px.Utils.Validation.DataValidation @@ -15,7 +16,6 @@ namespace Px.Utils.Validation.DataValidation public class DataValidator(int rowLen, int numOfRows, int startRow, PxFileConfiguration? conf = null) : IPxFileStreamValidator, IPxFileStreamValidatorAsync { private const int _streamBufferSize = 4096; - private static ReadOnlySpan MissingValueStartBytes => [CharacterConstants.QUOTATIONMARK, (byte)'-', (byte)'.']; private readonly PxFileConfiguration _conf = conf ?? PxFileConfiguration.Default; @@ -26,6 +26,7 @@ public class DataValidator(int rowLen, int numOfRows, int startRow, PxFileConfig private EntryType _currentEntryType = EntryType.Unknown; private List _currentEntry = []; + private List _currentRow = []; private int _lineNumber = 1; private int _charPosition; private EntryType _currentCharacterType; @@ -56,7 +57,7 @@ public ValidationResult Validate( SetValidationParameters(encoding, filename); ValidationFeedback validationFeedbacks = []; - int dataStartIndex = GetStreamIndexOfFirstDataValue(stream, ref validationFeedbacks); + long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); if (dataStartIndex == -1) { KeyValuePair feedback = @@ -65,7 +66,7 @@ public ValidationResult Validate( new(filename, 0, 0)); validationFeedbacks.Add(feedback); - return new (validationFeedbacks); + return new(validationFeedbacks); } stream.Position = dataStartIndex; ValidationFeedback dataStreamFeedbacks = ValidateDataStream(stream); @@ -73,7 +74,7 @@ public ValidationResult Validate( ResetValidator(); - return new (validationFeedbacks); + return new(validationFeedbacks); } /// @@ -100,7 +101,7 @@ public async Task ValidateAsync( SetValidationParameters(encoding, filename); ValidationFeedback validationFeedbacks = []; - int dataStartIndex = GetStreamIndexOfFirstDataValue(stream, ref validationFeedbacks); + long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); if (dataStartIndex == -1) { KeyValuePair feedback = @@ -109,16 +110,16 @@ public async Task ValidateAsync( new(filename, 0, 0)); validationFeedbacks.Add(feedback); - return new (validationFeedbacks); + return new(validationFeedbacks); } stream.Position = dataStartIndex; - ValidationFeedback dataStreamFeedbacks = await Task.Factory.StartNew(() => + ValidationFeedback dataStreamFeedbacks = await Task.Factory.StartNew(() => ValidateDataStream(stream, cancellationToken), cancellationToken); validationFeedbacks.AddRange(dataStreamFeedbacks); ResetValidator(); - return new (validationFeedbacks); + return new(validationFeedbacks); } private void SetValidationParameters(Encoding encoding, string filename) @@ -139,6 +140,7 @@ private ValidationFeedback ValidateDataStream(Stream stream, CancellationToken? ValidationFeedback validationFeedbacks = []; byte endOfData = (byte)_conf.Symbols.EntrySeparator; _currentEntry = new(_streamBufferSize); + _currentRow = new(_streamBufferSize); byte[] buffer = new byte[_streamBufferSize]; int bytesRead = 0; @@ -164,10 +166,15 @@ private ValidationFeedback ValidateDataStream(Stream stream, CancellationToken? HandleNonSeparatorType(ref validationFeedbacks); } _currentEntryType = _currentCharacterType; + // Console.WriteLine($"entry: {_encoding.GetString(_currentEntry.ToArray())}"); _currentEntry.Clear(); } _currentEntry.Add(currentByte); + if (_currentCharacterType != EntryType.LineSeparator) + { + _currentRow.Add(currentByte); + } _charPosition++; } } @@ -210,7 +217,7 @@ private void HandleEntryTypeChange(ref ValidationFeedback validationFeedbacks) KeyValuePair? feedback = validator.Validate( _currentEntry, _currentEntryType, - _encoding, + _encoding, _lineNumber + startRow, _charPosition, _filename); @@ -231,10 +238,13 @@ private void HandleNonSeparatorType(ref ValidationFeedback validationFeedbacks) } else if (_currentCharacterType == EntryType.LineSeparator) { + Console.WriteLine($"row at {_lineNumber}: {_encoding.GetString(_currentRow.ToArray())}"); + Console.WriteLine($"{_currentRowLength} vs {rowLen} items"); + _currentRow.Clear(); if (_currentRowLength != rowLen) { validationFeedbacks.Add(new( - new (ValidationFeedbackLevel.Error, + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidRowLength), new(_filename, _lineNumber + startRow, _charPosition, $"Expected {rowLen}, got row length of {_currentRowLength}.")) @@ -254,45 +264,30 @@ private void ResetValidator() _dataSeparatorValidators.Clear(); _currentEntryType = EntryType.Unknown; _currentEntry.Clear(); + _currentRow.Clear(); _lineNumber = 1; _charPosition = 0; _currentRowLength = 0; } - private int GetStreamIndexOfFirstDataValue(Stream stream, ref ValidationFeedback feedbacks) + private static long GetStreamIndexOfFirstDataValue(Stream stream) { - byte[] buffer = new byte[_streamBufferSize]; - int bytesRead; - do + if (stream.Position == 0) { - bytesRead = stream.Read(buffer, 0, buffer.Length); - for (int i = 0; i < bytesRead; i++) + return StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, _streamBufferSize); + } + + int currentByte; + while ((currentByte = stream.ReadByte()) != -1) + { + if (currentByte is not CharacterConstants.SPACE and not CharacterConstants.HORIZONTALTAB and not CharacterConstants.CARRIAGERETURN and not CharacterConstants.LINEFEED) { - byte currentByte = buffer[i]; - char currentChar = (char)currentByte; - if (IsDataValueStartByte(currentByte)) - { - return (int)stream.Position - bytesRead + i; - } - else if (!CharacterConstants.WhitespaceCharacters.Contains(currentChar)) - { - feedbacks.Add(new( - new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.DataValidationFeedbackInvalidChar), - new(_filename, _lineNumber + startRow, _charPosition)) - ); - } + return stream.Position - 1; } - } while (bytesRead > 0); + } return -1; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsDataValueStartByte(byte currentByte) - { - return currentByte >= CharacterConstants.Zero && currentByte <= CharacterConstants.Nine || MissingValueStartBytes.Contains(currentByte); - } } /// diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationResult.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationResult.cs index cbb17daf..dc037752 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationResult.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationResult.cs @@ -1,4 +1,4 @@ -namespace Px.Utils.Validation.SyntaxValidation +namespace Px.Utils.Validation.SyntaxValidation { /// /// Represents the result of a syntax validation operation. This struct contains a validation report and a list of structured validation entries. @@ -6,14 +6,24 @@ /// A dictionary of amd objects produced by the syntax validation operation. /// A list of objects produced by the syntax validation operation. /// The row number where the data section starts in the file. - /// The stream position where the data section starts in the file. - public class SyntaxValidationResult(ValidationFeedback feedbacks, List result, int dataStartRow, int dataStartStreamPosition) : ValidationResult(feedbacks) + /// The absolute raw byte offset of the first non-whitespace data value after the DATA keyword, + /// suitable for direct assignment to ; otherwise -1 when the DATA keyword or a value cannot be found. + public class SyntaxValidationResult(ValidationFeedback feedbacks, List result, int dataStartRow, long dataStartStreamPosition) : ValidationResult(feedbacks) { /// /// Gets the list of objects produced by the syntax validation operation. /// public List Result { get; } = result; + + /// + /// Gets the row number where the data section starts in the file. + /// public int DataStartRow { get; } = dataStartRow; + + /// + /// Gets the absolute raw byte offset of the first non-whitespace data value after the DATA keyword. + /// The value is suitable for direct assignment to and is -1 when the DATA keyword or a value cannot be found. + /// public long DataStartStreamPosition { get; } = dataStartStreamPosition; } } diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs index 5528fb99..6f1ebcbc 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs @@ -1,4 +1,5 @@ -using Px.Utils.PxFile; +using Px.Utils.PxFile; +using Px.Utils.PxFile.Data; using Px.Utils.Validation.DatabaseValidation; using System.Runtime.CompilerServices; using System.Text; @@ -19,7 +20,7 @@ public class SyntaxValidator( { private const int _bufferSize = 4096; private int _dataSectionStartRow = -1; - private int _dataSectionStartStreamPosition = -1; + private long _dataSectionStartStreamPosition = -1; /// /// Validates the syntax of a PX file's metadata. @@ -52,6 +53,8 @@ public SyntaxValidationResult Validate( } conf ??= PxFileConfiguration.Default; + ResetDataSectionPosition(); + _dataSectionStartStreamPosition = StreamUtilities.FindDataStartPosition(stream, conf, _bufferSize); ValidationFeedback validationFeedbacks = []; List stringEntries = BuildValidationEntries(stream, encoding, conf, filename, _bufferSize); @@ -97,6 +100,8 @@ public async Task ValidateAsync( } conf ??= PxFileConfiguration.Default; + ResetDataSectionPosition(); + _dataSectionStartStreamPosition = await StreamUtilities.FindDataStartPositionAsync(stream, conf, _bufferSize, cancellationToken); ValidationFeedback validationFeedbacks = []; List entries = await BuildValidationEntriesAsync(stream, encoding, conf, filename, _bufferSize, cancellationToken); validationFeedbacks.AddRange(ValidateEntries(entries, stringValidationFunctions, conf)); @@ -251,8 +256,6 @@ private List BuildValidationEntries(Stream stream, Encoding enc if (IsEndOfMetadataSection(buffer[i], syntaxConf, entryBuilder, isProcessingString)) { _dataSectionStartRow = lineChangeIndexes.Count; - // This here should find the actual start of the data section, after line changes, spaces and whatnot. - _dataSectionStartStreamPosition = characterIndex + 1; return entries; } UpdateLineAndCharacter(buffer[i], syntaxConf, ref characterIndex, ref lineChangeIndexes, ref isProcessingString); @@ -311,7 +314,6 @@ private async Task> BuildValidationEntriesAsync( if (IsEndOfMetadataSection(buffer[i], syntaxConf, entryBuilder, isProcessingString)) { _dataSectionStartRow = lineChangeIndexes.Count; - _dataSectionStartStreamPosition = characterIndex + 1; return entries; } UpdateLineAndCharacter(buffer[i], syntaxConf, ref characterIndex, ref lineChangeIndexes, ref isProcessingString); @@ -335,6 +337,12 @@ private async Task> BuildValidationEntriesAsync( return entries; } + private void ResetDataSectionPosition() + { + _dataSectionStartRow = -1; + _dataSectionStartStreamPosition = -1; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void UpdateLineAndCharacter(char currentCharacter, PxFileConfiguration syntaxConf, ref int characterIndex, ref List linebreakIndexes, ref bool isProcessingString) { From bccbcaebef3f965123a95fcd485965f03821abe3 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Wed, 5 Aug 2026 16:55:25 +0300 Subject: [PATCH 02/12] Memory bloat safe guard options for validation process --- Px.Utils.TestingApp/Commands/Benchmark.cs | 26 ++++- .../Commands/DataValidationBenchmark.cs | 7 +- .../Commands/DatabaseValidationBenchmark.cs | 8 +- .../MetadataContentValidationBenchmark.cs | 5 +- .../MetadataSyntaxValidationBenchmark.cs | 7 +- .../Commands/PxFileValidationBenchmark.cs | 7 +- .../Validation/ValidationFeedbackSinkTests.cs | 64 +++++++++++ .../ContentValidation/ContentValidator.cs | 16 +++ .../DataValidation/DataValidator.cs | 67 ++++++++++-- .../DatabaseValidation/DatabaseValidator.cs | 41 ++++--- Px.Utils/Validation/PxFileValidator.cs | 82 +++++++++----- .../SyntaxValidation/SyntaxValidator.cs | 54 ++++++++++ Px.Utils/Validation/ValidationFeedbackSink.cs | 100 ++++++++++++++++++ Px.Utils/Validation/ValidationOptions.cs | 18 ++++ 14 files changed, 433 insertions(+), 69 deletions(-) create mode 100644 Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs create mode 100644 Px.Utils/Validation/ValidationFeedbackSink.cs create mode 100644 Px.Utils/Validation/ValidationOptions.cs diff --git a/Px.Utils.TestingApp/Commands/Benchmark.cs b/Px.Utils.TestingApp/Commands/Benchmark.cs index 4c7b2b0d..86bf5660 100644 --- a/Px.Utils.TestingApp/Commands/Benchmark.cs +++ b/Px.Utils.TestingApp/Commands/Benchmark.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Reflection; +using Px.Utils.Validation; namespace Px.Utils.TestingApp.Commands { @@ -77,12 +78,18 @@ internal BenchmarkResult(string name, IReadOnlyList iterationTimesMs) /// protected Func[] BenchmarkFunctionsAsync { get; set; } = []; + /// + /// Feedback retention options used by validation benchmark commands. + /// + protected ValidationOptions ValidationOptions { get; private set; } = new(); + private static readonly string[] iterFlags = ["-i", "-iter"]; + private static readonly string[] feedbackLimitFlags = ["-l", "-limit"]; /// /// List of flags that can be used to provide parameters to the benchmark command. /// - protected List ParameterFlags { get; } = [iterFlags]; + protected List ParameterFlags { get; } = [iterFlags, feedbackLimitFlags]; internal List Results { get; } = []; private int processesCompleted; @@ -137,7 +144,8 @@ internal override async Task Run(bool batchMode, List? inputs = null) protected virtual void SetRunParameters() { - if (Parameters.Keys.Count == ParameterFlags.Count) + ValidationOptions = new(); + if (Parameters.Keys.All(key => Array.Exists(ParameterFlags.ToArray(), flag => flag.Contains(key)))) { foreach (string key in Parameters.Keys) { @@ -146,9 +154,9 @@ protected virtual void SetRunParameters() { Iterations = iterations; } - else if(!Array.Exists(ParameterFlags.ToArray(), flag => flag.Contains(key))) + else if (feedbackLimitFlags.Contains(key) && Parameters[key].Count == 1) { - throw new ArgumentException($"Invalid argument {key} {string.Join(' ', Parameters[key])}"); + ValidationOptions = ParseValidationOptions(Parameters[key][0]); } } } @@ -176,6 +184,16 @@ protected virtual void StartInteractiveMode() Iterations = value; } + private static ValidationOptions ParseValidationOptions(string value) + { + if (int.TryParse(value, out int limit) && limit > 0) + { + return new ValidationOptions { MaxFeedbackItemsPerSignature = limit }; + } + + return ValidationOptions.Unlimited; + } + /// /// Setup method for the benchmark. Called before running the benchmarks. Marked as virtual to allow for custom setup in derived classes. /// diff --git a/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs index 5f3a9065..eaa96a02 100644 --- a/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs @@ -17,7 +17,8 @@ internal sealed class DataValidationBenchmark : FileBenchmark internal override string Help => "Validates the px file data." + Environment.NewLine + "\t-r, -rows: How many data rows the validator should expect." + Environment.NewLine + - "\t-c, -cols: How many data colums the validator should expect."; + "\t-c, -cols: How many data colums the validator should expect." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Benchmarks the data validation capabilities of the DataValidator."; @@ -52,7 +53,7 @@ private void ValidateDataBenchmarks() using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); stream.Position = start; DataValidator validator = new(expectedCols, expectedRows, 0); - validator.Validate(stream, TestFilePath, encoding); + validator.Validate(stream, TestFilePath, encoding, null, ValidationOptions); } private async Task ValidateDataBenchmarksAsync() @@ -61,7 +62,7 @@ private async Task ValidateDataBenchmarksAsync() stream.Position = start; DataValidator validator = new(expectedCols, expectedRows, 0); - await validator.ValidateAsync(stream, TestFilePath, encoding); + await validator.ValidateAsync(stream, TestFilePath, encoding, null, ValidationOptions); } protected override void SetRunParameters() diff --git a/Px.Utils.TestingApp/Commands/DatabaseValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/DatabaseValidationBenchmark.cs index b59d3243..2efccabf 100644 --- a/Px.Utils.TestingApp/Commands/DatabaseValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/DatabaseValidationBenchmark.cs @@ -4,7 +4,9 @@ namespace Px.Utils.TestingApp.Commands { internal sealed class DatabaseValidationBenchmark : Benchmark { - internal override string Help => "Validates a px path database."; + internal override string Help => + "Validates a px path database." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Validates a px path database."; private static readonly string[] directoryFlags = ["-d", "-directory"]; @@ -59,13 +61,13 @@ protected override async Task OneTimeBenchmarkSetupAsync() private void ValidationBenchmark() { if(validator is null) throw new InvalidOperationException("Validator not initialized."); - validator.Validate(); + validator.Validate(ValidationOptions); } private async Task ValidationBenchmarkAsync() { if(validator is null) throw new InvalidOperationException("Validator not initialized."); - await validator.ValidateAsync(); + await validator.ValidateAsync(ValidationOptions); } } } diff --git a/Px.Utils.TestingApp/Commands/MetadataContentValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/MetadataContentValidationBenchmark.cs index 87fa9cec..a2dd89f3 100644 --- a/Px.Utils.TestingApp/Commands/MetadataContentValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/MetadataContentValidationBenchmark.cs @@ -9,7 +9,8 @@ internal sealed class MetadataContentValidationBenchmark : FileBenchmark internal override string Help => "Validates the contents of the Px file metadata given amount of times." + Environment.NewLine + "\t-f, -file: The path to the px file to read." + Environment.NewLine + - "\t-i, -iter: The number of iterations to run."; + "\t-i, -iter: The number of iterations to run." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Benchmarks the metadata content validation of Px.Utils/Validation/SyntaxValidator."; @@ -34,7 +35,7 @@ protected override async Task OneTimeBenchmarkSetupAsync() private void ValidateContentBenchmark() { ContentValidator validator = new(TestFilePath, Encoding.Default, [.. _entries]); - validator.Validate(); + validator.Validate(ValidationOptions); } } } diff --git a/Px.Utils.TestingApp/Commands/MetadataSyntaxValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/MetadataSyntaxValidationBenchmark.cs index e85df6cd..cd3ccf52 100644 --- a/Px.Utils.TestingApp/Commands/MetadataSyntaxValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/MetadataSyntaxValidationBenchmark.cs @@ -9,7 +9,8 @@ internal sealed class MetadataSyntaxValidationBenchmark : FileBenchmark internal override string Help => "Validates the syntax of the Px file metadata given amount of times." + Environment.NewLine + "\t-f, -file: The path to the px file to read." + Environment.NewLine + - "\t-i, -iter: The number of iterations to run."; + "\t-i, -iter: The number of iterations to run." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Benchmarks the metadata syntax validation of Px.Utils/Validation/SyntaxValidator."; @@ -34,7 +35,7 @@ private void SyntaxValidationBenchmark() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); SyntaxValidator validator = new(); - validator.Validate(stream, TestFilePath, encoding); + validator.Validate(stream, TestFilePath, encoding, null, ValidationOptions); stream.Close(); } @@ -42,7 +43,7 @@ private async Task SyntaxValidationBenchmarkAsync() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); SyntaxValidator validator = new(); - await validator.ValidateAsync(stream, TestFilePath, encoding); + await validator.ValidateAsync(stream, TestFilePath, encoding, null, ValidationOptions); stream.Close(); } } diff --git a/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs index f8a175a2..53240ae1 100644 --- a/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs @@ -8,7 +8,8 @@ namespace Px.Utils.TestingApp.Commands internal sealed class PxFileValidationBenchmark : FileBenchmark { internal override string Help => - "Runs through the whole px file validation process (metadata syntax- and contents-, data-) for the given file."; + "Runs through the whole px file validation process (metadata syntax- and contents-, data-) for the given file." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Benchmarks the px file validation capabilities of the PxFileValidator."; @@ -42,14 +43,14 @@ private void ValidatePxFileBenchmarks() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); PxFileValidator validator = new(); - validator.Validate(stream, TestFilePath, encoding); + var result = validator.Validate(stream, TestFilePath, encoding, null, ValidationOptions); } private async Task ValidatePxFileBenchmarksAsync() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); PxFileValidator validator = new(); - await validator.ValidateAsync(stream, TestFilePath, encoding); + await validator.ValidateAsync(stream, TestFilePath, encoding, null, ValidationOptions); } } } diff --git a/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs b/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs new file mode 100644 index 00000000..a6ce79a0 --- /dev/null +++ b/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs @@ -0,0 +1,64 @@ +using Px.Utils.Validation; + +namespace Px.Utils.UnitTests.Validation +{ + [TestClass] + public class ValidationFeedbackSinkTests + { + [TestMethod] + public void ReportMatchingFeedbackBeyondLimitRetainsLimitAndAnnotatesFinalItem() + { + ValidationFeedbackSink sink = new(new ValidationOptions { MaxFeedbackItemsPerSignature = 2 }); + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + + sink.Report(key, new ValidationFeedbackValue("file.px", 1)); + sink.Report(key, new ValidationFeedbackValue("file.px", 2, additionalInfo: "Original information.")); + sink.Report(key, new ValidationFeedbackValue("file.px", 3)); + sink.Report(key, new ValidationFeedbackValue("file.px", 4)); + + ValidationFeedback feedback = sink.ToFeedback(); + List values = feedback[key]; + + Assert.AreEqual(2, values.Count); + StringAssert.Contains(values[1].AdditionalInfo, "Original information."); + StringAssert.Contains(values[1].AdditionalInfo, "Feedback limit of 2 instances"); + } + + [TestMethod] + public void ReportUnlimitedFeedbackRetainsAllItems() + { + ValidationFeedbackSink sink = new(ValidationOptions.Unlimited); + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + + sink.Report(key, new ValidationFeedbackValue("file.px", 1)); + sink.Report(key, new ValidationFeedbackValue("file.px", 2)); + sink.Report(key, new ValidationFeedbackValue("file.px", 3)); + + Assert.AreEqual(3, sink.ToFeedback()[key].Count); + } + + [TestMethod] + public void ReportDifferentSignaturesRetainsSeparateLimits() + { + ValidationFeedbackSink sink = new(new ValidationOptions { MaxFeedbackItemsPerSignature = 1 }); + ValidationFeedbackKey errorKey = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + ValidationFeedbackKey warningKey = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + + sink.Report(errorKey, new ValidationFeedbackValue("first.px")); + sink.Report(errorKey, new ValidationFeedbackValue("first.px")); + sink.Report(errorKey, new ValidationFeedbackValue("second.px")); + sink.Report(warningKey, new ValidationFeedbackValue("first.px")); + + ValidationFeedback feedback = sink.ToFeedback(); + + Assert.AreEqual(2, feedback[errorKey].Select(value => value.Filename).Distinct(StringComparer.Ordinal).Count()); + Assert.AreEqual(1, feedback[warningKey].Count); + } + + [TestMethod] + public void ValidationOptionsNonPositiveLimitThrows() + { + Assert.ThrowsExactly(() => new ValidationFeedbackSink(new ValidationOptions { MaxFeedbackItemsPerSignature = 0 })); + } + } +} diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.cs index 87c4e27f..edf8e344 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.cs @@ -93,6 +93,22 @@ public ContentValidationResult Validate() return new ContentValidationResult(feedbackItems, lengthOfDataRows, amountOfDataRows); } + /// + /// Validates contents of PX file metadata using the specified feedback retention options. + /// + public ContentValidationResult Validate(ValidationOptions options) + { + ValidationFeedbackSink sink = new(options); + return Validate(sink); + } + + internal ContentValidationResult Validate(ValidationFeedbackSink sink) + { + ContentValidationResult result = Validate(); + sink.ReportRange(result.FeedbackItems); + return new ContentValidationResult(sink.ToFeedback(), result.DataRowLength, result.DataRowAmount); + } + #region Interface implementation ValidationResult IValidator.Validate() diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index 361b0ea6..3d18c406 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -26,7 +26,6 @@ public class DataValidator(int rowLen, int numOfRows, int startRow, PxFileConfig private EntryType _currentEntryType = EntryType.Unknown; private List _currentEntry = []; - private List _currentRow = []; private int _lineNumber = 1; private int _charPosition; private EntryType _currentCharacterType; @@ -68,6 +67,7 @@ public ValidationResult Validate( return new(validationFeedbacks); } + stream.Position = dataStartIndex; ValidationFeedback dataStreamFeedbacks = ValidateDataStream(stream); validationFeedbacks.AddRange(dataStreamFeedbacks); @@ -77,6 +77,60 @@ public ValidationResult Validate( return new(validationFeedbacks); } + /// + /// Validates data using the specified feedback retention options. + /// + public ValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options) + { + ValidationFeedbackSink sink = new(options); + return Validate(stream, filename, encoding, fileSystem, sink); + } + + internal ValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink) + { + ValidationResult result = Validate(stream, filename, encoding, fileSystem); + sink.ReportRange(result.FeedbackItems); + return new ValidationResult(sink.ToFeedback()); + } + + /// + /// Asynchronously validates data using the specified feedback retention options. + /// + public async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options, + CancellationToken cancellationToken = default) + { + ValidationFeedbackSink sink = new(options); + return await ValidateAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + } + + internal async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink, + CancellationToken cancellationToken = default) + { + ValidationResult result = await ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); + sink.ReportRange(result.FeedbackItems); + return new ValidationResult(sink.ToFeedback()); + } + /// /// Validates the data in the specified stream asynchronously. /// Assumes that the stream is at the start of the data section (after 'DATA='-keyword) at the first data item. @@ -86,6 +140,7 @@ public ValidationResult Validate( /// Name of the file being validated. If not provided, validator tries to find the encoding. /// File system used for file operations. If not provided, default file system is used. /// Cancellation token for cancelling the validation process + /// /// object that contains a collection of /// validation feedback key value pairs representing the feedback for the data validation. /// @@ -140,7 +195,6 @@ private ValidationFeedback ValidateDataStream(Stream stream, CancellationToken? ValidationFeedback validationFeedbacks = []; byte endOfData = (byte)_conf.Symbols.EntrySeparator; _currentEntry = new(_streamBufferSize); - _currentRow = new(_streamBufferSize); byte[] buffer = new byte[_streamBufferSize]; int bytesRead = 0; @@ -166,15 +220,10 @@ private ValidationFeedback ValidateDataStream(Stream stream, CancellationToken? HandleNonSeparatorType(ref validationFeedbacks); } _currentEntryType = _currentCharacterType; - // Console.WriteLine($"entry: {_encoding.GetString(_currentEntry.ToArray())}"); _currentEntry.Clear(); } _currentEntry.Add(currentByte); - if (_currentCharacterType != EntryType.LineSeparator) - { - _currentRow.Add(currentByte); - } _charPosition++; } } @@ -238,9 +287,6 @@ private void HandleNonSeparatorType(ref ValidationFeedback validationFeedbacks) } else if (_currentCharacterType == EntryType.LineSeparator) { - Console.WriteLine($"row at {_lineNumber}: {_encoding.GetString(_currentRow.ToArray())}"); - Console.WriteLine($"{_currentRowLength} vs {rowLen} items"); - _currentRow.Clear(); if (_currentRowLength != rowLen) { validationFeedbacks.Add(new( @@ -264,7 +310,6 @@ private void ResetValidator() _dataSeparatorValidators.Clear(); _currentEntryType = EntryType.Unknown; _currentEntry.Clear(); - _currentRow.Clear(); _lineNumber = 1; _charPosition = 0; _currentRowLength = 0; diff --git a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs index 3a164955..ebaf26b2 100644 --- a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs +++ b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs @@ -38,8 +38,14 @@ public class DatabaseValidator( /// /// object that contains feedback gathered during the validation process. public ValidationResult Validate() + => Validate(new ValidationOptions()); + + /// + /// Runs database validation using the specified feedback retention options. + /// + public ValidationResult Validate(ValidationOptions options) { - ValidationFeedback feedbacks = []; + ValidationFeedbackSink sink = new(options); ConcurrentBag pxFiles = []; ConcurrentBag aliasFiles = []; List fileTasks = []; @@ -49,9 +55,9 @@ public ValidationResult Validate() { fileTasks.Add(Task.Run(() => { - (DatabaseFileInfo? file, ValidationFeedback feedback) = ProcessPxFile(fileName); + (DatabaseFileInfo? file, ValidationFeedback feedback) = ProcessPxFile(fileName, sink); if (file != null) pxFiles.Add(file); - feedbacks.AddRange(feedback); + sink.ReportRange(feedback); })); } @@ -66,8 +72,8 @@ public ValidationResult Validate() } Task.WaitAll([.. fileTasks]); - feedbacks.AddRange(ValidateDatabaseContents(pxFiles, aliasFiles)); - return new (feedbacks); + sink.ReportRange(ValidateDatabaseContents(pxFiles, aliasFiles)); + return new(sink.ToFeedback()); } /// @@ -76,8 +82,14 @@ public ValidationResult Validate() /// Optional cancellation token /// object that contains feedback gathered during the validation process. public async Task ValidateAsync(CancellationToken cancellationToken = default) + => await ValidateAsync(new ValidationOptions(), cancellationToken); + + /// + /// Runs database validation asynchronously using the specified feedback retention options. + /// + public async Task ValidateAsync(ValidationOptions options, CancellationToken cancellationToken = default) { - ValidationFeedback feedbacks = []; + ValidationFeedbackSink sink = new(options); ConcurrentBag pxFiles = []; ConcurrentBag aliasFiles = []; List fileTasks = []; @@ -87,9 +99,9 @@ public async Task ValidateAsync(CancellationToken cancellation { fileTasks.Add(Task.Run(async () => { - (DatabaseFileInfo? file, ValidationFeedback feedback) = await ProcessPxFileAsync(fileName, cancellationToken); + (DatabaseFileInfo? file, ValidationFeedback feedback) = await ProcessPxFileAsync(fileName, sink, cancellationToken); if (file != null) pxFiles.Add(file); - feedbacks.AddRange(feedback); + sink.ReportRange(feedback); }, cancellationToken)); } @@ -104,11 +116,11 @@ public async Task ValidateAsync(CancellationToken cancellation } await Task.WhenAll(fileTasks); - feedbacks.AddRange(ValidateDatabaseContents(pxFiles, aliasFiles)); - return new (feedbacks); + sink.ReportRange(ValidateDatabaseContents(pxFiles, aliasFiles)); + return new(sink.ToFeedback()); } - private (DatabaseFileInfo?, ValidationFeedback) ProcessPxFile(string fileName) + private (DatabaseFileInfo?, ValidationFeedback) ProcessPxFile(string fileName, ValidationFeedbackSink sink) { ValidationFeedback feedbacks = []; using Stream stream = _fileSystem.GetFileStream(fileName); @@ -120,7 +132,7 @@ public async Task ValidateAsync(CancellationToken cancellation } stream.Position = 0; PxFileValidator validator = new(_conf); - feedbacks.AddRange(validator.Validate(stream, fileName, fileInfo.Encoding).FeedbackItems); + validator.Validate(stream, fileName, fileInfo.Encoding, null, sink); return (fileInfo, feedbacks); } @@ -130,7 +142,7 @@ private DatabaseFileInfo ProcessAliasFile(string fileName) return GetAliasFileInfo(fileName, stream); } - private async Task<(DatabaseFileInfo?, ValidationFeedback)> ProcessPxFileAsync(string fileName, CancellationToken cancellationToken) + private async Task<(DatabaseFileInfo?, ValidationFeedback)> ProcessPxFileAsync(string fileName, ValidationFeedbackSink sink, CancellationToken cancellationToken) { ValidationFeedback feedbacks = []; using Stream stream = _fileSystem.GetFileStream(fileName); @@ -142,8 +154,7 @@ private DatabaseFileInfo ProcessAliasFile(string fileName) } stream.Position = 0; PxFileValidator validator = new(_conf); - ValidationResult result = await validator.ValidateAsync(stream, fileName, fileInfo.Encoding, cancellationToken: cancellationToken); - feedbacks.AddRange(result.FeedbackItems); + await validator.ValidateAsync(stream, fileName, fileInfo.Encoding, null, sink, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); return (fileInfo, feedbacks); } diff --git a/Px.Utils/Validation/PxFileValidator.cs b/Px.Utils/Validation/PxFileValidator.cs index aae28c06..86b687bd 100644 --- a/Px.Utils/Validation/PxFileValidator.cs +++ b/Px.Utils/Validation/PxFileValidator.cs @@ -1,4 +1,4 @@ -using Px.Utils.PxFile; +using Px.Utils.PxFile; using Px.Utils.Validation.ContentValidation; using Px.Utils.Validation.DatabaseValidation; using Px.Utils.Validation.DataValidation; @@ -66,28 +66,44 @@ public ValidationResult Validate( string filename, Encoding? encoding = null, IFileSystem? fileSystem = null) + => Validate(stream, filename, encoding, fileSystem, new ValidationFeedbackSink()); + + /// + /// Validates the PX file using the specified feedback retention options. + /// + public ValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options) + => Validate(stream, filename, encoding, fileSystem, new ValidationFeedbackSink(options)); + + internal ValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink) { encoding ??= new LocalFileSystem().GetEncoding(stream); conf ??= PxFileConfiguration.Default; - ValidationFeedback feedbacks = []; SyntaxValidator syntaxValidator = new(conf, _customSyntaxValidationFunctions); - SyntaxValidationResult syntaxValidationResult = syntaxValidator.Validate(stream, filename, encoding, fileSystem); - feedbacks.AddRange(syntaxValidationResult.FeedbackItems); + SyntaxValidationResult syntaxValidationResult = syntaxValidator.Validate(stream, filename, encoding, fileSystem, sink); ContentValidator contentValidator = new(filename, encoding, [.. syntaxValidationResult.Result], _customContentValidationFunctions, conf); - ContentValidationResult contentValidationResult = contentValidator.Validate(); - feedbacks.AddRange(contentValidationResult.FeedbackItems); + ContentValidationResult contentValidationResult = contentValidator.Validate(sink); if (syntaxValidationResult.DataStartStreamPosition == -1) { - feedbacks.Add(new( + sink.Report(new KeyValuePair( new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), new(filename, 0, 0) )); - return new (feedbacks); + return new(sink.ToFeedback()); } stream.Position = syntaxValidationResult.DataStartStreamPosition; @@ -96,15 +112,14 @@ public ValidationResult Validate( contentValidationResult.DataRowAmount, syntaxValidationResult.DataStartRow, conf); - ValidationResult dataValidationResult = dataValidator.Validate(stream, filename, encoding, fileSystem); - feedbacks.AddRange(dataValidationResult.FeedbackItems); + dataValidator.Validate(stream, filename, encoding, fileSystem, sink); if (_customStreamValidators is not null) { foreach (IPxFileStreamValidator customValidator in _customStreamValidators) { ValidationResult customValidationResult = customValidator.Validate(stream, filename, encoding, fileSystem); - feedbacks.AddRange(customValidationResult.FeedbackItems); + sink.ReportRange(customValidationResult.FeedbackItems); } } if (_customValidators is not null) @@ -112,11 +127,11 @@ public ValidationResult Validate( foreach (IValidator customValidator in _customValidators) { ValidationResult customValidationResult = customValidator.Validate(); - feedbacks.AddRange(customValidationResult.FeedbackItems); + sink.ReportRange(customValidationResult.FeedbackItems); } } stream.Close(); - return new ValidationResult(feedbacks); + return new ValidationResult(sink.ToFeedback()); } /// @@ -135,28 +150,46 @@ public async Task ValidateAsync( Encoding? encoding = null, IFileSystem? fileSystem = null, CancellationToken cancellationToken = default) + => await ValidateAsync(stream, filename, encoding, fileSystem, new ValidationFeedbackSink(), cancellationToken); + + /// + /// Asynchronously validates the PX file using the specified feedback retention options. + /// + public async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options, + CancellationToken cancellationToken = default) + => await ValidateAsync(stream, filename, encoding, fileSystem, new ValidationFeedbackSink(options), cancellationToken); + + internal async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink, + CancellationToken cancellationToken = default) { encoding ??= await new LocalFileSystem().GetEncodingAsync(stream, cancellationToken); conf ??= PxFileConfiguration.Default; - ValidationFeedback feedbacks = []; SyntaxValidator syntaxValidator = new(conf, _customSyntaxValidationFunctions); - SyntaxValidationResult syntaxValidationResult = await syntaxValidator.ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); - feedbacks.AddRange(syntaxValidationResult.FeedbackItems); + SyntaxValidationResult syntaxValidationResult = await syntaxValidator.ValidateAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); ContentValidator contentValidator = new(filename, encoding, [..syntaxValidationResult.Result], _customContentValidationFunctions, conf); - ContentValidationResult contentValidationResult = contentValidator.Validate(); - feedbacks.AddRange(contentValidationResult.FeedbackItems); + ContentValidationResult contentValidationResult = contentValidator.Validate(sink); if (syntaxValidationResult.DataStartStreamPosition == -1) { - feedbacks.Add(new( + sink.Report(new KeyValuePair( new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), new(filename, 0, 0) )); - return new (feedbacks); + return new(sink.ToFeedback()); } stream.Position = syntaxValidationResult.DataStartStreamPosition; @@ -166,15 +199,14 @@ public async Task ValidateAsync( syntaxValidationResult.DataStartRow, conf); - ValidationResult dataValidationResult = await dataValidator.ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); - feedbacks.AddRange(dataValidationResult.FeedbackItems); + await dataValidator.ValidateAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); if (_customStreamAsyncValidators is not null) { foreach (IPxFileStreamValidatorAsync customValidator in _customStreamAsyncValidators) { ValidationResult customValidationResult = await customValidator.ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); - feedbacks.AddRange(customValidationResult.FeedbackItems); + sink.ReportRange(customValidationResult.FeedbackItems); } } if (_customAsyncValidators is not null) @@ -182,11 +214,11 @@ public async Task ValidateAsync( foreach (IValidatorAsync customValidator in _customAsyncValidators) { ValidationResult customValidationResult = await customValidator.ValidateAsync(cancellationToken); - feedbacks.AddRange(customValidationResult.FeedbackItems); + sink.ReportRange(customValidationResult.FeedbackItems); } } stream.Close(); - return new ValidationResult(feedbacks); + return new ValidationResult(sink.ToFeedback()); } } } diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs index 6f1ebcbc..36d3df7f 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs @@ -67,6 +67,32 @@ public SyntaxValidationResult Validate( return new SyntaxValidationResult(validationFeedbacks, structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); } + /// + /// Validates the syntax of a PX file's metadata using the specified feedback retention options. + /// + public SyntaxValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options) + { + ValidationFeedbackSink sink = new(options); + return Validate(stream, filename, encoding, fileSystem, sink); + } + + internal SyntaxValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink) + { + SyntaxValidationResult result = Validate(stream, filename, encoding, fileSystem); + sink.ReportRange(result.FeedbackItems); + return new SyntaxValidationResult(sink.ToFeedback(), [.. result.Result], result.DataStartRow, result.DataStartStreamPosition); + } + /// /// Asynchronously validates the syntax of a PX file's metadata. /// @@ -113,6 +139,34 @@ public async Task ValidateAsync( return new SyntaxValidationResult(validationFeedbacks, structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); } + /// + /// Asynchronously validates the syntax of a PX file's metadata using the specified feedback retention options. + /// + public async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options, + CancellationToken cancellationToken = default) + { + ValidationFeedbackSink sink = new(options); + return await ValidateAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + } + + internal async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink, + CancellationToken cancellationToken = default) + { + SyntaxValidationResult result = await ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); + sink.ReportRange(result.FeedbackItems); + return new SyntaxValidationResult(sink.ToFeedback(), [.. result.Result], result.DataStartRow, result.DataStartStreamPosition); + } + #region Interface implementation ValidationResult IPxFileStreamValidator.Validate(Stream stream, string filename, Encoding? encoding, IFileSystem? fileSystem) diff --git a/Px.Utils/Validation/ValidationFeedbackSink.cs b/Px.Utils/Validation/ValidationFeedbackSink.cs new file mode 100644 index 00000000..d236c8cc --- /dev/null +++ b/Px.Utils/Validation/ValidationFeedbackSink.cs @@ -0,0 +1,100 @@ +using System.Collections.Concurrent; + +namespace Px.Utils.Validation +{ + internal sealed class ValidationFeedbackSink + { + private readonly ConcurrentDictionary _buckets = new(ValidationFeedbackSignatureComparer.Instance); + private readonly int? _maxFeedbackItemsPerSignature; + + public ValidationFeedbackSink(ValidationOptions? options = null) + { + _maxFeedbackItemsPerSignature = options?.MaxFeedbackItemsPerSignature ?? 100; + if (_maxFeedbackItemsPerSignature is <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), "The maximum number of feedback items per signature must be positive or unlimited."); + } + } + + public void Report(KeyValuePair feedback) + => Report(feedback.Key, feedback.Value); + + public void Report(ValidationFeedbackKey key, ValidationFeedbackValue value) + { + ValidationFeedbackSignature signature = new(value.Filename, key.Level, key.Rule); + FeedbackBucket bucket = _buckets.GetOrAdd(signature, static _ => new FeedbackBucket()); + + lock (bucket) + { + if (_maxFeedbackItemsPerSignature is null || bucket.Values.Count < _maxFeedbackItemsPerSignature.Value) + { + bucket.Values.Add(value); + return; + } + + if (!bucket.IsTruncated) + { + ValidationFeedbackValue finalValue = bucket.Values[^1]; + string truncationNote = $"Feedback limit of {_maxFeedbackItemsPerSignature.Value} instances for this file, level, and rule was reached. Additional instances were detected but not logged."; + string additionalInfo = string.IsNullOrEmpty(finalValue.AdditionalInfo) + ? truncationNote + : $"{finalValue.AdditionalInfo}{Environment.NewLine}{truncationNote}"; + bucket.Values[^1] = new ValidationFeedbackValue(finalValue.Filename, finalValue.Line, finalValue.Character, additionalInfo); + bucket.IsTruncated = true; + } + } + } + + public void ReportRange(ValidationFeedback feedback) + { + foreach (KeyValuePair> feedbackGroup in feedback) + { + foreach (ValidationFeedbackValue value in feedbackGroup.Value) + { + Report(feedbackGroup.Key, value); + } + } + } + + public ValidationFeedback ToFeedback() + { + ValidationFeedback feedback = []; + foreach (KeyValuePair pair in _buckets) + { + lock (pair.Value) + { + ValidationFeedbackKey key = new(pair.Key.Level, pair.Key.Rule); + if (!feedback.TryGetValue(key, out List? values)) + { + values = []; + feedback[key] = values; + } + + values.AddRange(pair.Value.Values); + } + } + + return feedback; + } + + private sealed class FeedbackBucket + { + public List Values { get; } = []; + + public bool IsTruncated { get; set; } + } + } + + internal readonly record struct ValidationFeedbackSignature(string Filename, ValidationFeedbackLevel Level, ValidationFeedbackRule Rule); + + internal sealed class ValidationFeedbackSignatureComparer : IEqualityComparer + { + public static ValidationFeedbackSignatureComparer Instance { get; } = new(); + + public bool Equals(ValidationFeedbackSignature x, ValidationFeedbackSignature y) + => x.Level == y.Level && x.Rule == y.Rule && StringComparer.Ordinal.Equals(x.Filename, y.Filename); + + public int GetHashCode(ValidationFeedbackSignature obj) + => HashCode.Combine(StringComparer.Ordinal.GetHashCode(obj.Filename), obj.Level, obj.Rule); + } +} diff --git a/Px.Utils/Validation/ValidationOptions.cs b/Px.Utils/Validation/ValidationOptions.cs new file mode 100644 index 00000000..3f530a21 --- /dev/null +++ b/Px.Utils/Validation/ValidationOptions.cs @@ -0,0 +1,18 @@ +namespace Px.Utils.Validation +{ + /// + /// Configures validation feedback retention. + /// + public sealed class ValidationOptions + { + /// + /// Gets a configuration that retains every feedback item. + /// + public static ValidationOptions Unlimited { get; } = new() { MaxFeedbackItemsPerSignature = null }; + + /// + /// Gets or initializes the maximum number of feedback items retained for each filename, level, and rule signature. A null value retains all feedback items. + /// + public int? MaxFeedbackItemsPerSignature { get; init; } = 100; + } +} From 96a8ab566dde42d7a65ecb1af4062cdf22aa5d62 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Wed, 5 Aug 2026 17:14:26 +0300 Subject: [PATCH 03/12] Documentation update --- docs/README.md | 15 +++++++++++++++ docs/architecture.readers.md | 4 +++- docs/architecture.testing.md | 2 ++ docs/architecture.validation.md | 6 ++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 3114422d..ab00061d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,6 +57,8 @@ The entries need to be in the same key-value format as the output of the ```PxFi **IMPORTANT!** The target map must have the same order as the complete file map. This is for performance reasons, we do not want to move back and forth in the file or generate a second indexer for placing the data in the buffer. +When the reader is created at stream position `0`, it finds the first non-whitespace data value after the top-level `DATA=` entry automatically. This also works with a UTF-8 BOM and multibyte metadata. The overload that accepts `dataStart` expects the absolute raw byte offset of that first value; use `StreamUtilities.FindDataStartPosition()` or `FindDataStartPositionAsync()` to obtain the offset from a seekable stream. Both helpers restore the original stream position and return `-1` when no data value is found. + ### Metadata example ```csharp // Read meta @@ -225,6 +227,14 @@ Validator classes implement either ```IPxFileStreamValidator``` or ```IPxFileStr - encoding (Encoding, optional): Encoding of the px file. Default is Encoding.Default - fileSystem (IFileSystem, optional): Object that defines the file system used for the validation process. Default file called LocalFileSystem system is used if none provided. +#### Feedback retention +All concrete validators provide overloads that accept a `ValidationOptions` instance. By default, validation retains at most 100 feedback items for each filename, feedback level, and rule combination. When a limit is reached, the final retained item is annotated to indicate that additional matching feedback was detected but not logged. Set `MaxFeedbackItemsPerSignature` to a positive number to choose a limit, or use `ValidationOptions.Unlimited` to retain every item. + +```csharp +ValidationOptions options = new() { MaxFeedbackItemsPerSignature = 500 }; +ValidationResult result = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8, null, options); +``` + #### PxFileValidator : IPxFileStreamValidator, IPxFileStreamValidatorAsync ```PxFileValidator``` is a class that validates the whole px file including its data, metadata syntax and metadata contents. The class can be instantiated with the following parameters: - conf (PxFileConfiguration, optional): Object that contains px file configuration. @@ -236,6 +246,7 @@ Once the PxFileValidator object is instantiated, either the Validate or Validate PxFileValidator validator = new PxFileValidator(); ValidationResult result = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8); ValidationResult asyncResult = await validator.ValidateAsync(fileStream, "path/to/file.px", Encoding.UTF8, cancellationToken: cancellationToken); + ValidationResult limitedResult = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8, null, new ValidationOptions { MaxFeedbackItemsPerSignature = 500 }); ``` #### SyntaxValidator : IPxFileStreamValidator, IPxFileStreamValidatorAsync @@ -249,6 +260,7 @@ The class can be instantiated with the following parameters: SyntaxValidator validator = new SyntaxValidator(); SyntaxValidationResult result = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8); SyntaxValidationResult asyncResult = await validator.ValidateAsync(fileStream, "path/to/file.px", Encoding.UTF8, cancellationToken: cancellationToken); + SyntaxValidationResult limitedResult = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8, null, ValidationOptions.Unlimited); ``` #### ContentValidator : IValidator @@ -267,6 +279,7 @@ The class can be instantiated with the following parameters: SyntaxValidationResult syntaxResult = syntaxValidator.Validate(fileStream, "path/to/file.px", encoding); ContentValidator validator = new ContentValidator("path/to/file.px", encoding, syntaxResult.Result); ValidationResult result = validator.Validate(); + ValidationResult limitedResult = validator.Validate(new ValidationOptions { MaxFeedbackItemsPerSignature = 500 }); ``` #### DataValidator : IPxFileStreamValidator, IPxFileStreamValidatorAsync @@ -286,6 +299,7 @@ The class can be instantiated with the following parameters: ValidationResult contentResult = contentValidator.Validate(); DataValidator validator = new DataValidator(contentResult.DataRowLength, contentResult.DataRowAmount, syntaxResult.DataStartRow); ValidationResult result = validator.Validate(fileStream, "path/to/file.px", encoding); + ValidationResult limitedResult = validator.Validate(fileStream, "path/to/file.px", encoding, null, new ValidationOptions { MaxFeedbackItemsPerSignature = 500 }); ``` #### DatabaseValidator : IValidator, IValidatorAsync @@ -305,6 +319,7 @@ The database needs to contain alias files for each language used in the database DatabaseValidator validator = new DatabaseValidator("path/to/database"); ValidationResult result = validator.Validate(); ValidationResult asyncResult = await validator.ValidateAsync(cancellationToken); + ValidationResult limitedResult = validator.Validate(new ValidationOptions { MaxFeedbackItemsPerSignature = 500 }); ``` ### Computing diff --git a/docs/architecture.readers.md b/docs/architecture.readers.md index 635b2e81..62a77b7f 100644 --- a/docs/architecture.readers.md +++ b/docs/architecture.readers.md @@ -32,13 +32,15 @@ void ReadDecimalDataValues(DecimalDataValue[] buffer, int offset, IMatrixMap tar Async variants available. Implements `IDisposable`. Depends on: `PxFileConfiguration`. +When constructed at stream position `0`, `PxFileStreamDataReader` locates the first non-whitespace value following the top-level `DATA=` entry before reading. The overload that accepts `dataStart` requires that value's absolute raw byte offset. + ### Helpers | File | Purpose | |---|---| | `DataIndexer.cs` | Index mapping between source and target `IMatrixMap` | | `DataValueParsers.cs` | Parse raw data values from byte spans | -| `StreamUtilities.cs` | Stream helper methods | +| `StreamUtilities.cs` | `FindDataStartPosition` and async equivalent locate the first value after top-level `DATA=` as an absolute raw byte offset, preserving the original position of a seekable stream | ## Binary Data diff --git a/docs/architecture.testing.md b/docs/architecture.testing.md index b3305d6b..fbcd94ea 100644 --- a/docs/architecture.testing.md +++ b/docs/architecture.testing.md @@ -18,6 +18,7 @@ Framework: MSTest + Moq. Naming: `MethodNameStateUnderTestExpectedBehavior`. | `DataStringValueValidatorTests` | `Validation/DataValidationTests/DataStringValueValidatorTests.cs` | String data validation | | `DataSeparatorValidatorTest` | `Validation/DataValidationTests/DataSeparatorValidatorTest.cs` | Separator validation | | `DataStructureValidationTests` | `Validation/DataValidationTests/DataStructureValidationTests.cs` | Data structure validation | +| `ValidationFeedbackSinkTests` | `Validation/ValidationFeedbackSinkTests.cs` | Per-signature feedback limits, truncation annotations, unlimited retention, and invalid limits | | `DatabaseValidatorTests` | `Validation/DatabaseValidation/DatabaseValidatorTests.cs` | `DatabaseValidator` | | `DatabaseValidatorFunctionTests` | `Validation/DatabaseValidation/DatabaseValidatorFunctionTests.cs` | Database validator functions | | `PxFileValidationTests` | `Validation/PxFileValidationTests/PxFileValidationTests.cs` | `PxFileValidator` | @@ -34,6 +35,7 @@ Framework: MSTest + Moq. Naming: `MethodNameStateUnderTestExpectedBehavior`. | `MultiPartReadingTests` | `PxFileTests/DataTests/PxFileStreamDataReaderTests/MultiPartReadingTests.cs` | Multi-part reads | | `DataIndexerTests` | `PxFileTests/DataTests/DataIndexerTests.cs` | `DataIndexer` | | `DataValueParserTests` | `PxFileTests/DataTests/DataValueParserTests.cs` | `DataValueParsers` | +| `StreamUtilitiesTests` | `PxFileTests/DataTests/StreamUtilitiesTests.cs` | Byte-accurate `DATA=` start offsets with BOM, multibyte metadata, whitespace, buffer splits, and missing data | ### Model & Builder Tests diff --git a/docs/architecture.validation.md b/docs/architecture.validation.md index 84736d06..1afddb53 100644 --- a/docs/architecture.validation.md +++ b/docs/architecture.validation.md @@ -22,12 +22,16 @@ Validate(stream, filename, encoding?, fileSystem?) ValidateAsync(stream, filename, encoding?, fileSystem?, cancellationToken) ``` +Concrete validators also expose `ValidationOptions` overloads. `ValidationOptions.MaxFeedbackItemsPerSignature` defaults to `100` and limits retained feedback by filename, level, and rule; set it to `null` through `ValidationOptions.Unlimited` to retain all feedback. When a limit is exceeded, the last retained item is annotated with a truncation notice. `ValidationFeedbackSink` applies this policy safely while database validation processes files concurrently. + ### SyntaxValidator Validates PX file metadata syntax (key-value structure, encoding, characters). File: `Validation/SyntaxValidation/SyntaxValidator.cs` Partial helpers: `SyntaxValidationFunctions.StringValidationFunctions.cs`, `KeyValueValidationFunctions.cs`, `StructuredValidationFunctions.cs` +Before parsing metadata, it locates the first non-whitespace value after the top-level `DATA=` entry using `StreamUtilities`. `SyntaxValidationResult.DataStartStreamPosition` is the resulting absolute raw byte offset, suitable for direct assignment to `Stream.Position`, or `-1` if no data value is found. + ### ContentValidator Validates metadata content (required keys, language definitions, dimension consistency). @@ -93,6 +97,8 @@ Validation/ ├── IPxFileStreamValidator.cs -- IPxFileStreamValidator, IPxFileStreamValidatorAsync ├── IValidationResult.cs -- ValidationResult ├── ValidationFeedback.cs -- ValidationFeedbackKey, ValidationFeedbackValue, ValidationFeedback +├── ValidationOptions.cs -- Feedback retention configuration +├── ValidationFeedbackSink.cs -- Concurrent feedback retention and truncation ├── ValidationObject.cs -- Validation context ├── Enums.cs -- ValidationFeedbackLevel, ValidationFeedbackRule, ValueType ├── PxFileValidator.cs -- Orchestrator From 792eecfdcdf40c1303dbbc92ad7ace3bf457943c Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 6 Aug 2026 10:10:37 +0300 Subject: [PATCH 04/12] Performance and bug fixes --- .../Commands/PxFileValidationBenchmark.cs | 2 +- .../DataTests/StreamUtilitiesTests.cs | 16 ++ .../ContentValidationTests.cs | 21 +++ .../DataValidationTests/DataValidationTest.cs | 34 ++++ .../StreamSyntaxValidationTests.cs | 18 ++ .../Validation/ValidationFeedbackSinkTests.cs | 19 +- Px.Utils/Px.Utils.csproj | 2 +- Px.Utils/PxFile/Data/StreamUtilities.cs | 68 ++++---- .../ContentValidation/ContentValidator.cs | 38 +++- .../DataValidation/DataValidator.cs | 163 ++++++++++++++++-- .../DatabaseValidation/DatabaseValidator.cs | 39 +++++ Px.Utils/Validation/PxFileValidator.cs | 13 ++ .../SyntaxValidation/SyntaxValidator.cs | 129 +++++++++++++- Px.Utils/Validation/ValidationFeedbackSink.cs | 8 +- docs/README.md | 2 +- docs/architecture.validation.md | 6 +- 16 files changed, 507 insertions(+), 71 deletions(-) diff --git a/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs index 53240ae1..fbdfca81 100644 --- a/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs @@ -43,7 +43,7 @@ private void ValidatePxFileBenchmarks() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); PxFileValidator validator = new(); - var result = validator.Validate(stream, TestFilePath, encoding, null, ValidationOptions); + validator.Validate(stream, TestFilePath, encoding, null, ValidationOptions); } private async Task ValidatePxFileBenchmarksAsync() diff --git a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs index 0da963f7..4bf90aa5 100644 --- a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs +++ b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs @@ -74,5 +74,21 @@ public void FindDataStartPositionDataWithoutValueReturnsNegative1() // Assert Assert.AreEqual(-1, position); } + + [TestMethod] + [DataRow("DATA=;")] + [DataRow("DATA= ;")] + [DataRow("DATA=\r\n\t;")] + public async Task FindDataStartPositionEmptyDataEntryReturnsNegative1(string content) + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + + long synchronousPosition = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 1); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionAsync(stream, PxFileConfiguration.Default, 1); + + Assert.AreEqual(-1, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.AreEqual(0, stream.Position); + } } } diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index 3fcc6928..909b0201 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -49,6 +49,27 @@ public void ValidatePxFileContentCalledWithMinimalStructuredEntryReturnsValidRes Assert.HasCount(0, feedback.FeedbackItems); } + [TestMethod] + public void ValidateWithLimitRepeatedCustomContentFeedbackRetainsOnlyConfiguredCount() + { + const int limit = 2; + ContentValidationEntryValidator entryValidator = static (entry, _) => new(new( + new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.ValueIsNotInUpperCase), + new(entry.File, entry.KeyStartLineIndex))); + CustomContentValidationFunctions functions = new([], [entryValidator]); + ValidationStructuredEntry[] entries = [ + new("content.px", new("ONE", null, null, null), "1", 1, [], 0, null), + new("content.px", new("TWO", null, null, null), "2", 2, [], 0, null), + new("content.px", new("THREE", null, null, null), "3", 3, [], 0, null)]; + ContentValidator validator = new("content.px", Encoding.UTF8, entries, functions); + + ContentValidationResult result = validator.Validate(new ValidationOptions { MaxFeedbackItemsPerSignature = limit }); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.ValueIsNotInUpperCase); + Assert.HasCount(limit, result.FeedbackItems[key]); + Assert.Contains("Feedback limit of 2 instances", result.FeedbackItems[key][^1].AdditionalInfo!); + } + [TestMethod] public void ValidateCalledWithSharedDimensionNameAcrossLanguagesCalculatesRowCountsFromDefaultLanguage() { diff --git a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs index 5813b4f0..88aed54d 100644 --- a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs +++ b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs @@ -3,6 +3,7 @@ using Px.Utils.UnitTests.Validation.Fixtures; using Px.Utils.Validation; using Px.Utils.Validation.DataValidation; +using Px.Utils.PxFile; namespace Px.Utils.UnitTests.Validation.DataValidationTests { @@ -45,6 +46,39 @@ public void ValidateDataReturnsExpectedErrorCount( Assert.AreEqual(expectedTotalErrorCount, actualErrorCount); } + [TestMethod] + public async Task ValidateAndValidateAsyncCustomDataKeywordAtStreamOriginFindDataSection() + { + PxFileConfiguration configuration = PxFileConfiguration.Default; + configuration.Tokens.KeyWords.Data = "VALUES"; + byte[] data = Encoding.UTF8.GetBytes("TITLE=\"test\";VALUES=1;"); + DataValidator validator = new(1, 0, 0, configuration); + + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + ValidationResult synchronousResult = validator.Validate(synchronousStream, "custom.px", Encoding.UTF8); + ValidationResult asynchronousResult = await validator.ValidateAsync(asynchronousStream, "custom.px", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound); + Assert.IsFalse(synchronousResult.FeedbackItems.ContainsKey(key)); + Assert.IsFalse(asynchronousResult.FeedbackItems.ContainsKey(key)); + } + + [TestMethod] + public void ValidateWithLimitLargeInvalidDataRetainsOnlyConfiguredFeedbackCount() + { + const int limit = 3; + string invalidData = string.Concat(Enumerable.Repeat("! ", 200)) + ";"; + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes("DATA=" + invalidData)); + DataValidator validator = new(0, 0, 0); + + ValidationResult result = validator.Validate(stream, "invalid.px", Encoding.UTF8, null, new ValidationOptions { MaxFeedbackItemsPerSignature = limit }); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + Assert.HasCount(limit, result.FeedbackItems[key]); + Assert.Contains("Feedback limit of 3 instances", result.FeedbackItems[key][^1].AdditionalInfo!); + } + [TestMethod] [DataRow(DataStreamContents.SIMPLE_VALID_DATA, 0, 0)] [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] diff --git a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs index d2eee2e1..b42b52a1 100644 --- a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs @@ -79,6 +79,24 @@ public async Task ValidateAndValidateAsyncDataWithBomAndMultibyteMetadataReturnS Assert.AreEqual('2', (char)synchronousStream.ReadByte()); } + [TestMethod] + public void ValidateWithLimitRepeatedCustomSyntaxFeedbackRetainsOnlyConfiguredCount() + { + const int limit = 2; + List entryFunctions = [static (entry, _) => new( + new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.MultipleEntriesOnOneLine), + new(entry.File, entry.KeyStartLineIndex))]; + CustomSyntaxValidationFunctions functions = new(entryFunctions, [], []); + SyntaxValidator validator = new(customValidationFunctions: functions); + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes("A=1;B=2;C=3;D=4;DATA=1;")); + + SyntaxValidationResult result = validator.Validate(stream, "syntax.px", Encoding.UTF8, null, new ValidationOptions { MaxFeedbackItemsPerSignature = limit }); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.MultipleEntriesOnOneLine); + Assert.HasCount(limit, result.FeedbackItems[key]); + Assert.Contains("Feedback limit of 2 instances", result.FeedbackItems[key][^1].AdditionalInfo); + } + [TestMethod] public void ValidateObjectsCalledWithMultipleEntriesInSingleLineReturnsWithWarnings() { diff --git a/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs b/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs index a6ce79a0..04a3ad47 100644 --- a/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs +++ b/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs @@ -19,9 +19,9 @@ public void ReportMatchingFeedbackBeyondLimitRetainsLimitAndAnnotatesFinalItem() ValidationFeedback feedback = sink.ToFeedback(); List values = feedback[key]; - Assert.AreEqual(2, values.Count); - StringAssert.Contains(values[1].AdditionalInfo, "Original information."); - StringAssert.Contains(values[1].AdditionalInfo, "Feedback limit of 2 instances"); + Assert.HasCount(2, values); + Assert.Contains("Original information.", values[1].AdditionalInfo); + Assert.Contains("Feedback limit of 2 instances", values[1].AdditionalInfo); } [TestMethod] @@ -30,11 +30,14 @@ public void ReportUnlimitedFeedbackRetainsAllItems() ValidationFeedbackSink sink = new(ValidationOptions.Unlimited); ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); - sink.Report(key, new ValidationFeedbackValue("file.px", 1)); - sink.Report(key, new ValidationFeedbackValue("file.px", 2)); - sink.Report(key, new ValidationFeedbackValue("file.px", 3)); + for (int line = 1; line <= 101; line++) + { + sink.Report(key, new ValidationFeedbackValue("file.px", line)); + } - Assert.AreEqual(3, sink.ToFeedback()[key].Count); + List values = sink.ToFeedback()[key]; + Assert.HasCount(101, values); + Assert.DoesNotContain(value => value.AdditionalInfo?.Contains("Feedback limit", StringComparison.Ordinal) == true, values); } [TestMethod] @@ -52,7 +55,7 @@ public void ReportDifferentSignaturesRetainsSeparateLimits() ValidationFeedback feedback = sink.ToFeedback(); Assert.AreEqual(2, feedback[errorKey].Select(value => value.Filename).Distinct(StringComparer.Ordinal).Count()); - Assert.AreEqual(1, feedback[warningKey].Count); + Assert.HasCount(1, feedback[warningKey]); } [TestMethod] diff --git a/Px.Utils/Px.Utils.csproj b/Px.Utils/Px.Utils.csproj index 2e855475..146ca665 100644 --- a/Px.Utils/Px.Utils.csproj +++ b/Px.Utils/Px.Utils.csproj @@ -2,7 +2,7 @@ Px.Utils - 1.5.0 + 1.6.0 net10.0 enable enable diff --git a/Px.Utils/PxFile/Data/StreamUtilities.cs b/Px.Utils/PxFile/Data/StreamUtilities.cs index 5c25e034..e0e4f456 100644 --- a/Px.Utils/PxFile/Data/StreamUtilities.cs +++ b/Px.Utils/PxFile/Data/StreamUtilities.cs @@ -57,7 +57,11 @@ private static long FindDataStartPositionImpl(Stream stream, PxFileConfiguration { byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); byte[] buffer = new byte[bufferSize]; - DataStartSearchState state = new(); + DataStartSearchState state = new( + dataKeywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.KeywordSeparator, + (byte)conf.Symbols.Key.StringDelimeter); int bytesRead; while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) @@ -66,10 +70,6 @@ private static long FindDataStartPositionImpl(Stream stream, PxFileConfiguration if (TryFindDataStartPosition( buffer.AsSpan(0, bytesRead), bufferStart, - dataKeywordBytes, - (byte)conf.Symbols.EntrySeparator, - (byte)conf.Symbols.KeywordSeparator, - (byte)conf.Symbols.Key.StringDelimeter, ref state, out long dataStartPosition)) { @@ -84,7 +84,11 @@ private static async Task FindDataStartPositionImplAsync(Stream stream, Px { byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); byte[] buffer = new byte[bufferSize]; - DataStartSearchState state = new(); + DataStartSearchState state = new( + dataKeywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.KeywordSeparator, + (byte)conf.Symbols.Key.StringDelimeter); int bytesRead; while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) @@ -93,10 +97,6 @@ private static async Task FindDataStartPositionImplAsync(Stream stream, Px if (TryFindDataStartPosition( buffer.AsSpan(0, bytesRead), bufferStart, - dataKeywordBytes, - (byte)conf.Symbols.EntrySeparator, - (byte)conf.Symbols.KeywordSeparator, - (byte)conf.Symbols.Key.StringDelimeter, ref state, out long dataStartPosition)) { @@ -111,20 +111,21 @@ private static async Task FindDataStartPositionImplAsync(Stream stream, Px private static bool TryFindDataStartPosition( ReadOnlySpan buffer, long bufferStart, - ReadOnlySpan dataKeywordBytes, - byte entrySeparator, - byte keywordSeparator, - byte stringDelimiter, ref DataStartSearchState state, out long dataStartPosition) { for (int i = 0; i < buffer.Length; i++) { - if (TryProcessDataStartByte(buffer[i], dataKeywordBytes, entrySeparator, keywordSeparator, stringDelimiter, ref state)) + if (TryProcessDataStartByte(buffer[i], ref state)) { dataStartPosition = bufferStart + i; return true; } + if (state.IsDataEntryEmpty) + { + dataStartPosition = -1; + return true; + } } dataStartPosition = -1; @@ -132,40 +133,39 @@ private static bool TryFindDataStartPosition( } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool TryProcessDataStartByte( - byte currentByte, - ReadOnlySpan dataKeywordBytes, - byte entrySeparator, - byte keywordSeparator, - byte stringDelimiter, - ref DataStartSearchState state) + private static bool TryProcessDataStartByte(byte currentByte, ref DataStartSearchState state) { if (state.IsAfterDataKeyword) { + if (currentByte == state.EntrySeparator) + { + state.IsDataEntryEmpty = true; + return false; + } return !IsWhitespace(currentByte); } if (state.IsInString) { - state.IsInString = currentByte != stringDelimiter; + state.IsInString = currentByte != state.StringDelimiter; return false; } - if (currentByte == stringDelimiter) + if (currentByte == state.StringDelimiter) { state.IsInString = true; return false; } - if (currentByte == entrySeparator) + if (currentByte == state.EntrySeparator) { state.ResetEntry(); return false; } if (state.IsAtEntryStart && IsWhitespace(currentByte)) return false; - if (state.IsAtEntryStart && state.MatchedKeywordBytes < dataKeywordBytes.Length && currentByte == dataKeywordBytes[state.MatchedKeywordBytes]) + if (state.IsAtEntryStart && state.MatchedKeywordBytes < state.DataKeywordBytes.Length && currentByte == state.DataKeywordBytes[state.MatchedKeywordBytes]) { state.MatchedKeywordBytes++; return false; } - if (state.MatchedKeywordBytes == dataKeywordBytes.Length && currentByte == keywordSeparator) + if (state.MatchedKeywordBytes == state.DataKeywordBytes.Length && currentByte == state.KeywordSeparator) { state.IsAfterDataKeyword = true; return false; @@ -182,17 +182,17 @@ private static bool IsWhitespace(byte value) return value is CharacterConstants.SPACE or CharacterConstants.HORIZONTALTAB or CharacterConstants.CARRIAGERETURN or CharacterConstants.LINEFEED; } - private struct DataStartSearchState + private struct DataStartSearchState(byte[] dataKeywordBytes, byte entrySeparator, byte keywordSeparator, byte stringDelimiter) { + public readonly byte[] DataKeywordBytes = dataKeywordBytes; + public readonly byte EntrySeparator = entrySeparator; + public readonly byte KeywordSeparator = keywordSeparator; + public readonly byte StringDelimiter = stringDelimiter; public int MatchedKeywordBytes; - public bool IsAtEntryStart; + public bool IsAtEntryStart = true; public bool IsInString; public bool IsAfterDataKeyword; - - public DataStartSearchState() - { - IsAtEntryStart = true; - } + public bool IsDataEntryEmpty; [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ResetEntry() diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.cs index edf8e344..1a1a2883 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.cs @@ -96,6 +96,8 @@ public ContentValidationResult Validate() /// /// Validates contents of PX file metadata using the specified feedback retention options. /// + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The content validation result with retained feedback and calculated data dimensions. public ContentValidationResult Validate(ValidationOptions options) { ValidationFeedbackSink sink = new(options); @@ -104,9 +106,39 @@ public ContentValidationResult Validate(ValidationOptions options) internal ContentValidationResult Validate(ValidationFeedbackSink sink) { - ContentValidationResult result = Validate(); - sink.ReportRange(result.FeedbackItems); - return new ContentValidationResult(sink.ToFeedback(), result.DataRowLength, result.DataRowAmount); + IEnumerable contentValidationEntryFunctions = DefaultContentValidationEntryFunctions; + IEnumerable contentValidationFindKeywordFunctions = DefaultContentValidationFindKeywordFunctions; + + if (customContentValidationFunctions is not null) + { + contentValidationEntryFunctions = contentValidationEntryFunctions.Concat(customContentValidationFunctions.CustomContentValidationEntryFunctions); + contentValidationFindKeywordFunctions = contentValidationFindKeywordFunctions.Concat(customContentValidationFunctions.CustomContentValidationFindKeywordFunctions); + } + + foreach (ContentValidationFindKeywordValidator findingFunction in contentValidationFindKeywordFunctions) + { + ValidationFeedback? feedback = findingFunction(entries, this); + if (feedback is not null) + { + sink.ReportRange(feedback); + } + } + foreach (ContentValidationEntryValidator entryFunction in contentValidationEntryFunctions) + { + foreach (ValidationStructuredEntry entry in entries) + { + ValidationFeedback? feedback = entryFunction(entry, this); + if (feedback is not null) + { + sink.ReportRange(feedback); + } + } + } + int lengthOfDataRows = _headingDimensionNames is not null ? GetProductOfDimensionValues(_headingDimensionNames) : 0; + int amountOfDataRows = _stubDimensionNames is not null ? GetProductOfDimensionValues(_stubDimensionNames) : 0; + ResetFields(); + + return new ContentValidationResult(sink.ToFeedback(), lengthOfDataRows, amountOfDataRows); } #region Interface implementation diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index 3d18c406..688998b8 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -80,6 +80,12 @@ public ValidationResult Validate( /// /// Validates data using the specified feedback retention options. /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The data validation result with retained feedback. public ValidationResult Validate( Stream stream, string filename, @@ -98,14 +104,37 @@ internal ValidationResult Validate( IFileSystem? fileSystem, ValidationFeedbackSink sink) { - ValidationResult result = Validate(stream, filename, encoding, fileSystem); - sink.ReportRange(result.FeedbackItems); + fileSystem ??= new LocalFileSystem(); + encoding ??= fileSystem.GetEncoding(stream); + SetValidationParameters(encoding, filename); + + long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); + if (dataStartIndex == -1) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), + new(filename, 0, 0))); + ResetValidator(); + return new ValidationResult(sink.ToFeedback()); + } + + stream.Position = dataStartIndex; + ValidateDataStream(stream, sink); + ResetValidator(); + return new ValidationResult(sink.ToFeedback()); } /// /// Asynchronously validates data using the specified feedback retention options. /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// A token that cancels the operation. + /// A task that produces the data validation result with retained feedback. public async Task ValidateAsync( Stream stream, string filename, @@ -126,24 +155,36 @@ internal async Task ValidateAsync( ValidationFeedbackSink sink, CancellationToken cancellationToken = default) { - ValidationResult result = await ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); - sink.ReportRange(result.FeedbackItems); + fileSystem ??= new LocalFileSystem(); + encoding ??= await fileSystem.GetEncodingAsync(stream, cancellationToken); + SetValidationParameters(encoding, filename); + + long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); + if (dataStartIndex == -1) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), + new(filename, 0, 0))); + ResetValidator(); + return new ValidationResult(sink.ToFeedback()); + } + + stream.Position = dataStartIndex; + await Task.Factory.StartNew(() => ValidateDataStream(stream, sink, cancellationToken), cancellationToken); + ResetValidator(); + return new ValidationResult(sink.ToFeedback()); } /// /// Validates the data in the specified stream asynchronously. /// Assumes that the stream is at the start of the data section (after 'DATA='-keyword) at the first data item. - /// /// Px file stream to be validated - /// Encoding of the stream /// Name of the file being validated. If not provided, validator tries to find the encoding. + /// Encoding of the stream. /// File system used for file operations. If not provided, default file system is used. - /// Cancellation token for cancelling the validation process - /// - /// object that contains a collection of - /// validation feedback key value pairs representing the feedback for the data validation. - /// + /// Cancellation token for cancelling the validation process. + /// A task that produces validation feedback for the data section. public async Task ValidateAsync( Stream stream, string filename, @@ -241,6 +282,52 @@ private ValidationFeedback ValidateDataStream(Stream stream, CancellationToken? return validationFeedbacks; } + private void ValidateDataStream(Stream stream, ValidationFeedbackSink sink, CancellationToken? cancellationToken = null) + { + byte endOfData = (byte)_conf.Symbols.EntrySeparator; + _currentEntry = new(_streamBufferSize); + byte[] buffer = new byte[_streamBufferSize]; + int bytesRead = 0; + + do + { + cancellationToken?.ThrowIfCancellationRequested(); + for (int i = 0; i < bytesRead; i++) + { + byte currentByte = buffer[i]; + _currentCharacterType = currentByte switch + { + CharacterConstants.SPACE or CharacterConstants.HORIZONTALTAB => EntryType.DataItemSeparator, + CharacterConstants.LINEFEED or CharacterConstants.CARRIAGERETURN => EntryType.LineSeparator, + >= CharacterConstants.QUOTATIONMARK and not CharacterConstants.SEMICOLON => EntryType.DataItem, + _ when currentByte == endOfData => EntryType.EndOfData, + _ => EntryType.Unknown + }; + if (_currentCharacterType != _currentEntryType) + { + HandleEntryTypeChange(sink); + if (_currentCharacterType != EntryType.DataItemSeparator) + { + HandleNonSeparatorType(sink); + } + _currentEntryType = _currentCharacterType; + _currentEntry.Clear(); + } + + _currentEntry.Add(currentByte); + _charPosition++; + } + } + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0); + + if (numOfRows != _lineNumber - 1) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidRowCount), + new(_filename, _lineNumber + startRow, _charPosition, $" Expected {numOfRows} rows, got {_lineNumber - 1} rows."))); + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void HandleEntryTypeChange(ref ValidationFeedback validationFeedbacks) { @@ -278,6 +365,35 @@ private void HandleEntryTypeChange(ref ValidationFeedback validationFeedbacks) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void HandleEntryTypeChange(ValidationFeedbackSink sink) + { + if (_currentEntryType == EntryType.Unknown && (_lineNumber > 1 || _charPosition > 0)) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar), + new(_filename, _lineNumber + startRow, _charPosition))); + return; + } + + List validators = _currentEntryType switch + { + EntryType.DataItemSeparator => _dataSeparatorValidators, + EntryType.DataItem => _currentEntry[^1] > '.' ? _dataNumValidators : _dataStringValidators, + _ => _commonValidators + }; + + foreach (IDataValidator validator in validators) + { + KeyValuePair? feedback = validator.Validate( + _currentEntry, _currentEntryType, _encoding, _lineNumber + startRow, _charPosition, _filename); + if (feedback is not null) + { + sink.Report(feedback.Value); + } + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void HandleNonSeparatorType(ref ValidationFeedback validationFeedbacks) { @@ -302,6 +418,27 @@ private void HandleNonSeparatorType(ref ValidationFeedback validationFeedbacks) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void HandleNonSeparatorType(ValidationFeedbackSink sink) + { + if (_currentCharacterType == EntryType.DataItem) + { + _currentRowLength++; + } + else if (_currentCharacterType == EntryType.LineSeparator) + { + if (_currentRowLength != rowLen) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidRowLength), + new(_filename, _lineNumber + startRow, _charPosition, $"Expected {rowLen}, got row length of {_currentRowLength}."))); + } + _lineNumber++; + _currentRowLength = 0; + _charPosition = 0; + } + } + private void ResetValidator() { _commonValidators.Clear(); @@ -315,11 +452,11 @@ private void ResetValidator() _currentRowLength = 0; } - private static long GetStreamIndexOfFirstDataValue(Stream stream) + private long GetStreamIndexOfFirstDataValue(Stream stream) { if (stream.Position == 0) { - return StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, _streamBufferSize); + return StreamUtilities.FindDataStartPosition(stream, _conf, _streamBufferSize); } int currentByte; diff --git a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs index ebaf26b2..60b00f63 100644 --- a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs +++ b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs @@ -43,6 +43,8 @@ public ValidationResult Validate() /// /// Runs database validation using the specified feedback retention options. /// + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The database validation result with retained feedback. public ValidationResult Validate(ValidationOptions options) { ValidationFeedbackSink sink = new(options); @@ -87,6 +89,9 @@ public async Task ValidateAsync(CancellationToken cancellation /// /// Runs database validation asynchronously using the specified feedback retention options. /// + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// A token that cancels the operation. + /// A task that produces the database validation result with retained feedback. public async Task ValidateAsync(ValidationOptions options, CancellationToken cancellationToken = default) { ValidationFeedbackSink sink = new(options); @@ -424,21 +429,55 @@ private async Task GetAliasFileInfoAsync(string filename, Stre } } + /// + /// Represents a database validation item. + /// + /// The path of the database validation item. public class DatabaseValidationItem(string path) { + /// + /// Gets the path of the file or directory. + /// public string Path { get; } = path; } + /// + /// Represents a px file or alias file within a px file database for validation purposes. + /// + /// Name of the file. + /// Path of the file's directory. + /// Languages associated with the file. + /// Encoding of the file. public class DatabaseFileInfo(string name, string location, string[] languages, Encoding encoding) : DatabaseValidationItem(name) { + /// + /// Gets the name of the file. + /// public string Name { get; } = name; + /// + /// Gets the path of the file's directory. + /// public string Location { get; } = location; + /// + /// Gets the languages associated with the file. + /// public string[] Languages { get; } = languages; + /// + /// Gets the encoding of the file. + /// public Encoding Encoding { get; } = encoding; } + /// + /// Validator interface for validating a px file database including px files, alias files, and directory structures. + /// public interface IDatabaseValidator { + /// + /// Validates a given and returns a feedback entry if validation fails, or if validation passes. + /// + /// The database validation item to validate. + /// A key-value pair representing the validation feedback if validation fails, or if validation passes. public KeyValuePair? Validate(DatabaseValidationItem item); } } diff --git a/Px.Utils/Validation/PxFileValidator.cs b/Px.Utils/Validation/PxFileValidator.cs index 86b687bd..85062ee5 100644 --- a/Px.Utils/Validation/PxFileValidator.cs +++ b/Px.Utils/Validation/PxFileValidator.cs @@ -71,6 +71,12 @@ public ValidationResult Validate( /// /// Validates the PX file using the specified feedback retention options. /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The complete PX file validation result with retained feedback. public ValidationResult Validate( Stream stream, string filename, @@ -155,6 +161,13 @@ public async Task ValidateAsync( /// /// Asynchronously validates the PX file using the specified feedback retention options. /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// A token that cancels the operation. + /// A task that produces the complete PX file validation result with retained feedback. public async Task ValidateAsync( Stream stream, string filename, diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs index 36d3df7f..dd0b2af0 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs @@ -70,6 +70,12 @@ public SyntaxValidationResult Validate( /// /// Validates the syntax of a PX file's metadata using the specified feedback retention options. /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The syntax result with retained validation feedback and parsed metadata entries. public SyntaxValidationResult Validate( Stream stream, string filename, @@ -88,9 +94,33 @@ internal SyntaxValidationResult Validate( IFileSystem? fileSystem, ValidationFeedbackSink sink) { - SyntaxValidationResult result = Validate(stream, filename, encoding, fileSystem); - sink.ReportRange(result.FeedbackItems); - return new SyntaxValidationResult(sink.ToFeedback(), [.. result.Result], result.DataStartRow, result.DataStartStreamPosition); + fileSystem ??= new LocalFileSystem(); + encoding ??= fileSystem.GetEncoding(stream); + + SyntaxValidationFunctions validationFunctions = new(); + IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; + IEnumerable keyValueValidationFunctions = validationFunctions.DefaultKeyValueValidationFunctions; + IEnumerable structuredValidationFunctions = validationFunctions.DefaultStructuredValidationFunctions; + + if (customValidationFunctions is not null) + { + stringValidationFunctions = stringValidationFunctions.Concat(customValidationFunctions.CustomStringValidationFunctions); + keyValueValidationFunctions = keyValueValidationFunctions.Concat(customValidationFunctions.CustomKeyValueValidationFunctions); + structuredValidationFunctions = structuredValidationFunctions.Concat(customValidationFunctions.CustomStructuredValidationFunctions); + } + + conf ??= PxFileConfiguration.Default; + ResetDataSectionPosition(); + _dataSectionStartStreamPosition = StreamUtilities.FindDataStartPosition(stream, conf, _bufferSize); + + List stringEntries = BuildValidationEntries(stream, encoding, conf, filename, _bufferSize); + ReportEntryFeedback(stringEntries, stringValidationFunctions, conf, sink); + List keyValuePairs = BuildKeyValuePairs(stringEntries, conf); + ReportKeyValuePairFeedback(keyValuePairs, keyValueValidationFunctions, conf, sink); + List structuredEntries = BuildValidationStructureEntries(keyValuePairs, conf); + ReportStructuredFeedback(structuredEntries, structuredValidationFunctions, conf, sink); + + return new SyntaxValidationResult(sink.ToFeedback(), structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); } /// @@ -142,6 +172,13 @@ public async Task ValidateAsync( /// /// Asynchronously validates the syntax of a PX file's metadata using the specified feedback retention options. /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// A token that cancels the operation. + /// A task that produces the syntax result with retained validation feedback and parsed metadata entries. public async Task ValidateAsync( Stream stream, string filename, @@ -162,9 +199,32 @@ internal async Task ValidateAsync( ValidationFeedbackSink sink, CancellationToken cancellationToken = default) { - SyntaxValidationResult result = await ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); - sink.ReportRange(result.FeedbackItems); - return new SyntaxValidationResult(sink.ToFeedback(), [.. result.Result], result.DataStartRow, result.DataStartStreamPosition); + fileSystem ??= new LocalFileSystem(); + encoding ??= await fileSystem.GetEncodingAsync(stream, cancellationToken); + + SyntaxValidationFunctions validationFunctions = new(); + IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; + IEnumerable keyValueValidationFunctions = validationFunctions.DefaultKeyValueValidationFunctions; + IEnumerable structuredValidationFunctions = validationFunctions.DefaultStructuredValidationFunctions; + + if (customValidationFunctions is not null) + { + stringValidationFunctions = stringValidationFunctions.Concat(customValidationFunctions.CustomStringValidationFunctions); + keyValueValidationFunctions = keyValueValidationFunctions.Concat(customValidationFunctions.CustomKeyValueValidationFunctions); + structuredValidationFunctions = structuredValidationFunctions.Concat(customValidationFunctions.CustomStructuredValidationFunctions); + } + + conf ??= PxFileConfiguration.Default; + ResetDataSectionPosition(); + _dataSectionStartStreamPosition = await StreamUtilities.FindDataStartPositionAsync(stream, conf, _bufferSize, cancellationToken); + List entries = await BuildValidationEntriesAsync(stream, encoding, conf, filename, _bufferSize, cancellationToken); + ReportEntryFeedback(entries, stringValidationFunctions, conf, sink); + List keyValuePairs = BuildKeyValuePairs(entries, conf); + ReportKeyValuePairFeedback(keyValuePairs, keyValueValidationFunctions, conf, sink); + List structuredEntries = BuildValidationStructureEntries(keyValuePairs, conf); + ReportStructuredFeedback(structuredEntries, structuredValidationFunctions, conf, sink); + + return new SyntaxValidationResult(sink.ToFeedback(), structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); } #region Interface implementation @@ -219,6 +279,25 @@ private static ValidationFeedback ValidateEntries(IEnumerable e return validationFeedback; } + private static void ReportEntryFeedback( + IEnumerable entries, + IEnumerable validationFunctions, + PxFileConfiguration syntaxConf, + ValidationFeedbackSink sink) + { + foreach (ValidationEntry entry in entries) + { + foreach (EntryValidationFunction function in validationFunctions) + { + KeyValuePair? feedback = function(entry, syntaxConf); + if (feedback is not null) + { + sink.Report(feedback.Value); + } + } + } + } + private static ValidationFeedback ValidateKeyValuePairs( IEnumerable kvpObjects, IEnumerable validationFunctions, @@ -239,6 +318,25 @@ private static ValidationFeedback ValidateKeyValuePairs( return validationFeedback; } + private static void ReportKeyValuePairFeedback( + IEnumerable kvpObjects, + IEnumerable validationFunctions, + PxFileConfiguration syntaxConf, + ValidationFeedbackSink sink) + { + foreach (ValidationKeyValuePair kvpObject in kvpObjects) + { + foreach (KeyValuePairValidationFunction function in validationFunctions) + { + KeyValuePair? feedback = function(kvpObject, syntaxConf); + if (feedback is not null) + { + sink.Report(feedback.Value); + } + } + } + } + private static ValidationFeedback ValidateStructs( IEnumerable structuredEntries, IEnumerable validationFunctions, @@ -259,6 +357,25 @@ private static ValidationFeedback ValidateStructs( return validationFeedback; } + private static void ReportStructuredFeedback( + IEnumerable structuredEntries, + IEnumerable validationFunctions, + PxFileConfiguration syntaxConf, + ValidationFeedbackSink sink) + { + foreach (ValidationStructuredEntry structuredEntry in structuredEntries) + { + foreach (StructuredValidationFunction function in validationFunctions) + { + KeyValuePair? feedback = function(structuredEntry, syntaxConf); + if (feedback is not null) + { + sink.Report(feedback.Value); + } + } + } + } + private static List BuildKeyValuePairs(List validationEntries, PxFileConfiguration syntaxConf) { return validationEntries.Select(entry => diff --git a/Px.Utils/Validation/ValidationFeedbackSink.cs b/Px.Utils/Validation/ValidationFeedbackSink.cs index d236c8cc..1947563f 100644 --- a/Px.Utils/Validation/ValidationFeedbackSink.cs +++ b/Px.Utils/Validation/ValidationFeedbackSink.cs @@ -4,12 +4,15 @@ namespace Px.Utils.Validation { internal sealed class ValidationFeedbackSink { + private const int DEFAULT_MAX_FEEDBACK_ITEMS_PER_SIGNATURE = 100; private readonly ConcurrentDictionary _buckets = new(ValidationFeedbackSignatureComparer.Instance); private readonly int? _maxFeedbackItemsPerSignature; public ValidationFeedbackSink(ValidationOptions? options = null) { - _maxFeedbackItemsPerSignature = options?.MaxFeedbackItemsPerSignature ?? 100; + _maxFeedbackItemsPerSignature = options is null + ? DEFAULT_MAX_FEEDBACK_ITEMS_PER_SIGNATURE + : options.MaxFeedbackItemsPerSignature; if (_maxFeedbackItemsPerSignature is <= 0) { throw new ArgumentOutOfRangeException(nameof(options), "The maximum number of feedback items per signature must be positive or unlimited."); @@ -35,7 +38,8 @@ public void Report(ValidationFeedbackKey key, ValidationFeedbackValue value) if (!bucket.IsTruncated) { ValidationFeedbackValue finalValue = bucket.Values[^1]; - string truncationNote = $"Feedback limit of {_maxFeedbackItemsPerSignature.Value} instances for this file, level, and rule was reached. Additional instances were detected but not logged."; + string truncationNote = $"Feedback limit of {_maxFeedbackItemsPerSignature.Value} instances for this file, level, and rule was reached. " + + $"Additional instances were detected but not logged."; string additionalInfo = string.IsNullOrEmpty(finalValue.AdditionalInfo) ? truncationNote : $"{finalValue.AdditionalInfo}{Environment.NewLine}{truncationNote}"; diff --git a/docs/README.md b/docs/README.md index ab00061d..9059f2ae 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,7 +57,7 @@ The entries need to be in the same key-value format as the output of the ```PxFi **IMPORTANT!** The target map must have the same order as the complete file map. This is for performance reasons, we do not want to move back and forth in the file or generate a second indexer for placing the data in the buffer. -When the reader is created at stream position `0`, it finds the first non-whitespace data value after the top-level `DATA=` entry automatically. This also works with a UTF-8 BOM and multibyte metadata. The overload that accepts `dataStart` expects the absolute raw byte offset of that first value; use `StreamUtilities.FindDataStartPosition()` or `FindDataStartPositionAsync()` to obtain the offset from a seekable stream. Both helpers restore the original stream position and return `-1` when no data value is found. +When the reader is created at stream position `0`, it finds the first non-whitespace data value after the top-level `DATA=` entry automatically. This also works with a UTF-8 BOM and multibyte metadata. The overload that accepts `dataStart` expects the absolute raw byte offset of that first value; use `StreamUtilities.FindDataStartPosition()` or `FindDataStartPositionAsync()` to obtain the offset from a seekable stream. Both helpers restore the original stream position and return `-1` when no data value is found, including an explicitly empty entry such as `DATA=;`. ### Metadata example ```csharp diff --git a/docs/architecture.validation.md b/docs/architecture.validation.md index 1afddb53..e2f0fe3a 100644 --- a/docs/architecture.validation.md +++ b/docs/architecture.validation.md @@ -22,7 +22,7 @@ Validate(stream, filename, encoding?, fileSystem?) ValidateAsync(stream, filename, encoding?, fileSystem?, cancellationToken) ``` -Concrete validators also expose `ValidationOptions` overloads. `ValidationOptions.MaxFeedbackItemsPerSignature` defaults to `100` and limits retained feedback by filename, level, and rule; set it to `null` through `ValidationOptions.Unlimited` to retain all feedback. When a limit is exceeded, the last retained item is annotated with a truncation notice. `ValidationFeedbackSink` applies this policy safely while database validation processes files concurrently. +Concrete validators also expose `ValidationOptions` overloads. `ValidationOptions.MaxFeedbackItemsPerSignature` defaults to `100` and limits retained feedback by filename, level, and rule; set it to `null` through `ValidationOptions.Unlimited` to retain all feedback. Validators report each discovered finding directly to `ValidationFeedbackSink`, so configured limits bound retained feedback during syntax, content, and data scans instead of only truncating completed results. When a limit is exceeded, the last retained item is annotated with a truncation notice. `ValidationFeedbackSink` applies this policy safely while database validation processes files concurrently. ### SyntaxValidator @@ -30,7 +30,7 @@ Validates PX file metadata syntax (key-value structure, encoding, characters). File: `Validation/SyntaxValidation/SyntaxValidator.cs` Partial helpers: `SyntaxValidationFunctions.StringValidationFunctions.cs`, `KeyValueValidationFunctions.cs`, `StructuredValidationFunctions.cs` -Before parsing metadata, it locates the first non-whitespace value after the top-level `DATA=` entry using `StreamUtilities`. `SyntaxValidationResult.DataStartStreamPosition` is the resulting absolute raw byte offset, suitable for direct assignment to `Stream.Position`, or `-1` if no data value is found. +Before parsing metadata, it locates the first non-whitespace value after the top-level `DATA=` entry using `StreamUtilities`. `SyntaxValidationResult.DataStartStreamPosition` is the resulting absolute raw byte offset, suitable for direct assignment to `Stream.Position`, or `-1` if no data value is found. An empty entry such as `DATA=;` is treated as missing data and produces error-level `StartOfDataSectionNotFound` feedback in file and standalone data validation. ### ContentValidator @@ -45,6 +45,8 @@ Dimension type validation uses `PxFileConfiguration.TokenDefinitions.VariableTyp Validates data section (row counts, row lengths, value types, separators). File: `Validation/DataValidation/DataValidator.cs` +When invoked at the start of a stream, it uses its supplied `PxFileConfiguration` to locate the configured DATA keyword and syntax separators. + ### DatabaseValidator Validates entire PX database directory (all `.px` files, alias files, directory structure). From 78c9a4580531f199ee116cfd09cad1598fe6bdc1 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 6 Aug 2026 10:47:13 +0300 Subject: [PATCH 05/12] Documentation fixes --- .../ModelBuilders/MatrixMetadataBuilder.cs | 2 +- .../DivisionMatrixFunctionExtensions.cs | 14 +++---- .../MultiplicationMatrixFunction.cs | 7 +++- .../Operations/SumMatrixFunctionExtensions.cs | 7 +++- .../PxFile/Data/PxFileStreamDataReader.cs | 5 ++- .../PxFile/Metadata/IPxFileMetadataReader.cs | 7 +++- .../ContentValidator.UtilityMethods.cs | 3 +- ...ntentValidator.ValidationEntryFunctions.cs | 2 +- .../DataValidation/DataValidator.cs | 1 + .../DataValidation/DataValidatorFunctions.cs | 7 +++- .../DatabaseValidation/IFileSystem.cs | 39 ++++++++++++++++++- .../DatabaseValidation/LocalFileSystem.cs | 13 +++++-- .../SyntaxValidationUtilityMethods.cs | 1 + .../SyntaxValidation/SyntaxValidator.cs | 3 +- .../ValidationStructuredEntry.cs | 3 +- 15 files changed, 88 insertions(+), 26 deletions(-) diff --git a/Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs b/Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs index a41ba4ae..52249d76 100644 --- a/Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs +++ b/Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs @@ -51,7 +51,7 @@ public MatrixMetadata Build(IEnumerable> metadataIn /// /// Builds a object from a given set of metadata entries. /// - /// A of key-value pairs representing the metadata entries in the Px-File format. + /// A of key-value pairs representing the metadata entries in the Px-File format. /// A object constructed from the input metadata entries. public MatrixMetadata Build(IReadOnlyDictionary metadataInput) { diff --git a/Px.Utils/Operations/DivisionMatrixFunctionExtensions.cs b/Px.Utils/Operations/DivisionMatrixFunctionExtensions.cs index db377e2b..1875fb6f 100644 --- a/Px.Utils/Operations/DivisionMatrixFunctionExtensions.cs +++ b/Px.Utils/Operations/DivisionMatrixFunctionExtensions.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models; +using Px.Utils.Models; using Px.Utils.Models.Metadata; using System.Numerics; @@ -13,7 +13,7 @@ public static class DivisionMatrixFunctionExtensions /// Divides all datapoints defined by the with a given constant. /// /// Type of the data values in the matrix, must implement - /// and + /// and /// The source matrix for the operation. /// Defines the datapoints to be divided. /// The constant used to divide the datapoints. @@ -28,7 +28,7 @@ public static Matrix DivideSubsetByConstant(this Matrix inp /// Asyncronously divides all datapoints defined by the with a given constant. /// /// Type of the data values in the matrix, must implement - /// and + /// and /// The source matrix for the operation. /// Defines the datapoints to be divided. /// The constant used to divide the datapoints. @@ -43,7 +43,7 @@ public static async Task> DivideSubsetByConstantAsync(this /// Asyncronously divides all datapoints defined by the with a given constant. /// /// Type of the data values in the matrix, must implement - /// and + /// and /// A tasks that produces the source matrix for the operation. /// Defines the datapoints to be divided. /// The constant used to divide the datapoints. @@ -58,7 +58,7 @@ public static async Task> DivideSubsetByConstantAsync(this /// Divides datapoints defined by by the values of datapoints defined by the . /// /// Type of the data values in the matrix, must implement - /// and + /// and /// The source matrix for the operation. /// The datapoints defined by these values will be divided. /// The set of datapoints defined by this dimension value @@ -74,7 +74,7 @@ public static Matrix DivideSubsetBySelectedValue(this Matrix by the values of datapoints defined by the . /// /// Type of the data values in the matrix, must implement - /// and + /// and /// The source matrix for the operation. /// The datapoints defined by these values will be divided. /// The set of datapoints defined by this dimension value @@ -90,7 +90,7 @@ public static async Task> DivideSubsetBySelectedValueAsync( /// Asyncronously divides datapoints defined by by the values of datapoints defined by the . /// /// Type of the data values in the matrix, must implement - /// and + /// and /// A tasks that produces the source matrix for the operation. /// The datapoints defined by these values will be divided. /// The set of datapoints defined by this dimension value diff --git a/Px.Utils/Operations/MultiplicationMatrixFunction.cs b/Px.Utils/Operations/MultiplicationMatrixFunction.cs index ce2685a5..1fb9e1fc 100644 --- a/Px.Utils/Operations/MultiplicationMatrixFunction.cs +++ b/Px.Utils/Operations/MultiplicationMatrixFunction.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models; +using Px.Utils.Models; using Px.Utils.Models.Metadata; using Px.Utils.Models.Metadata.Dimensions; using System.Numerics; @@ -19,6 +19,7 @@ public static class MultiplicationMatrixFunctionExtensions /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. /// Also defines which s are included in the sum. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the operation. public static Matrix MultiplyToNewValue(this Matrix input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) where TData : IMultiplyOperators, IMultiplicativeIdentity @@ -26,7 +27,7 @@ public static Matrix MultiplyToNewValue(this Matrix input, return input.ApplyOverDimension(newValue, multiplicationMap, Multiply, TData.MultiplicativeIdentity, insertIndex); } - + /// /// Asynchronously multiplies the values defined in the together and places the product in the new value which will be added to the dimension. /// @@ -36,6 +37,7 @@ public static Matrix MultiplyToNewValue(this Matrix input, /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. /// Also defines which s are included in the sum. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the operation. public async static Task> MultiplyToNewValueAsync(this Matrix input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) where TData : IMultiplyOperators, IMultiplicativeIdentity @@ -51,6 +53,7 @@ public async static Task> MultiplyToNewValueAsync(this Matr /// The source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// Also defines which s are included in the sum. /// A new object that contais the results of the operation. public async static Task> MultiplyToNewValueAsync(this Task> input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) diff --git a/Px.Utils/Operations/SumMatrixFunctionExtensions.cs b/Px.Utils/Operations/SumMatrixFunctionExtensions.cs index eaaa35ca..a42c1927 100644 --- a/Px.Utils/Operations/SumMatrixFunctionExtensions.cs +++ b/Px.Utils/Operations/SumMatrixFunctionExtensions.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models; +using Px.Utils.Models; using Px.Utils.Models.Metadata; using Px.Utils.Models.Metadata.Dimensions; using System.Numerics; @@ -19,6 +19,7 @@ public static class SumMatrixFunctionExtensions /// The source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the sums are calculated. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// Also defines which s are included in the sum. /// A new object that contais the results of the additions. public static Matrix SumToNewValue(this Matrix input, DimensionValue newValue, IDimensionMap sumMap, int insertIndex = -1) @@ -35,7 +36,8 @@ public static Matrix SumToNewValue(this Matrix input, Dimen /// and /// The source matrix for the operation /// This value will be added to the dimension defined by the - /// Defines the relative to which the sums are calculated. + /// Defines the relative to which the sums are calculated. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// Also defines which s are included in the sum. /// A new object that contais the results of the additions. public async static Task> SumToNewValueAsync(this Matrix input, DimensionValue newValue, IDimensionMap sumMap, int insetIndex = -1) @@ -52,6 +54,7 @@ public async static Task> SumToNewValueAsync(this MatrixThe source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the sums are calculated. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// Also defines which s are included in the sum. /// A new object that contais the results of the additions. public async static Task> SumToNewValueAsync(this Task> input, DimensionValue newValue, IDimensionMap sumMap, int insetIndex = -1) diff --git a/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs b/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs index 08ce2d41..0567a99b 100644 --- a/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs +++ b/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs @@ -25,6 +25,7 @@ public sealed class PxFileStreamDataReader : IPxFileStreamDataReader, IDisposabl /// /// Px file stream /// Px file syntax configuration + /// The size of the buffer used for reading from the stream. public PxFileStreamDataReader(Stream stream, PxFileConfiguration? conf = null, int readBufferSize = 4096) { _stream = stream; @@ -38,6 +39,7 @@ public PxFileStreamDataReader(Stream stream, PxFileConfiguration? conf = null, i /// Px file stream /// Absolute raw byte offset of the first non-whitespace data value after DATA=. /// Px file syntax configuration + /// The size of the buffer used for reading from the stream. public PxFileStreamDataReader(Stream stream, long dataStart, PxFileConfiguration? conf = null, int readBufferSize = 4096) { _stream = stream; @@ -244,7 +246,8 @@ await Task.Factory.StartNew(() => /// /// The buffer to store the read values. /// The starting index in the buffer to begin storing the read values. - /// Provides the indexes where the data will be read. + /// Map defining the data to be read. Must be a submap of the map. + /// Map defining the complete data set. /// A to observe while waiting for the task to complete. public async Task ReadDecimalDataValuesAsync(DecimalDataValue[] buffer, int offset, IMatrixMap target, IMatrixMap complete, CancellationToken cancellationToken) { diff --git a/Px.Utils/PxFile/Metadata/IPxFileMetadataReader.cs b/Px.Utils/PxFile/Metadata/IPxFileMetadataReader.cs index f31cca63..3bdc9e67 100644 --- a/Px.Utils/PxFile/Metadata/IPxFileMetadataReader.cs +++ b/Px.Utils/PxFile/Metadata/IPxFileMetadataReader.cs @@ -1,4 +1,4 @@ -using System.Text; +using System.Text; namespace Px.Utils.PxFile.Metadata { @@ -13,11 +13,12 @@ public interface IPxFileMetadataReader /// The stream from which to determine the encoding. /// The determined encoding of the stream. Encoding GetEncoding(Stream stream); - + /// /// Asynchronously determines the encoding of the provided stream based on the Byte Order Mark (BOM) or the CODEPAGE keyword in the metadata. /// /// The stream from which to determine the encoding. + /// A token that can be used to cancel the operation. /// The determined encoding of the stream. Task GetEncodingAsync(Stream stream, CancellationToken cancellationToken = default); @@ -38,6 +39,7 @@ public interface IPxFileMetadataReader /// The stream from which to read the metadata. /// The encoding to use when reading the stream. /// The size of the buffer to use when reading the stream. If not specified, the default buffer size is used. + /// A token that can be used to cancel the operation. /// An of key-value pairs representing the metadata entries in the file. IAsyncEnumerable> ReadMetadataAsync( Stream stream, @@ -63,6 +65,7 @@ IAsyncEnumerable> ReadMetadataAsync( /// The stream from which to read the metadata. /// The encoding to use when reading the stream. /// The size of the buffer to use when reading the stream. If not specified, the default buffer size is used. + /// A token that can be used to cancel the operation. /// A dictionary containing the metadata entries in the file. Task> ReadMetadataToDictionaryAsync( Stream stream, diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.UtilityMethods.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.UtilityMethods.cs index 6a2aaaab..abd916c2 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.UtilityMethods.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.UtilityMethods.cs @@ -1,4 +1,4 @@ -using Px.Utils.PxFile; +using Px.Utils.PxFile; using Px.Utils.Validation.SyntaxValidation; namespace Px.Utils.Validation.ContentValidation @@ -154,6 +154,7 @@ [.. values.Select(v => SyntaxValidationUtilityMethods.CleanString(v, conf))] /// /// Keywords to search for that are language specific /// Language agnostic keywords + /// Recommended keywords that should be present in the Px file /// Structured entries of Px file metadata to be searched from /// Object that provides information of the ongoing content validation process /// KeyValuePair that contains the processed language as key and the dimension name as value diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index 5f0ab0ad..5ca89057 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -25,7 +25,7 @@ public sealed partial class ContentValidator /// /// Validates that given entry does not contain specifiers if it is not allowed to have them based on keyword. /// - /// Px file metadata entries in an array of objects + /// Px file metadata entry represented by a object /// object that stores information that is gathered during the validation process /// Key value pair containing information about the rule violation is returned if an unexpected specifier is detected. public static ValidationFeedback? ValidateUnexpectedSpecifiers(ValidationStructuredEntry entry, ContentValidator validator) diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index 688998b8..d8c4cf97 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -179,6 +179,7 @@ internal async Task ValidateAsync( /// /// Validates the data in the specified stream asynchronously. /// Assumes that the stream is at the start of the data section (after 'DATA='-keyword) at the first data item. + /// /// Px file stream to be validated /// Name of the file being validated. If not provided, validator tries to find the encoding. /// Encoding of the stream. diff --git a/Px.Utils/Validation/DataValidation/DataValidatorFunctions.cs b/Px.Utils/Validation/DataValidation/DataValidatorFunctions.cs index 727b5a77..42c9eafb 100644 --- a/Px.Utils/Validation/DataValidation/DataValidatorFunctions.cs +++ b/Px.Utils/Validation/DataValidation/DataValidatorFunctions.cs @@ -1,4 +1,4 @@ -using Px.Utils.PxFile; +using Px.Utils.PxFile; using System.Globalization; using System.Text; @@ -32,6 +32,7 @@ public class DataStringValidator : IDataValidator /// Encoding format of the Px file. /// Line number for the validation item. /// Represents the position relative to the line for the validation item. + /// The name of the file being validated. /// Key value pair containing information about the rule violation if the entry is not a missing value string sequence, otherwise null. public KeyValuePair? Validate(List entry, EntryType entryType, Encoding encoding, int lineNumber, int charPos, string filename) { @@ -63,6 +64,7 @@ public class DataNumberValidator : IDataValidator /// Encoding format of the Px file. /// Line number for the validation item. /// Represents the position relative to the line for the validation item. + /// The name of the file being validated. /// Key value pair containing information about the rule violation if the entry is not a valid number, otherwise null. public KeyValuePair? Validate(List entry, EntryType entryType, Encoding encoding, int lineNumber, int charPos, string filename) { @@ -145,6 +147,7 @@ public class DataSeparatorValidator : IDataValidator /// Encoding format of the Px file. /// Line number for the validation item. /// Represents the position relative to the line for the validation item. + /// The name of the file being validated. /// Key value pair containing information about the rule violation if the entry is not a valid item separator, otherwise null. public KeyValuePair? Validate(List entry, EntryType entryType, Encoding encoding, int lineNumber, int charPos, string filename) { @@ -188,7 +191,7 @@ public class DataStructureValidator : IDataValidator /// Encoding format of the Px file. /// Line number for the validation item. /// Represents the position relative to the line for the validation item. - /// Reference to a list of feedback items to which any validation feedback is added to. + /// The name of the file being validated. /// Key value pair containing information about the rule violation if the entry sequence is invalid. Otherwise null. public KeyValuePair? Validate(List entry, EntryType entryType, Encoding encoding, int lineNumber, int charPos, string filename) { diff --git a/Px.Utils/Validation/DatabaseValidation/IFileSystem.cs b/Px.Utils/Validation/DatabaseValidation/IFileSystem.cs index 64394039..31fad9ae 100644 --- a/Px.Utils/Validation/DatabaseValidation/IFileSystem.cs +++ b/Px.Utils/Validation/DatabaseValidation/IFileSystem.cs @@ -1,4 +1,4 @@ -using Px.Utils.PxFile.Metadata; +using Px.Utils.PxFile.Metadata; using System.Diagnostics.CodeAnalysis; using System.Text; @@ -9,12 +9,49 @@ namespace Px.Utils.Validation.DatabaseValidation /// public interface IFileSystem { + /// + /// Enumerates all files in the specified directory and its subdirectories that match the given search pattern. + /// + /// Path to the directory to search. + /// Pattern to match against the names of files in the directory. This parameter can contain a combination of literal and wildcard characters. + /// Enumerable collection of file paths that match the search pattern. public IEnumerable EnumerateFiles(string path, string searchPattern); + /// + /// Gets a read-only stream for the specified file path. + /// + /// Path to the file to open. + /// Stream for reading the specified file. public Stream GetFileStream(string path); + /// + /// Enumerates all directories in the specified path and its subdirectories. + /// + /// Path to the directory to search. + /// Enumerable collection of directory paths. public IEnumerable EnumerateDirectories(string path); + /// + /// Gets the file name and extension of the specified path string. + /// + /// Path string from which to get the file name and extension. + /// Name and extension of the specified path string. public string GetFileName(string path); + /// + /// Gets the directory name of the specified path string. + /// + /// Path string from which to get the directory name. + /// Name of the directory from the specified path string. public string GetDirectoryName(string path); + /// + /// Gets the encoding of the specified stream. This method may read from the stream to determine its encoding, so the stream position may be changed after calling this method. + /// + /// Stream from which to determine the encoding. + /// Encoding of the specified stream. public Encoding GetEncoding(Stream stream); + /// + /// Asynchronously gets the encoding of the specified stream. This method may read from the stream to determine its encoding, so the stream position may be changed after calling this method. + /// + /// Stream from which to determine the encoding. + /// Token to monitor for cancellation requests. + /// Task representing the asynchronous operation, with the encoding of the specified stream as the result. public Task GetEncodingAsync(Stream stream, CancellationToken cancellationToken); } } diff --git a/Px.Utils/Validation/DatabaseValidation/LocalFileSystem.cs b/Px.Utils/Validation/DatabaseValidation/LocalFileSystem.cs index 0e08c638..9c8914ad 100644 --- a/Px.Utils/Validation/DatabaseValidation/LocalFileSystem.cs +++ b/Px.Utils/Validation/DatabaseValidation/LocalFileSystem.cs @@ -1,49 +1,56 @@ -using Px.Utils.PxFile.Metadata; +using Px.Utils.PxFile.Metadata; using System.Diagnostics.CodeAnalysis; using System.Text; namespace Px.Utils.Validation.DatabaseValidation { - // Excluded from code coverage because it is a wrapper around the file system and testing IO operations is not feasible. - [ExcludeFromCodeCoverage] /// /// Default file system used for database validation process. Contains default implementations of numerous IO operations /// and a function for determining a file's encoding format. /// + // Excluded from code coverage because it is a wrapper around the file system and testing IO operations is not feasible. + [ExcludeFromCodeCoverage] public class LocalFileSystem : IFileSystem { + /// public IEnumerable EnumerateFiles(string path, string searchPattern) { return Directory.EnumerateFiles(path, searchPattern, SearchOption.AllDirectories); } + /// public Stream GetFileStream(string path) { return new FileStream(path, FileMode.Open, FileAccess.Read); } + /// public IEnumerable EnumerateDirectories(string path) { return Directory.EnumerateDirectories(path, "*", SearchOption.AllDirectories); } + /// public string GetFileName(string path) { return Path.GetFileName(path); } + /// public string GetDirectoryName(string path) { string? directory = Path.GetDirectoryName(path); return directory ?? throw new ArgumentException("Path does not contain a directory."); } + /// public Encoding GetEncoding(Stream stream) { PxFileMetadataReader reader = new(); return reader.GetEncoding(stream); } + /// public async Task GetEncodingAsync(Stream stream, CancellationToken cancellationToken) { PxFileMetadataReader reader = new(); diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs index 035b050e..fb4fe058 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs @@ -44,6 +44,7 @@ public static class SyntaxValidationUtilityMethods /// /// The input string to extract from /// Symbol that starts enclosement + /// Symbol that encloses a string /// Optional symbol that closes the enclosement. If none given, startSymbol is used for both starting and ending the enclosement /// Returns an object that contains the extracted sections, /// the string that remains after the operation and starting indexes of extracted sections diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs index dd0b2af0..96bd87a9 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs @@ -9,10 +9,9 @@ namespace Px.Utils.Validation.SyntaxValidation /// /// Provides methods for validating the syntax of a PX file. Validation can be done using both synchronous and asynchronous methods. /// Additionally custom validation functions can be provided to be used during validation. + /// /// Object that stores syntax specific symbols and tokens for the PX file /// Object that contains any optional additional validation functions - /// his is required if multiple validations are executed for the same stream. - /// public class SyntaxValidator( PxFileConfiguration? conf = null, CustomSyntaxValidationFunctions? customValidationFunctions = null) diff --git a/Px.Utils/Validation/SyntaxValidation/ValidationStructuredEntry.cs b/Px.Utils/Validation/SyntaxValidation/ValidationStructuredEntry.cs index a6e4878c..32f3d9bd 100644 --- a/Px.Utils/Validation/SyntaxValidation/ValidationStructuredEntry.cs +++ b/Px.Utils/Validation/SyntaxValidation/ValidationStructuredEntry.cs @@ -1,4 +1,4 @@ -namespace Px.Utils.Validation.SyntaxValidation +namespace Px.Utils.Validation.SyntaxValidation { /// /// Represents a key for a . A key consists of a keyword and two optional language and specifier strings. @@ -50,6 +50,7 @@ public ValidationStructuredEntryKey(string keyword, string? language = null, str /// Index of the line where the entry starts. /// Character indexes of the line changes in the entry starting from the entry start. /// Index of the first character of the value in the entry. + /// Value type of the value part of the entry, if found. public class ValidationStructuredEntry( string file, ValidationStructuredEntryKey key, From 1b67617a12dd51bf0a5a95908617666c909e8c18 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 6 Aug 2026 12:22:07 +0300 Subject: [PATCH 06/12] Copilot suggested fixes --- Px.Utils.TestingApp/Commands/Benchmark.cs | 6 +- .../DataTests/StreamUtilitiesTests.cs | 34 +++++ .../MultiplicationMatrixFunction.cs | 6 +- .../Operations/SumMatrixFunctionExtensions.cs | 4 +- Px.Utils/PxFile/Data/StreamUtilities.cs | 122 ++++++++++++++++-- docs/README.md | 2 + docs/architecture.readers.md | 2 +- 7 files changed, 156 insertions(+), 20 deletions(-) diff --git a/Px.Utils.TestingApp/Commands/Benchmark.cs b/Px.Utils.TestingApp/Commands/Benchmark.cs index 86bf5660..5c2c14a1 100644 --- a/Px.Utils.TestingApp/Commands/Benchmark.cs +++ b/Px.Utils.TestingApp/Commands/Benchmark.cs @@ -186,12 +186,12 @@ protected virtual void StartInteractiveMode() private static ValidationOptions ParseValidationOptions(string value) { - if (int.TryParse(value, out int limit) && limit > 0) + if (!int.TryParse(value, out int limit) || limit <= 0) { - return new ValidationOptions { MaxFeedbackItemsPerSignature = limit }; + throw new ArgumentOutOfRangeException(nameof(value), "Feedback limit must be a positive integer."); } - return ValidationOptions.Unlimited; + return new ValidationOptions { MaxFeedbackItemsPerSignature = limit }; } /// diff --git a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs index 4bf90aa5..44286bf1 100644 --- a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs +++ b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs @@ -12,6 +12,40 @@ public class StreamUtilitiesTests * THIS TEST SET ASSUMES THAT THE INPUT IS VALIDATED AND DOES NOT CONTAIN ANY ERRORS */ + [TestMethod] + public void FindKeywordPositionKeywordSplitAcrossBuffersReturnsKeywordOffset() + { + // Arrange + string content = "TITLE=\"DATA=79\";\r\n VALUES=\"x\";\r\nDATA=1;"; + byte[] data = Encoding.UTF8.GetBytes(content); + using Stream stream = new MemoryStream(data); + long expectedPosition = Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("DATA=", StringComparison.Ordinal)]); + + // Act + long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default, 3); + + // Assert + Assert.AreEqual(expectedPosition, position); + Assert.IsTrue(stream.Position > position); + } + + [TestMethod] + public async Task FindKeywordPositionAsyncKeywordSplitAcrossBuffersReturnsKeywordOffset() + { + // Arrange + string content = "TITLE=\"DATA=79\";\r\n VALUES=\"x\";\r\nDATA=1;"; + byte[] data = Encoding.UTF8.GetBytes(content); + using Stream stream = new MemoryStream(data); + long expectedPosition = Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("DATA=", StringComparison.Ordinal)]); + + // Act + long position = await StreamUtilities.FindKeywordPositionAsync(stream, "DATA", PxFileConfiguration.Default, CancellationToken.None, 3); + + // Assert + Assert.AreEqual(expectedPosition, position); + Assert.IsTrue(stream.Position > position); + } + [TestMethod] [DataRow("1")] [DataRow("-1")] diff --git a/Px.Utils/Operations/MultiplicationMatrixFunction.cs b/Px.Utils/Operations/MultiplicationMatrixFunction.cs index 1fb9e1fc..97bd1693 100644 --- a/Px.Utils/Operations/MultiplicationMatrixFunction.cs +++ b/Px.Utils/Operations/MultiplicationMatrixFunction.cs @@ -18,7 +18,7 @@ public static class MultiplicationMatrixFunctionExtensions /// The source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. - /// Also defines which s are included in the sum. + /// Also defines which s are included in the product. /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the operation. public static Matrix MultiplyToNewValue(this Matrix input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) @@ -36,7 +36,7 @@ public static Matrix MultiplyToNewValue(this Matrix input, /// The source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. - /// Also defines which s are included in the sum. + /// Also defines which s are included in the product. /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the operation. public async static Task> MultiplyToNewValueAsync(this Matrix input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) @@ -53,8 +53,8 @@ public async static Task> MultiplyToNewValueAsync(this Matr /// The source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. + /// Also defines which s are included in the product. /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. - /// Also defines which s are included in the sum. /// A new object that contais the results of the operation. public async static Task> MultiplyToNewValueAsync(this Task> input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) where TData : IMultiplyOperators, IMultiplicativeIdentity diff --git a/Px.Utils/Operations/SumMatrixFunctionExtensions.cs b/Px.Utils/Operations/SumMatrixFunctionExtensions.cs index a42c1927..0cd46226 100644 --- a/Px.Utils/Operations/SumMatrixFunctionExtensions.cs +++ b/Px.Utils/Operations/SumMatrixFunctionExtensions.cs @@ -18,9 +18,9 @@ public static class SumMatrixFunctionExtensions /// and /// The source matrix for the operation /// This value will be added to the dimension defined by the - /// Defines the relative to which the sums are calculated. - /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. + /// Defines the relative to which the sums are calculated. /// Also defines which s are included in the sum. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the additions. public static Matrix SumToNewValue(this Matrix input, DimensionValue newValue, IDimensionMap sumMap, int insertIndex = -1) where TData : IAdditionOperators, IAdditiveIdentity diff --git a/Px.Utils/PxFile/Data/StreamUtilities.cs b/Px.Utils/PxFile/Data/StreamUtilities.cs index e0e4f456..5b939d48 100644 --- a/Px.Utils/PxFile/Data/StreamUtilities.cs +++ b/Px.Utils/PxFile/Data/StreamUtilities.cs @@ -8,6 +8,33 @@ namespace Px.Utils.PxFile.Data /// public static class StreamUtilities { + /// + /// Finds the absolute raw byte offset of the first occurrence of a keyword at the start of a top-level PX entry. + /// + /// The PX file stream to search from its current position. + /// The keyword to search for. + /// A configuration object that contains PX syntax symbols. + /// The size of the buffer to use when reading from the stream. Defaults to 4096. + /// The absolute raw byte offset of the keyword, or -1 when the keyword cannot be found. + public static long FindKeywordPosition(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize = 4096) + { + return FindKeywordPositionImpl(stream, keyword, conf, bufferSize); + } + + /// + /// Asynchronously finds the absolute raw byte offset of the first occurrence of a keyword at the start of a top-level PX entry. + /// + /// The PX file stream to search from its current position. + /// The keyword to search for. + /// A configuration object that contains PX syntax symbols. + /// A token that can be used to cancel the operation. + /// The size of the buffer to use when reading from the stream. Defaults to 4096. + /// The absolute raw byte offset of the keyword, or -1 when the keyword cannot be found. + public static Task FindKeywordPositionAsync(Stream stream, string keyword, PxFileConfiguration conf, CancellationToken? cancellationToken = null, int bufferSize = 4096) + { + return FindKeywordPositionImplAsync(stream, keyword, conf, bufferSize, cancellationToken ?? CancellationToken.None); + } + /// /// Finds the absolute raw byte offset of the first non-whitespace data value after a top-level DATA entry. /// The stream position is restored before this method returns. Returns -1 when the DATA entry or its first value cannot be found. @@ -53,6 +80,60 @@ public static async Task FindDataStartPositionAsync(Stream stream, PxFileC } } + private static long FindKeywordPositionImpl(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize) + { + byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); + byte[] buffer = new byte[bufferSize]; + KeywordSearchState state = new(keywordBytes, (byte)conf.Symbols.EntrySeparator); + + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + { + long bufferStart = stream.Position - bytesRead; + long keywordPosition = FindKeywordPosition(buffer.AsSpan(0, bytesRead), bufferStart, ref state); + if (keywordPosition >= 0) + { + return keywordPosition; + } + } + + return -1; + } + + private static async Task FindKeywordPositionImplAsync(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) + { + byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); + byte[] buffer = new byte[bufferSize]; + KeywordSearchState state = new(keywordBytes, (byte)conf.Symbols.EntrySeparator); + + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) + { + long bufferStart = stream.Position - bytesRead; + long keywordPosition = FindKeywordPosition(buffer.AsSpan(0, bytesRead), bufferStart, ref state); + if (keywordPosition >= 0) + { + return keywordPosition; + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static long FindKeywordPosition(ReadOnlySpan buffer, long bufferStart, ref KeywordSearchState state) + { + for (int i = 0; i < buffer.Length; i++) + { + if (TryProcessKeywordByte(buffer[i], ref state)) + { + return bufferStart + i - state.KeywordBytes.Length + 1; + } + } + + return -1; + } + private static long FindDataStartPositionImpl(Stream stream, PxFileConfiguration conf, int bufferSize) { byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); @@ -156,20 +237,34 @@ private static bool TryProcessDataStartByte(byte currentByte, ref DataStartSearc } if (currentByte == state.EntrySeparator) { - state.ResetEntry(); + state.KeywordSearchState.Reset(); return false; } - if (state.IsAtEntryStart && IsWhitespace(currentByte)) return false; - if (state.IsAtEntryStart && state.MatchedKeywordBytes < state.DataKeywordBytes.Length && currentByte == state.DataKeywordBytes[state.MatchedKeywordBytes]) + if (TryProcessKeywordByte(currentByte, ref state.KeywordSearchState)) { - state.MatchedKeywordBytes++; + state.IsAfterDataKeyword = true; + } + + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryProcessKeywordByte(byte currentByte, ref KeywordSearchState state) + { + if (currentByte == state.EntrySeparator) + { + state.Reset(); return false; } - if (state.MatchedKeywordBytes == state.DataKeywordBytes.Length && currentByte == state.KeywordSeparator) + if (!state.IsAtEntryStart || IsWhitespace(currentByte)) { - state.IsAfterDataKeyword = true; return false; } + if (state.MatchedKeywordBytes < state.KeywordBytes.Length && currentByte == state.KeywordBytes[state.MatchedKeywordBytes]) + { + state.MatchedKeywordBytes++; + return state.MatchedKeywordBytes == state.KeywordBytes.Length; + } state.IsAtEntryStart = false; state.MatchedKeywordBytes = 0; @@ -184,18 +279,23 @@ private static bool IsWhitespace(byte value) private struct DataStartSearchState(byte[] dataKeywordBytes, byte entrySeparator, byte keywordSeparator, byte stringDelimiter) { - public readonly byte[] DataKeywordBytes = dataKeywordBytes; public readonly byte EntrySeparator = entrySeparator; - public readonly byte KeywordSeparator = keywordSeparator; public readonly byte StringDelimiter = stringDelimiter; - public int MatchedKeywordBytes; - public bool IsAtEntryStart = true; + public KeywordSearchState KeywordSearchState = new([.. dataKeywordBytes, keywordSeparator], entrySeparator); public bool IsInString; public bool IsAfterDataKeyword; public bool IsDataEntryEmpty; + } + + private struct KeywordSearchState(byte[] keywordBytes, byte entrySeparator) + { + public readonly byte[] KeywordBytes = keywordBytes; + public readonly byte EntrySeparator = entrySeparator; + public int MatchedKeywordBytes; + public bool IsAtEntryStart = true; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ResetEntry() + public void Reset() { MatchedKeywordBytes = 0; IsAtEntryStart = true; diff --git a/docs/README.md b/docs/README.md index 9059f2ae..7f0a4fbf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -59,6 +59,8 @@ The entries need to be in the same key-value format as the output of the ```PxFi When the reader is created at stream position `0`, it finds the first non-whitespace data value after the top-level `DATA=` entry automatically. This also works with a UTF-8 BOM and multibyte metadata. The overload that accepts `dataStart` expects the absolute raw byte offset of that first value; use `StreamUtilities.FindDataStartPosition()` or `FindDataStartPositionAsync()` to obtain the offset from a seekable stream. Both helpers restore the original stream position and return `-1` when no data value is found, including an explicitly empty entry such as `DATA=;`. +`StreamUtilities.FindKeywordPosition()` and `FindKeywordPositionAsync()` locate a specified keyword at the start of a top-level PX entry and return that keyword's raw byte offset. Use `FindDataStartPosition` or `FindDataStartPositionAsync` when locating the first data value after `DATA=`. Unlike the DATA-specific helpers, the generic keyword methods search from the stream's current position and leave it advanced after reading. + ### Metadata example ```csharp // Read meta diff --git a/docs/architecture.readers.md b/docs/architecture.readers.md index 62a77b7f..353de76c 100644 --- a/docs/architecture.readers.md +++ b/docs/architecture.readers.md @@ -40,7 +40,7 @@ When constructed at stream position `0`, `PxFileStreamDataReader` locates the fi |---|---| | `DataIndexer.cs` | Index mapping between source and target `IMatrixMap` | | `DataValueParsers.cs` | Parse raw data values from byte spans | -| `StreamUtilities.cs` | `FindDataStartPosition` and async equivalent locate the first value after top-level `DATA=` as an absolute raw byte offset, preserving the original position of a seekable stream | +| `StreamUtilities.cs` | `FindDataStartPosition` and async equivalent locate the first value after top-level `DATA=` as an absolute raw byte offset, preserving the original position of a seekable stream. `FindKeywordPosition` and async equivalent locate any top-level entry keyword from the current position and leave the stream advanced. | ## Binary Data From ddd8c8d3c7a13479cbf9c1b07f3a98430da43dbf Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 6 Aug 2026 12:37:55 +0300 Subject: [PATCH 07/12] Fix stream position after encoding detection --- .../DataValidation/DataValidator.cs | 31 ++++++++++++++++--- Px.Utils/Validation/PxFileValidator.cs | 16 ++++++++-- .../SyntaxValidation/SyntaxValidator.cs | 28 ++++++++++++++--- 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index d8c4cf97..59593d8e 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -52,7 +52,13 @@ public ValidationResult Validate( IFileSystem? fileSystem = null) { fileSystem ??= new LocalFileSystem(); - encoding ??= fileSystem.GetEncoding(stream); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = fileSystem.GetEncoding(stream); + stream.Position = originalPosition; + } + SetValidationParameters(encoding, filename); ValidationFeedback validationFeedbacks = []; @@ -105,7 +111,12 @@ internal ValidationResult Validate( ValidationFeedbackSink sink) { fileSystem ??= new LocalFileSystem(); - encoding ??= fileSystem.GetEncoding(stream); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = fileSystem.GetEncoding(stream); + stream.Position = originalPosition; + } SetValidationParameters(encoding, filename); long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); @@ -156,9 +167,14 @@ internal async Task ValidateAsync( CancellationToken cancellationToken = default) { fileSystem ??= new LocalFileSystem(); - encoding ??= await fileSystem.GetEncodingAsync(stream, cancellationToken); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } SetValidationParameters(encoding, filename); - + long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); if (dataStartIndex == -1) { @@ -194,7 +210,12 @@ public async Task ValidateAsync( CancellationToken cancellationToken = default) { fileSystem ??= new LocalFileSystem(); - encoding ??= await fileSystem.GetEncodingAsync(stream, cancellationToken); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } SetValidationParameters(encoding, filename); ValidationFeedback validationFeedbacks = []; diff --git a/Px.Utils/Validation/PxFileValidator.cs b/Px.Utils/Validation/PxFileValidator.cs index 85062ee5..2bf6f795 100644 --- a/Px.Utils/Validation/PxFileValidator.cs +++ b/Px.Utils/Validation/PxFileValidator.cs @@ -92,7 +92,13 @@ internal ValidationResult Validate( IFileSystem? fileSystem, ValidationFeedbackSink sink) { - encoding ??= new LocalFileSystem().GetEncoding(stream); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = new LocalFileSystem().GetEncoding(stream); + stream.Position = originalPosition; + } + conf ??= PxFileConfiguration.Default; SyntaxValidator syntaxValidator = new(conf, _customSyntaxValidationFunctions); @@ -185,7 +191,13 @@ internal async Task ValidateAsync( ValidationFeedbackSink sink, CancellationToken cancellationToken = default) { - encoding ??= await new LocalFileSystem().GetEncodingAsync(stream, cancellationToken); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await new LocalFileSystem().GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } + conf ??= PxFileConfiguration.Default; SyntaxValidator syntaxValidator = new(conf, _customSyntaxValidationFunctions); diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs index 96bd87a9..88251adf 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs @@ -37,7 +37,12 @@ public SyntaxValidationResult Validate( IFileSystem? fileSystem = null) { fileSystem ??= new LocalFileSystem(); - encoding ??= fileSystem.GetEncoding(stream); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = fileSystem.GetEncoding(stream); + stream.Position = originalPosition; + } SyntaxValidationFunctions validationFunctions = new(); IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; @@ -94,7 +99,12 @@ internal SyntaxValidationResult Validate( ValidationFeedbackSink sink) { fileSystem ??= new LocalFileSystem(); - encoding ??= fileSystem.GetEncoding(stream); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = fileSystem.GetEncoding(stream); + stream.Position = originalPosition; + } SyntaxValidationFunctions validationFunctions = new(); IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; @@ -140,7 +150,12 @@ public async Task ValidateAsync( CancellationToken cancellationToken = default) { fileSystem ??= new LocalFileSystem(); - encoding ??= await fileSystem.GetEncodingAsync(stream, cancellationToken); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } SyntaxValidationFunctions validationFunctions = new(); IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; @@ -199,7 +214,12 @@ internal async Task ValidateAsync( CancellationToken cancellationToken = default) { fileSystem ??= new LocalFileSystem(); - encoding ??= await fileSystem.GetEncodingAsync(stream, cancellationToken); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } SyntaxValidationFunctions validationFunctions = new(); IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; From 5e79e7862758d857c651d25d919454f7ee31f9ab Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 6 Aug 2026 14:53:04 +0300 Subject: [PATCH 08/12] Separate validation and data reading data start location paths for performance --- .../DataReaderTests.cs | 41 ++++ .../DataTests/StreamUtilitiesTests.cs | 61 ++++++ .../PxFile/Data/PxFileStreamDataReader.cs | 4 +- Px.Utils/PxFile/Data/StreamUtilities.cs | 176 +++++++++++++++++- docs/architecture.readers.md | 4 +- 5 files changed, 274 insertions(+), 12 deletions(-) diff --git a/Px.Utils.UnitTests/PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs b/Px.Utils.UnitTests/PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs index cafee345..9f6d3896 100644 --- a/Px.Utils.UnitTests/PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs +++ b/Px.Utils.UnitTests/PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs @@ -1,4 +1,5 @@ using Px.Utils.Models.Metadata; +using Px.Utils.PxFile; using Px.Utils.PxFile.Data; using PxFileTests.Fixtures; using Px.Utils.Models.Data; @@ -12,6 +13,46 @@ public class DataReaderTests { private readonly double[] missingMarkers = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]; + [TestMethod] + public void ReadDoubleDataValuesQuotedDataKeywordAndWhitespaceWithSmallBufferReturnsCorrectValue() + { + // Arrange + byte[] data = Encoding.UTF8.GetBytes("TITLE=\"DATA=not-an-entry\";\nDATA=\r\n\t1;"); + using Stream stream = new MemoryStream(data); + using PxFileStreamDataReader reader = new(stream, null, 2); + DoubleDataValue[] targetBuffer = new DoubleDataValue[1]; + MatrixMetadata metadata = TestModelBuilder.BuildTestMetadata([1]); + + // Act + reader.ReadDoubleDataValues(targetBuffer, 0, metadata, metadata); + + // Assert + Assert.AreEqual(1.0, targetBuffer[0].UnsafeValue); + } + + [TestMethod] + public void ReadDoubleDataValuesExplicitDataStartMatchesAutomaticPositioning() + { + // Arrange + byte[] data = Encoding.UTF8.GetBytes("TITLE=\"DATA=not-an-entry\";\nDATA= 1;"); + using Stream positionStream = new MemoryStream(data); + long dataStart = StreamUtilities.FindDataStartPosition(positionStream, PxFileConfiguration.Default); + using Stream automaticStream = new MemoryStream(data); + using Stream explicitPositionStream = new MemoryStream(data); + using PxFileStreamDataReader automaticReader = new(automaticStream, null, 2); + using PxFileStreamDataReader explicitPositionReader = new(explicitPositionStream, dataStart, null, 2); + DoubleDataValue[] automaticBuffer = new DoubleDataValue[1]; + DoubleDataValue[] explicitPositionBuffer = new DoubleDataValue[1]; + MatrixMetadata metadata = TestModelBuilder.BuildTestMetadata([1]); + + // Act + automaticReader.ReadDoubleDataValues(automaticBuffer, 0, metadata, metadata); + explicitPositionReader.ReadDoubleDataValues(explicitPositionBuffer, 0, metadata, metadata); + + // Assert + CollectionAssert.AreEqual(automaticBuffer, explicitPositionBuffer); + } + [TestMethod] public void ReadDoubleDataValuesValidIntegersReturnsCorrectDoubleDataValues() { diff --git a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs index 44286bf1..1ff77127 100644 --- a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs +++ b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs @@ -96,6 +96,65 @@ public async Task FindDataStartPositionAsyncDataSplitAcrossBuffersReturnsSameOff Assert.AreEqual(0, stream.Position); } + [TestMethod] + public async Task FindKeywordPositionQuotedKeywordWithBomAndMultibyteMetadataReturnsRawTopLevelOffset() + { + // Arrange + string content = "TITLE=\"DATA=not-an-entry\";\nVALUES=\"Ää\";\nDATA=1;"; + byte[] data = [.. Encoding.UTF8.GetPreamble(), .. Encoding.UTF8.GetBytes(content)]; + long expectedPosition = Encoding.UTF8.GetPreamble().Length + Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("DATA=", StringComparison.Ordinal)]); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + + // Act + long synchronousPosition = StreamUtilities.FindKeywordPosition(synchronousStream, "DATA", PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindKeywordPositionAsync(asynchronousStream, "DATA", PxFileConfiguration.Default, CancellationToken.None, 2); + + // Assert + Assert.AreEqual(expectedPosition, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.IsGreaterThan(synchronousPosition, synchronousStream.Position); + Assert.IsGreaterThan(asynchronousPosition, asynchronousStream.Position); + } + + [TestMethod] + public async Task FindDataStartPositionUncheckedQuotedKeywordAndWhitespaceReturnsOffsetAndAdvancesStream() + { + // Arrange + string content = "TITLE=\"DATA=not-an-entry\";\nVALUES=\"Ää\";\nDATA=\r\n\t1 2;"; + byte[] data = [.. Encoding.UTF8.GetPreamble(), .. Encoding.UTF8.GetBytes(content)]; + long expectedPosition = Encoding.UTF8.GetPreamble().Length + Encoding.UTF8.GetByteCount(content[..content.IndexOf('1')]); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + + // Act + long synchronousPosition = StreamUtilities.FindDataStartPositionUnchecked(synchronousStream, PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionUncheckedAsync(asynchronousStream, PxFileConfiguration.Default, 2, TestContext.CancellationToken); + + // Assert + Assert.AreEqual(expectedPosition, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.IsGreaterThan(synchronousPosition, synchronousStream.Position); + Assert.IsGreaterThan(asynchronousPosition, asynchronousStream.Position); + } + + [TestMethod] + public async Task FindDataStartPositionUncheckedMissingDataReturnsNegative1() + { + // Arrange + byte[] data = Encoding.UTF8.GetBytes("TITLE=\"value\";"); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + + // Act + long synchronousPosition = StreamUtilities.FindDataStartPositionUnchecked(synchronousStream, PxFileConfiguration.Default, 1); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionUncheckedAsync(asynchronousStream, PxFileConfiguration.Default, 1, TestContext.CancellationToken); + + // Assert + Assert.AreEqual(-1, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + } + [TestMethod] public void FindDataStartPositionDataWithoutValueReturnsNegative1() { @@ -124,5 +183,7 @@ public async Task FindDataStartPositionEmptyDataEntryReturnsNegative1(string con Assert.AreEqual(synchronousPosition, asynchronousPosition); Assert.AreEqual(0, stream.Position); } + + public TestContext TestContext { get; set; } } } diff --git a/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs b/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs index 0567a99b..80de91ee 100644 --- a/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs +++ b/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs @@ -275,7 +275,7 @@ public void Dispose() private void SetReaderPositionIfZero() { if (_stream.Position != 0) return; - long start = StreamUtilities.FindDataStartPosition(_stream, _conf, _readBufferSize); + long start = StreamUtilities.FindDataStartPositionUnchecked(_stream, _conf, _readBufferSize); if (start == -1) { throw new ArgumentException($"Could not find the first data value after '{_conf.Tokens.KeyWords.Data}='"); @@ -286,7 +286,7 @@ private void SetReaderPositionIfZero() private async Task SetReaderPositionIfZeroAsync(CancellationToken? cancellationToken = null) { if (_stream.Position != 0) return; - long start = await StreamUtilities.FindDataStartPositionAsync(_stream, _conf, _readBufferSize, cancellationToken ?? CancellationToken.None); + long start = await StreamUtilities.FindDataStartPositionUncheckedAsync(_stream, _conf, _readBufferSize, cancellationToken ?? CancellationToken.None); if (start == -1) { throw new ArgumentException($"Could not find the first data value after '{_conf.Tokens.KeyWords.Data}='"); diff --git a/Px.Utils/PxFile/Data/StreamUtilities.cs b/Px.Utils/PxFile/Data/StreamUtilities.cs index 5b939d48..a5663f64 100644 --- a/Px.Utils/PxFile/Data/StreamUtilities.cs +++ b/Px.Utils/PxFile/Data/StreamUtilities.cs @@ -57,6 +57,33 @@ public static long FindDataStartPosition(Stream stream, PxFileConfiguration conf } } + /// + /// Finds the first non-whitespace data value after a top-level DATA entry in already validated PX input. + /// The stream is read from its current position and remains advanced. + /// + /// The PX file stream to search from its current position. + /// A configuration object that contains the DATA keyword and PX syntax symbols. + /// The size of the buffer to use when reading from the stream. Defaults to 4096. + /// The absolute raw byte offset of the first data value, or -1. + internal static long FindDataStartPositionUnchecked(Stream stream, PxFileConfiguration conf, int bufferSize = 4096) + { + return FindDataStartPositionUncheckedImpl(stream, conf, bufferSize); + } + + /// + /// Asynchronously finds the first non-whitespace data value after a top-level DATA entry in already validated PX input. + /// The stream is read from its current position and remains advanced. + /// + /// The PX file stream to search from its current position. + /// A configuration object that contains the DATA keyword and PX syntax symbols. + /// The size of the buffer to use when reading from the stream. Defaults to 4096. + /// A token that can be used to cancel the operation. + /// The absolute raw byte offset of the first data value, or -1. + internal static Task FindDataStartPositionUncheckedAsync(Stream stream, PxFileConfiguration conf, int bufferSize = 4096, CancellationToken cancellationToken = default) + { + return FindDataStartPositionUncheckedImplAsync(stream, conf, bufferSize, cancellationToken); + } + /// /// Asynchronously finds the absolute raw byte offset of the first non-whitespace data value after a top-level DATA entry. /// The stream position is restored before this method returns. Returns -1 when the DATA entry or its first value cannot be found. @@ -84,7 +111,10 @@ private static long FindKeywordPositionImpl(Stream stream, string keyword, PxFil { byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); byte[] buffer = new byte[bufferSize]; - KeywordSearchState state = new(keywordBytes, (byte)conf.Symbols.EntrySeparator); + TopLevelKeywordSearchState state = new( + keywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.Key.StringDelimeter); int bytesRead; while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) @@ -100,11 +130,52 @@ private static long FindKeywordPositionImpl(Stream stream, string keyword, PxFil return -1; } + private static long FindDataStartPositionUncheckedImpl(Stream stream, PxFileConfiguration conf, int bufferSize) + { + byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data + conf.Symbols.KeywordSeparator); + byte[] buffer = new byte[bufferSize]; + UncheckedDataStartSearchState state = new(dataKeywordBytes, (byte)conf.Symbols.EntrySeparator, (byte)conf.Symbols.Key.StringDelimeter); + + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + { + long bufferStart = stream.Position - bytesRead; + if (TryFindDataStartPositionUnchecked(buffer.AsSpan(0, bytesRead), bufferStart, ref state, out long dataStartPosition)) + { + return dataStartPosition; + } + } + + return -1; + } + + private static async Task FindDataStartPositionUncheckedImplAsync(Stream stream, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) + { + byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data + conf.Symbols.KeywordSeparator); + byte[] buffer = new byte[bufferSize]; + UncheckedDataStartSearchState state = new(dataKeywordBytes, (byte)conf.Symbols.EntrySeparator, (byte)conf.Symbols.Key.StringDelimeter); + + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) + { + long bufferStart = stream.Position - bytesRead; + if (TryFindDataStartPositionUnchecked(buffer.AsSpan(0, bytesRead), bufferStart, ref state, out long dataStartPosition)) + { + return dataStartPosition; + } + } + + return -1; + } + private static async Task FindKeywordPositionImplAsync(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) { byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); byte[] buffer = new byte[bufferSize]; - KeywordSearchState state = new(keywordBytes, (byte)conf.Symbols.EntrySeparator); + TopLevelKeywordSearchState state = new( + keywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.Key.StringDelimeter); int bytesRead; while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) @@ -121,11 +192,11 @@ private static async Task FindKeywordPositionImplAsync(Stream stream, stri } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static long FindKeywordPosition(ReadOnlySpan buffer, long bufferStart, ref KeywordSearchState state) + private static long FindKeywordPosition(ReadOnlySpan buffer, long bufferStart, ref TopLevelKeywordSearchState state) { for (int i = 0; i < buffer.Length; i++) { - if (TryProcessKeywordByte(buffer[i], ref state)) + if (TryProcessTopLevelKeywordByte(buffer[i], ref state)) { return bufferStart + i - state.KeywordBytes.Length + 1; } @@ -240,7 +311,7 @@ private static bool TryProcessDataStartByte(byte currentByte, ref DataStartSearc state.KeywordSearchState.Reset(); return false; } - if (TryProcessKeywordByte(currentByte, ref state.KeywordSearchState)) + if (TryProcessEntryKeywordByte(currentByte, ref state.KeywordSearchState)) { state.IsAfterDataKeyword = true; } @@ -249,7 +320,39 @@ private static bool TryProcessDataStartByte(byte currentByte, ref DataStartSearc } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool TryProcessKeywordByte(byte currentByte, ref KeywordSearchState state) + private static bool TryFindDataStartPositionUnchecked(ReadOnlySpan buffer, long bufferStart, ref UncheckedDataStartSearchState state, out long dataStartPosition) + { + for (int i = 0; i < buffer.Length; i++) + { + byte currentByte = buffer[i]; + if (state.IsAfterDataKeyword) + { + if (currentByte == state.EntrySeparator) + { + dataStartPosition = -1; + return true; + } + if (!IsWhitespace(currentByte)) + { + dataStartPosition = bufferStart + i; + return true; + } + + continue; + } + + if (TryProcessTopLevelKeywordByte(currentByte, ref state.KeywordSearchState)) + { + state.IsAfterDataKeyword = true; + } + } + + dataStartPosition = -1; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryProcessEntryKeywordByte(byte currentByte, ref EntryKeywordSearchState state) { if (currentByte == state.EntrySeparator) { @@ -271,6 +374,39 @@ private static bool TryProcessKeywordByte(byte currentByte, ref KeywordSearchSta return false; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryProcessTopLevelKeywordByte(byte currentByte, ref TopLevelKeywordSearchState state) + { + if (state.IsInString) + { + state.IsInString = currentByte != state.StringDelimiter; + return false; + } + if (currentByte == state.StringDelimiter) + { + state.IsInString = true; + return false; + } + if (currentByte == state.EntrySeparator) + { + state.Reset(); + return false; + } + if (!state.IsAtEntryStart || IsWhitespace(currentByte)) + { + return false; + } + if (state.MatchedKeywordBytes < state.KeywordBytes.Length && currentByte == state.KeywordBytes[state.MatchedKeywordBytes]) + { + state.MatchedKeywordBytes++; + return state.MatchedKeywordBytes == state.KeywordBytes.Length; + } + + state.IsAtEntryStart = false; + state.MatchedKeywordBytes = 0; + return false; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsWhitespace(byte value) { @@ -281,18 +417,42 @@ private struct DataStartSearchState(byte[] dataKeywordBytes, byte entrySeparator { public readonly byte EntrySeparator = entrySeparator; public readonly byte StringDelimiter = stringDelimiter; - public KeywordSearchState KeywordSearchState = new([.. dataKeywordBytes, keywordSeparator], entrySeparator); + public EntryKeywordSearchState KeywordSearchState = new([.. dataKeywordBytes, keywordSeparator], entrySeparator); public bool IsInString; public bool IsAfterDataKeyword; public bool IsDataEntryEmpty; } - private struct KeywordSearchState(byte[] keywordBytes, byte entrySeparator) + private struct UncheckedDataStartSearchState(byte[] dataKeywordBytes, byte entrySeparator, byte stringDelimiter) + { + public readonly byte EntrySeparator = entrySeparator; + public TopLevelKeywordSearchState KeywordSearchState = new(dataKeywordBytes, entrySeparator, stringDelimiter); + public bool IsAfterDataKeyword; + } + + private struct EntryKeywordSearchState(byte[] keywordBytes, byte entrySeparator) + { + public readonly byte[] KeywordBytes = keywordBytes; + public readonly byte EntrySeparator = entrySeparator; + public int MatchedKeywordBytes; + public bool IsAtEntryStart = true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Reset() + { + MatchedKeywordBytes = 0; + IsAtEntryStart = true; + } + } + + private struct TopLevelKeywordSearchState(byte[] keywordBytes, byte entrySeparator, byte stringDelimiter) { public readonly byte[] KeywordBytes = keywordBytes; public readonly byte EntrySeparator = entrySeparator; + public readonly byte StringDelimiter = stringDelimiter; public int MatchedKeywordBytes; public bool IsAtEntryStart = true; + public bool IsInString; [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Reset() diff --git a/docs/architecture.readers.md b/docs/architecture.readers.md index 353de76c..aac5ae55 100644 --- a/docs/architecture.readers.md +++ b/docs/architecture.readers.md @@ -32,7 +32,7 @@ void ReadDecimalDataValues(DecimalDataValue[] buffer, int offset, IMatrixMap tar Async variants available. Implements `IDisposable`. Depends on: `PxFileConfiguration`. -When constructed at stream position `0`, `PxFileStreamDataReader` locates the first non-whitespace value following the top-level `DATA=` entry before reading. The overload that accepts `dataStart` requires that value's absolute raw byte offset. +When constructed at stream position `0`, `PxFileStreamDataReader` uses an internal trusted-input locator to find the first non-whitespace value following the top-level `DATA=` entry. This path retains quote-aware top-level matching but assumes that the PX syntax was validated before reading. The overload that accepts `dataStart` requires that value's absolute raw byte offset and avoids a metadata rescan; use a validated `SyntaxValidationResult.DataStartStreamPosition` when one is available. ### Helpers @@ -40,7 +40,7 @@ When constructed at stream position `0`, `PxFileStreamDataReader` locates the fi |---|---| | `DataIndexer.cs` | Index mapping between source and target `IMatrixMap` | | `DataValueParsers.cs` | Parse raw data values from byte spans | -| `StreamUtilities.cs` | `FindDataStartPosition` and async equivalent locate the first value after top-level `DATA=` as an absolute raw byte offset, preserving the original position of a seekable stream. `FindKeywordPosition` and async equivalent locate any top-level entry keyword from the current position and leave the stream advanced. | +| `StreamUtilities.cs` | Public `FindDataStartPosition` and async equivalent are validation-aware: they search from origin, return the first value after top-level `DATA=` as an absolute raw byte offset, and restore the original position of a seekable stream. Internal `FindDataStartPositionUnchecked` equivalents are for validated reader input, search from the current position, and leave the stream advanced. `FindKeywordPosition` and async equivalent locate quote-aware top-level entry keywords from the current position and leave the stream advanced. | ## Binary Data From cb2cf12d9994f55671b4bf1e388cae7bf8dc13b9 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 6 Aug 2026 15:46:58 +0300 Subject: [PATCH 09/12] Fixes --- .../DataTests/StreamUtilitiesTests.cs | 83 ++++++++++++++++++- Px.Utils/PxFile/Data/StreamUtilities.cs | 34 ++++++++ Px.Utils/Validation/PxFileValidator.cs | 8 +- 3 files changed, 120 insertions(+), 5 deletions(-) diff --git a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs index 1ff77127..d311c9f4 100644 --- a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs +++ b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs @@ -46,6 +46,61 @@ public async Task FindKeywordPositionAsyncKeywordSplitAcrossBuffersReturnsKeywor Assert.IsTrue(stream.Position > position); } + [TestMethod] + public void FindKeywordPositionAtStartOfMetadataReturnsFirstValueOffset() + { + // Arrange + string content = "METADATA=\"FOO\";\r\nDATA=1;"; + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + byte[] data = contentBytes; + using Stream stream = new MemoryStream(data); + long expectedPosition = Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("METADATA=", StringComparison.Ordinal)]); + + // Act + long position = StreamUtilities.FindKeywordPosition(stream, "METADATA", PxFileConfiguration.Default, 2); + + // Assert + Assert.AreEqual(expectedPosition, position); + } + + [TestMethod] + public async Task FindKeywordPositionAtStreamOriginWithBomReturnsRawKeywordOffset() + { + // Arrange + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] data = [.. bom, .. Encoding.UTF8.GetBytes("DATA=1;")]; + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + + // Act + long synchronousPosition = StreamUtilities.FindKeywordPosition(synchronousStream, "DATA", PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindKeywordPositionAsync(asynchronousStream, "DATA", PxFileConfiguration.Default, TestContext.CancellationToken, 2); + + // Assert + Assert.AreEqual(bom.Length, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.IsGreaterThan(synchronousPosition, synchronousStream.Position); + Assert.IsGreaterThan(asynchronousPosition, asynchronousStream.Position); + } + + [TestMethod] + public void FindKeywordPositionAtStartOfMetadataWithBomReturnsFirstValueOffset() + { + // Arrange + string content = "METADATA=\"FOO\";\r\nDATA=1;"; + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + byte[] data = [.. bom, .. contentBytes]; + using Stream stream = new MemoryStream(data); + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("METADATA=", StringComparison.Ordinal)]); + + // Act + long position = StreamUtilities.FindKeywordPosition(stream, "METADATA", PxFileConfiguration.Default, 2); + + // Assert + Assert.AreEqual(expectedPosition, position); + } + [TestMethod] [DataRow("1")] [DataRow("-1")] @@ -96,6 +151,25 @@ public async Task FindDataStartPositionAsyncDataSplitAcrossBuffersReturnsSameOff Assert.AreEqual(0, stream.Position); } + [TestMethod] + public async Task FindDataStartPositionAtStreamOriginWithBomReturnsFirstValueOffset() + { + // Arrange + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] data = [.. bom, .. Encoding.UTF8.GetBytes("DATA=\r\n\t1;")]; + long expectedPosition = bom.Length + "DATA=\r\n\t".Length; + using Stream stream = new MemoryStream(data); + + // Act + long synchronousPosition = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionAsync(stream, PxFileConfiguration.Default, 2, TestContext.CancellationToken); + + // Assert + Assert.AreEqual(expectedPosition, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.AreEqual(0, stream.Position); + } + [TestMethod] public async Task FindKeywordPositionQuotedKeywordWithBomAndMultibyteMetadataReturnsRawTopLevelOffset() { @@ -118,12 +192,15 @@ public async Task FindKeywordPositionQuotedKeywordWithBomAndMultibyteMetadataRet } [TestMethod] - public async Task FindDataStartPositionUncheckedQuotedKeywordAndWhitespaceReturnsOffsetAndAdvancesStream() + [DataRow(false)] + [DataRow(true)] + public async Task FindDataStartPositionUncheckedQuotedKeywordAndWhitespaceReturnsOffsetAndAdvancesStream(bool includesUtf8Bom) { // Arrange string content = "TITLE=\"DATA=not-an-entry\";\nVALUES=\"Ää\";\nDATA=\r\n\t1 2;"; - byte[] data = [.. Encoding.UTF8.GetPreamble(), .. Encoding.UTF8.GetBytes(content)]; - long expectedPosition = Encoding.UTF8.GetPreamble().Length + Encoding.UTF8.GetByteCount(content[..content.IndexOf('1')]); + byte[] bom = includesUtf8Bom ? Encoding.UTF8.GetPreamble() : []; + byte[] data = [.. bom, .. Encoding.UTF8.GetBytes(content)]; + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..content.IndexOf('1')]); using Stream synchronousStream = new MemoryStream(data); using Stream asynchronousStream = new MemoryStream(data); diff --git a/Px.Utils/PxFile/Data/StreamUtilities.cs b/Px.Utils/PxFile/Data/StreamUtilities.cs index a5663f64..720ed743 100644 --- a/Px.Utils/PxFile/Data/StreamUtilities.cs +++ b/Px.Utils/PxFile/Data/StreamUtilities.cs @@ -109,6 +109,7 @@ public static async Task FindDataStartPositionAsync(Stream stream, PxFileC private static long FindKeywordPositionImpl(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize) { + SkipUtf8BomAtStreamOrigin(stream); byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); byte[] buffer = new byte[bufferSize]; TopLevelKeywordSearchState state = new( @@ -170,6 +171,7 @@ private static async Task FindDataStartPositionUncheckedImplAsync(Stream s private static async Task FindKeywordPositionImplAsync(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) { + await SkipUtf8BomAtStreamOriginAsync(stream, cancellationToken); byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); byte[] buffer = new byte[bufferSize]; TopLevelKeywordSearchState state = new( @@ -207,6 +209,7 @@ private static long FindKeywordPosition(ReadOnlySpan buffer, long bufferSt private static long FindDataStartPositionImpl(Stream stream, PxFileConfiguration conf, int bufferSize) { + SkipUtf8BomAtStreamOrigin(stream); byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); byte[] buffer = new byte[bufferSize]; DataStartSearchState state = new( @@ -234,6 +237,7 @@ private static long FindDataStartPositionImpl(Stream stream, PxFileConfiguration private static async Task FindDataStartPositionImplAsync(Stream stream, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) { + await SkipUtf8BomAtStreamOriginAsync(stream, cancellationToken); byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); byte[] buffer = new byte[bufferSize]; DataStartSearchState state = new( @@ -413,6 +417,36 @@ private static bool IsWhitespace(byte value) return value is CharacterConstants.SPACE or CharacterConstants.HORIZONTALTAB or CharacterConstants.CARRIAGERETURN or CharacterConstants.LINEFEED; } + private static void SkipUtf8BomAtStreamOrigin(Stream stream) + { + if (!stream.CanSeek || stream.Position != 0 || stream.Length < Encoding.UTF8.Preamble.Length) + { + return; + } + + Span prefix = stackalloc byte[Encoding.UTF8.Preamble.Length]; + int bytesRead = stream.Read(prefix); + if (bytesRead != prefix.Length || !prefix.SequenceEqual(Encoding.UTF8.Preamble)) + { + stream.Position = 0; + } + } + + private static async Task SkipUtf8BomAtStreamOriginAsync(Stream stream, CancellationToken cancellationToken) + { + if (!stream.CanSeek || stream.Position != 0 || stream.Length < Encoding.UTF8.Preamble.Length) + { + return; + } + + byte[] prefix = new byte[Encoding.UTF8.Preamble.Length]; + int bytesRead = await stream.ReadAsync(prefix, cancellationToken); + if (bytesRead != prefix.Length || !prefix.AsSpan().SequenceEqual(Encoding.UTF8.Preamble)) + { + stream.Position = 0; + } + } + private struct DataStartSearchState(byte[] dataKeywordBytes, byte entrySeparator, byte keywordSeparator, byte stringDelimiter) { public readonly byte EntrySeparator = entrySeparator; diff --git a/Px.Utils/Validation/PxFileValidator.cs b/Px.Utils/Validation/PxFileValidator.cs index 2bf6f795..c93e0be0 100644 --- a/Px.Utils/Validation/PxFileValidator.cs +++ b/Px.Utils/Validation/PxFileValidator.cs @@ -92,10 +92,12 @@ internal ValidationResult Validate( IFileSystem? fileSystem, ValidationFeedbackSink sink) { + fileSystem ??= new LocalFileSystem(); + if (encoding is null) { long originalPosition = stream.Position; - encoding = new LocalFileSystem().GetEncoding(stream); + encoding = fileSystem.GetEncoding(stream); stream.Position = originalPosition; } @@ -191,10 +193,11 @@ internal async Task ValidateAsync( ValidationFeedbackSink sink, CancellationToken cancellationToken = default) { + fileSystem ??= new LocalFileSystem(); if (encoding is null) { long originalPosition = stream.Position; - encoding = await new LocalFileSystem().GetEncodingAsync(stream, cancellationToken); + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); stream.Position = originalPosition; } @@ -214,6 +217,7 @@ internal async Task ValidateAsync( new(filename, 0, 0) )); + stream.Close(); return new(sink.ToFeedback()); } From 60676f51734f670cf513d37a60edfe9d1c259bc0 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Fri, 7 Aug 2026 09:33:31 +0300 Subject: [PATCH 10/12] Specify supported encodings --- docs/PXFILE_SPECIFICATION.md | 6 ++++++ docs/README.md | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/docs/PXFILE_SPECIFICATION.md b/docs/PXFILE_SPECIFICATION.md index dc75d5be..24f6c3b9 100644 --- a/docs/PXFILE_SPECIFICATION.md +++ b/docs/PXFILE_SPECIFICATION.md @@ -2,6 +2,12 @@ Purpose of this document is to describe the syntax and content requirements for the pxfiles supported by this library. These requirements do not necessarily apply to other implementations of the pxfile format. +## Supported text encodings + +Supported PX file encodings are ASCII-compatible single-byte encodings, including ANSI code pages declared through `CODEPAGE`, and UTF-8. UTF-8 files may include a BOM or omit it, and metadata strings may contain multibyte UTF-8 characters. + +UTF-16 and UTF-32 are not supported, regardless of byte order or BOM presence. + ## Syntax ### Entries diff --git a/docs/README.md b/docs/README.md index 7f0a4fbf..084dfc62 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,12 @@ The read pipeline consists of the following components: Reading the metadata, bu Each of these components can be used separately and replaced with custom implementations. Especially if the contents of your px-files do not follow the standard px-file format, you might need to implement your own metadata builder. +### Supported text encodings + +Px.Utils supports ASCII-compatible single-byte encodings, including ANSI code pages specified through `CODEPAGE`, and UTF-8. UTF-8 files may be provided with or without a BOM. UTF-8 metadata may contain multibyte characters. + +UTF-16 and UTF-32, in either byte order and with or without a BOM, are not supported for full PX file reading or validation. + #### PxFileMetadataReader : IPxFileMetadataReader ```ReadMetadata(Stream stream, Encoding encoding)``` reads the metadata entries from the provided stream as a IEnumerable of ```KeyValuePair``` representing the keys and values of the entries. **This method does not perform any validation on the metadata entries.** From 51ae6aa455c8288c98863b221a13edc2b23d63ec Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Fri, 7 Aug 2026 09:34:03 +0300 Subject: [PATCH 11/12] Optimise multi stage validation pipeline --- .../ContentValidationOutput.cs | 4 + .../ContentValidation/ContentValidator.cs | 7 +- .../DataValidation/DataValidator.cs | 76 +++++++++---------- .../DatabaseValidation/DatabaseValidator.cs | 4 +- Px.Utils/Validation/PxFileValidator.cs | 70 ++++++++++------- .../SyntaxValidationOutput.cs | 7 ++ .../SyntaxValidation/SyntaxValidator.cs | 14 ++-- 7 files changed, 104 insertions(+), 78 deletions(-) create mode 100644 Px.Utils/Validation/ContentValidation/ContentValidationOutput.cs create mode 100644 Px.Utils/Validation/SyntaxValidation/SyntaxValidationOutput.cs diff --git a/Px.Utils/Validation/ContentValidation/ContentValidationOutput.cs b/Px.Utils/Validation/ContentValidation/ContentValidationOutput.cs new file mode 100644 index 00000000..79644a4d --- /dev/null +++ b/Px.Utils/Validation/ContentValidation/ContentValidationOutput.cs @@ -0,0 +1,4 @@ +namespace Px.Utils.Validation.ContentValidation +{ + internal readonly record struct ContentValidationOutput(int DataRowLength, int DataRowAmount); +} \ No newline at end of file diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.cs index 1a1a2883..5ab3f849 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.cs @@ -101,10 +101,11 @@ public ContentValidationResult Validate() public ContentValidationResult Validate(ValidationOptions options) { ValidationFeedbackSink sink = new(options); - return Validate(sink); + ContentValidationOutput output = ValidateIntoSink(sink); + return new ContentValidationResult(sink.ToFeedback(), output.DataRowLength, output.DataRowAmount); } - internal ContentValidationResult Validate(ValidationFeedbackSink sink) + internal ContentValidationOutput ValidateIntoSink(ValidationFeedbackSink sink) { IEnumerable contentValidationEntryFunctions = DefaultContentValidationEntryFunctions; IEnumerable contentValidationFindKeywordFunctions = DefaultContentValidationFindKeywordFunctions; @@ -138,7 +139,7 @@ internal ContentValidationResult Validate(ValidationFeedbackSink sink) int amountOfDataRows = _stubDimensionNames is not null ? GetProductOfDimensionValues(_stubDimensionNames) : 0; ResetFields(); - return new ContentValidationResult(sink.ToFeedback(), lengthOfDataRows, amountOfDataRows); + return new ContentValidationOutput(lengthOfDataRows, amountOfDataRows); } #region Interface implementation diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index 59593d8e..8120df85 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -100,10 +100,11 @@ public ValidationResult Validate( ValidationOptions options) { ValidationFeedbackSink sink = new(options); - return Validate(stream, filename, encoding, fileSystem, sink); + ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new ValidationResult(sink.ToFeedback()); } - internal ValidationResult Validate( + internal void ValidateIntoSink( Stream stream, string filename, Encoding? encoding, @@ -126,14 +127,12 @@ internal ValidationResult Validate( new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), new(filename, 0, 0))); ResetValidator(); - return new ValidationResult(sink.ToFeedback()); + return; } stream.Position = dataStartIndex; ValidateDataStream(stream, sink); ResetValidator(); - - return new ValidationResult(sink.ToFeedback()); } /// @@ -155,40 +154,7 @@ public async Task ValidateAsync( CancellationToken cancellationToken = default) { ValidationFeedbackSink sink = new(options); - return await ValidateAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); - } - - internal async Task ValidateAsync( - Stream stream, - string filename, - Encoding? encoding, - IFileSystem? fileSystem, - ValidationFeedbackSink sink, - CancellationToken cancellationToken = default) - { - fileSystem ??= new LocalFileSystem(); - if (encoding is null) - { - long originalPosition = stream.Position; - encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); - stream.Position = originalPosition; - } - SetValidationParameters(encoding, filename); - - long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); - if (dataStartIndex == -1) - { - sink.Report(new( - new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), - new(filename, 0, 0))); - ResetValidator(); - return new ValidationResult(sink.ToFeedback()); - } - - stream.Position = dataStartIndex; - await Task.Factory.StartNew(() => ValidateDataStream(stream, sink, cancellationToken), cancellationToken); - ResetValidator(); - + await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); return new ValidationResult(sink.ToFeedback()); } @@ -240,6 +206,38 @@ public async Task ValidateAsync( return new(validationFeedbacks); } + internal async Task ValidateIntoSinkAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink, + CancellationToken cancellationToken = default) + { + fileSystem ??= new LocalFileSystem(); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } + SetValidationParameters(encoding, filename); + + long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); + if (dataStartIndex == -1) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), + new(filename, 0, 0))); + ResetValidator(); + return; + } + + stream.Position = dataStartIndex; + await Task.Factory.StartNew(() => ValidateDataStream(stream, sink, cancellationToken), cancellationToken); + ResetValidator(); + } + private void SetValidationParameters(Encoding encoding, string filename) { _commonValidators.Add(new DataStructureValidator()); diff --git a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs index 60b00f63..59c6eec2 100644 --- a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs +++ b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs @@ -137,7 +137,7 @@ public async Task ValidateAsync(ValidationOptions options, Can } stream.Position = 0; PxFileValidator validator = new(_conf); - validator.Validate(stream, fileName, fileInfo.Encoding, null, sink); + validator.ValidateIntoSink(stream, fileName, fileInfo.Encoding, null, sink); return (fileInfo, feedbacks); } @@ -159,7 +159,7 @@ private DatabaseFileInfo ProcessAliasFile(string fileName) } stream.Position = 0; PxFileValidator validator = new(_conf); - await validator.ValidateAsync(stream, fileName, fileInfo.Encoding, null, sink, cancellationToken); + await validator.ValidateIntoSinkAsync(stream, fileName, fileInfo.Encoding, null, sink, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); return (fileInfo, feedbacks); } diff --git a/Px.Utils/Validation/PxFileValidator.cs b/Px.Utils/Validation/PxFileValidator.cs index c93e0be0..70a49885 100644 --- a/Px.Utils/Validation/PxFileValidator.cs +++ b/Px.Utils/Validation/PxFileValidator.cs @@ -66,7 +66,11 @@ public ValidationResult Validate( string filename, Encoding? encoding = null, IFileSystem? fileSystem = null) - => Validate(stream, filename, encoding, fileSystem, new ValidationFeedbackSink()); + { + ValidationFeedbackSink sink = new(); + ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new ValidationResult(sink.ToFeedback()); + } /// /// Validates the PX file using the specified feedback retention options. @@ -83,9 +87,13 @@ public ValidationResult Validate( Encoding? encoding, IFileSystem? fileSystem, ValidationOptions options) - => Validate(stream, filename, encoding, fileSystem, new ValidationFeedbackSink(options)); + { + ValidationFeedbackSink sink = new(options); + ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new ValidationResult(sink.ToFeedback()); + } - internal ValidationResult Validate( + internal void ValidateIntoSink( Stream stream, string filename, Encoding? encoding, @@ -104,12 +112,12 @@ internal ValidationResult Validate( conf ??= PxFileConfiguration.Default; SyntaxValidator syntaxValidator = new(conf, _customSyntaxValidationFunctions); - SyntaxValidationResult syntaxValidationResult = syntaxValidator.Validate(stream, filename, encoding, fileSystem, sink); + SyntaxValidationOutput syntaxValidationOutput = syntaxValidator.ValidateIntoSink(stream, filename, encoding, fileSystem, sink); - ContentValidator contentValidator = new(filename, encoding, [.. syntaxValidationResult.Result], _customContentValidationFunctions, conf); - ContentValidationResult contentValidationResult = contentValidator.Validate(sink); + ContentValidator contentValidator = new(filename, encoding, [.. syntaxValidationOutput.StructuredEntries], _customContentValidationFunctions, conf); + ContentValidationOutput contentValidationOutput = contentValidator.ValidateIntoSink(sink); - if (syntaxValidationResult.DataStartStreamPosition == -1) + if (syntaxValidationOutput.DataStartStreamPosition == -1) { sink.Report(new KeyValuePair( new(ValidationFeedbackLevel.Error, @@ -117,16 +125,16 @@ internal ValidationResult Validate( new(filename, 0, 0) )); - return new(sink.ToFeedback()); + return; } - stream.Position = syntaxValidationResult.DataStartStreamPosition; + stream.Position = syntaxValidationOutput.DataStartStreamPosition; DataValidator dataValidator = new( - contentValidationResult.DataRowLength, - contentValidationResult.DataRowAmount, - syntaxValidationResult.DataStartRow, + contentValidationOutput.DataRowLength, + contentValidationOutput.DataRowAmount, + syntaxValidationOutput.DataStartRow, conf); - dataValidator.Validate(stream, filename, encoding, fileSystem, sink); + dataValidator.ValidateIntoSink(stream, filename, encoding, fileSystem, sink); if (_customStreamValidators is not null) { @@ -145,7 +153,6 @@ internal ValidationResult Validate( } } stream.Close(); - return new ValidationResult(sink.ToFeedback()); } /// @@ -164,7 +171,11 @@ public async Task ValidateAsync( Encoding? encoding = null, IFileSystem? fileSystem = null, CancellationToken cancellationToken = default) - => await ValidateAsync(stream, filename, encoding, fileSystem, new ValidationFeedbackSink(), cancellationToken); + { + ValidationFeedbackSink sink = new(); + await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new ValidationResult(sink.ToFeedback()); + } /// /// Asynchronously validates the PX file using the specified feedback retention options. @@ -183,9 +194,13 @@ public async Task ValidateAsync( IFileSystem? fileSystem, ValidationOptions options, CancellationToken cancellationToken = default) - => await ValidateAsync(stream, filename, encoding, fileSystem, new ValidationFeedbackSink(options), cancellationToken); + { + ValidationFeedbackSink sink = new(options); + await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new ValidationResult(sink.ToFeedback()); + } - internal async Task ValidateAsync( + internal async Task ValidateIntoSinkAsync( Stream stream, string filename, Encoding? encoding, @@ -204,12 +219,12 @@ internal async Task ValidateAsync( conf ??= PxFileConfiguration.Default; SyntaxValidator syntaxValidator = new(conf, _customSyntaxValidationFunctions); - SyntaxValidationResult syntaxValidationResult = await syntaxValidator.ValidateAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + SyntaxValidationOutput syntaxValidationOutput = await syntaxValidator.ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); - ContentValidator contentValidator = new(filename, encoding, [..syntaxValidationResult.Result], _customContentValidationFunctions, conf); - ContentValidationResult contentValidationResult = contentValidator.Validate(sink); + ContentValidator contentValidator = new(filename, encoding, [..syntaxValidationOutput.StructuredEntries], _customContentValidationFunctions, conf); + ContentValidationOutput contentValidationOutput = contentValidator.ValidateIntoSink(sink); - if (syntaxValidationResult.DataStartStreamPosition == -1) + if (syntaxValidationOutput.DataStartStreamPosition == -1) { sink.Report(new KeyValuePair( new(ValidationFeedbackLevel.Error, @@ -218,17 +233,17 @@ internal async Task ValidateAsync( )); stream.Close(); - return new(sink.ToFeedback()); + return; } - stream.Position = syntaxValidationResult.DataStartStreamPosition; + stream.Position = syntaxValidationOutput.DataStartStreamPosition; DataValidator dataValidator = new( - contentValidationResult.DataRowLength, - contentValidationResult.DataRowAmount, - syntaxValidationResult.DataStartRow, + contentValidationOutput.DataRowLength, + contentValidationOutput.DataRowAmount, + syntaxValidationOutput.DataStartRow, conf); - await dataValidator.ValidateAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + await dataValidator.ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); if (_customStreamAsyncValidators is not null) { @@ -247,7 +262,6 @@ internal async Task ValidateAsync( } } stream.Close(); - return new ValidationResult(sink.ToFeedback()); } } } diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationOutput.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationOutput.cs new file mode 100644 index 00000000..9bcb58d3 --- /dev/null +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationOutput.cs @@ -0,0 +1,7 @@ +namespace Px.Utils.Validation.SyntaxValidation +{ + internal sealed record SyntaxValidationOutput( + List StructuredEntries, + int DataStartRow, + long DataStartStreamPosition); +} \ No newline at end of file diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs index 88251adf..4de1fec5 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs @@ -88,10 +88,11 @@ public SyntaxValidationResult Validate( ValidationOptions options) { ValidationFeedbackSink sink = new(options); - return Validate(stream, filename, encoding, fileSystem, sink); + SyntaxValidationOutput output = ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new SyntaxValidationResult(sink.ToFeedback(), output.StructuredEntries, output.DataStartRow, output.DataStartStreamPosition); } - internal SyntaxValidationResult Validate( + internal SyntaxValidationOutput ValidateIntoSink( Stream stream, string filename, Encoding? encoding, @@ -129,7 +130,7 @@ internal SyntaxValidationResult Validate( List structuredEntries = BuildValidationStructureEntries(keyValuePairs, conf); ReportStructuredFeedback(structuredEntries, structuredValidationFunctions, conf, sink); - return new SyntaxValidationResult(sink.ToFeedback(), structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); + return new SyntaxValidationOutput(structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); } /// @@ -202,10 +203,11 @@ public async Task ValidateAsync( CancellationToken cancellationToken = default) { ValidationFeedbackSink sink = new(options); - return await ValidateAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + SyntaxValidationOutput output = await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new SyntaxValidationResult(sink.ToFeedback(), output.StructuredEntries, output.DataStartRow, output.DataStartStreamPosition); } - internal async Task ValidateAsync( + internal async Task ValidateIntoSinkAsync( Stream stream, string filename, Encoding? encoding, @@ -243,7 +245,7 @@ internal async Task ValidateAsync( List structuredEntries = BuildValidationStructureEntries(keyValuePairs, conf); ReportStructuredFeedback(structuredEntries, structuredValidationFunctions, conf, sink); - return new SyntaxValidationResult(sink.ToFeedback(), structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); + return new SyntaxValidationOutput(structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); } #region Interface implementation From fff0fb5bc4ce5f2ec719023fb3a163951483b01e Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Fri, 7 Aug 2026 10:13:17 +0300 Subject: [PATCH 12/12] Cleanup --- .../ContentValidationTests.cs | 17 +++ .../DataValidationTests/DataValidationTest.cs | 20 +++ .../StreamSyntaxValidationTests.cs | 112 +++++++++++---- .../ContentValidation/ContentValidator.cs | 38 +----- .../DataValidation/DataValidator.cs | 65 +-------- .../SyntaxValidation/SyntaxValidator.cs | 128 +----------------- 6 files changed, 139 insertions(+), 241 deletions(-) diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index 909b0201..b003d3f1 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -70,6 +70,23 @@ public void ValidateWithLimitRepeatedCustomContentFeedbackRetainsOnlyConfiguredC Assert.Contains("Feedback limit of 2 instances", result.FeedbackItems[key][^1].AdditionalInfo!); } + [TestMethod] + public void ValidateDefaultOverloadRepeatedCustomContentFeedbackRetainsDefaultFeedbackCount() + { + static ValidationFeedback? entryValidator(ValidationStructuredEntry entry, ContentValidator _) => new(new( + new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.ValueIsNotInUpperCase), + new(entry.File, entry.KeyStartLineIndex))); + CustomContentValidationFunctions functions = new([], [entryValidator]); + ValidationStructuredEntry[] entries = [.. Enumerable.Range(1, 101).Select(index => new ValidationStructuredEntry("content.px", new($"KEY{index}", null, null, null), "1", index, [], 0, null))]; + ContentValidator validator = new("content.px", Encoding.UTF8, entries, functions); + + ContentValidationResult result = validator.Validate(); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.ValueIsNotInUpperCase); + Assert.HasCount(100, result.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", result.FeedbackItems[key][^1].AdditionalInfo!); + } + [TestMethod] public void ValidateCalledWithSharedDimensionNameAcrossLanguagesCalculatesRowCountsFromDefaultLanguage() { diff --git a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs index 88aed54d..0b2cff28 100644 --- a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs +++ b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs @@ -79,6 +79,26 @@ public void ValidateWithLimitLargeInvalidDataRetainsOnlyConfiguredFeedbackCount( Assert.Contains("Feedback limit of 3 instances", result.FeedbackItems[key][^1].AdditionalInfo!); } + [TestMethod] + public async Task ValidateDefaultOverloadsLargeInvalidDataRetainDefaultFeedbackCount() + { + string invalidData = string.Concat(Enumerable.Repeat("! ", 200)) + ";"; + byte[] data = Encoding.UTF8.GetBytes("DATA=" + invalidData); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + DataValidator synchronousValidator = new(0, 0, 0); + DataValidator asynchronousValidator = new(0, 0, 0); + + ValidationResult synchronousResult = synchronousValidator.Validate(synchronousStream, "invalid.px", Encoding.UTF8); + ValidationResult asynchronousResult = await asynchronousValidator.ValidateAsync(asynchronousStream, "invalid.px", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + Assert.HasCount(100, synchronousResult.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", synchronousResult.FeedbackItems[key][^1].AdditionalInfo!); + Assert.HasCount(100, asynchronousResult.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", asynchronousResult.FeedbackItems[key][^1].AdditionalInfo!); + } + [TestMethod] [DataRow(DataStreamContents.SIMPLE_VALID_DATA, 0, 0)] [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] diff --git a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs index b42b52a1..90df93e6 100644 --- a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs @@ -23,15 +23,54 @@ public class StreamSyntaxValidationTests public void Initialize() { entryValidationMethod = typeof(SyntaxValidator) - .GetMethod("ValidateEntries", BindingFlags.NonPublic | BindingFlags.Static); + .GetMethod("ReportEntryFeedback", BindingFlags.NonPublic | BindingFlags.Static); kvpValidationMethod = typeof(SyntaxValidator) - .GetMethod("ValidateKeyValuePairs", BindingFlags.NonPublic | BindingFlags.Static); + .GetMethod("ReportKeyValuePairFeedback", BindingFlags.NonPublic | BindingFlags.Static); structuredValidationMethod = typeof(SyntaxValidator) - .GetMethod("ValidateStructs", BindingFlags.NonPublic | BindingFlags.Static); + .GetMethod("ReportStructuredFeedback", BindingFlags.NonPublic | BindingFlags.Static); getValueTypeFromStringMethod = typeof(SyntaxValidationUtilityMethods) .GetMethod("GetValueTypeFromString", BindingFlags.NonPublic | BindingFlags.Static); } + private ValidationFeedback ReportEntryFeedback( + IEnumerable entries, + IEnumerable functions) + { + object sink = CreateFeedbackSink(); + entryValidationMethod!.Invoke(null, [entries, functions, conf, sink]); + return GetFeedback(sink); + } + + private ValidationFeedback ReportKeyValuePairFeedback( + IEnumerable keyValuePairs, + IEnumerable functions) + { + object sink = CreateFeedbackSink(); + kvpValidationMethod!.Invoke(null, [keyValuePairs, functions, conf, sink]); + return GetFeedback(sink); + } + + private ValidationFeedback ReportStructuredFeedback( + IEnumerable structuredEntries, + IEnumerable functions) + { + object sink = CreateFeedbackSink(); + structuredValidationMethod!.Invoke(null, [structuredEntries, functions, conf, sink]); + return GetFeedback(sink); + } + + private static object CreateFeedbackSink() + { + Type sinkType = typeof(SyntaxValidator).Assembly.GetType("Px.Utils.Validation.ValidationFeedbackSink")!; + return Activator.CreateInstance(sinkType, [null])!; + } + + private static ValidationFeedback GetFeedback(object sink) + { + MethodInfo toFeedbackMethod = sink.GetType().GetMethod("ToFeedback")!; + return (ValidationFeedback)toFeedbackMethod.Invoke(sink, null)!; + } + [TestMethod] public void ValidatePxFileSyntaxCalledWithMininalUtf8ReturnsValidResult() { @@ -97,6 +136,29 @@ public void ValidateWithLimitRepeatedCustomSyntaxFeedbackRetainsOnlyConfiguredCo Assert.Contains("Feedback limit of 2 instances", result.FeedbackItems[key][^1].AdditionalInfo); } + [TestMethod] + public async Task ValidateDefaultOverloadsRepeatedCustomSyntaxFeedbackRetainDefaultFeedbackCount() + { + List entryFunctions = [static (entry, _) => new( + new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.MultipleEntriesOnOneLine), + new(entry.File, entry.KeyStartLineIndex))]; + CustomSyntaxValidationFunctions functions = new(entryFunctions, [], []); + byte[] data = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Range(1, 101).Select(index => $"A{index}=1;")) + "DATA=1;"); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + SyntaxValidator synchronousValidator = new(customValidationFunctions: functions); + SyntaxValidator asynchronousValidator = new(customValidationFunctions: functions); + + SyntaxValidationResult synchronousResult = synchronousValidator.Validate(synchronousStream, "syntax.px", Encoding.UTF8); + SyntaxValidationResult asynchronousResult = await asynchronousValidator.ValidateAsync(asynchronousStream, "syntax.px", Encoding.UTF8); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.MultipleEntriesOnOneLine); + Assert.HasCount(100, synchronousResult.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", synchronousResult.FeedbackItems[key][^1].AdditionalInfo!); + Assert.HasCount(100, asynchronousResult.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", asynchronousResult.FeedbackItems[key][^1].AdditionalInfo!); + } + [TestMethod] public void ValidateObjectsCalledWithMultipleEntriesInSingleLineReturnsWithWarnings() { @@ -105,7 +167,7 @@ public void ValidateObjectsCalledWithMultipleEntriesInSingleLineReturnsWithWarni List functions = [SyntaxValidationFunctions.MultipleEntriesOnLine]; // Act - feedback = entryValidationMethod?.Invoke(null, [entries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportEntryFeedback(entries, functions); Assert.HasCount(1, feedback); Assert.HasCount(2, feedback.First().Value); @@ -175,7 +237,7 @@ public void ValidateObjectsCalledWithKvpWithMultipleLangParamsReturnsWithError() List functions = [SyntaxValidationFunctions.MoreThanOneLanguageParameter]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.MoreThanOneLanguageParameterSection, feedback.First().Key.Rule); @@ -190,7 +252,7 @@ public void ValidateObjectsCalledWithKvpWithMultipleSpecifierParamsSReturnsWithE List functions = [SyntaxValidationFunctions.MoreThanOneSpecifierParameter]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.MoreThanOneSpecifierParameterSection, feedback.First().Key.Rule); @@ -204,7 +266,7 @@ public void ValidateObjectsCalledWithKvpWithWrongOrderAndMissingKeywordReturnsWi List functions = [SyntaxValidationFunctions.WrongKeyOrderOrMissingKeyword]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(2, feedback); Assert.IsTrue(feedback.ContainsKey(new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.KeyHasWrongOrder))); @@ -226,7 +288,7 @@ public void ValidateObjectsCalledWithKvpWithInvalidSpecifiersReturnsWithErrors() ValidationFeedbackKey notEnclosedFeedbackKey = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.SpecifierPartNotEnclosed); // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(3, feedback); Assert.IsTrue(feedback.ContainsKey(missingDelimeterFeedbackKey)); @@ -243,7 +305,7 @@ public void ValidateObjectsCalledWithKvpWithIllegalSymbolsInLanguageParamReturns List functions = [SyntaxValidationFunctions.IllegalSymbolsInLanguageParamSection]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.HasCount(3, feedback.First().Value); @@ -258,7 +320,7 @@ public void ValidateObjectsCalledWithKvpWithIllegalSymbolsInSpecifierParamReturn List functions = [SyntaxValidationFunctions.IllegalCharactersInSpecifierSection]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.HasCount(3, feedback.First().Value); @@ -273,7 +335,7 @@ public void ValidateObjectsCalledWithKvpWithBadValuesReturnsErrors() List functions = [SyntaxValidationFunctions.InvalidValueFormat]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.HasCount(4, feedback.First().Value); @@ -288,7 +350,7 @@ public void ValidateObjectsCalledWithKvpWithExcessListValueWhitespaceReturnsWith List functions = [SyntaxValidationFunctions.ExcessWhitespaceInValue]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.ExcessWhitespaceInValue, feedback.First().Key.Rule); @@ -302,7 +364,7 @@ public void ValidateObjectsCalledWithKvpWithExcessKeyWhitespaceReturnsWithWarnin List functions = [SyntaxValidationFunctions.KeyContainsExcessWhiteSpace]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.KeyContainsExcessWhiteSpace, feedback.First().Key.Rule); @@ -316,7 +378,7 @@ public void ValidateObjectsCalledWithKvpWithShortMultilineValueReturnsWithWarnin List functions = [SyntaxValidationFunctions.ExcessNewLinesInValue]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.HasCount(2, feedback.First().Value); @@ -333,7 +395,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithInvalidKeywordsReturns ValidationFeedbackKey illegalCharactersFeedbackKey = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.IllegalCharactersInKeyword); // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(2, feedback); Assert.IsTrue(feedback.ContainsKey(startWithletterFeedbackKey)); @@ -349,7 +411,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithValidLanguagesReturnsW List functions = [SyntaxValidationFunctions.IllegalCharactersInLanguageParameter]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(0, feedback); } @@ -362,7 +424,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithInvalidLanguagesReturn List functions = [SyntaxValidationFunctions.IllegalCharactersInLanguageParameter]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.IllegalCharactersInLanguageSection, feedback.First().Key.Rule); @@ -376,7 +438,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithIllegalCharactersInSpe List functions = [SyntaxValidationFunctions.IllegalCharactersInSpecifierParts]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.IllegalCharactersInSpecifierPart, feedback.First().Key.Rule); @@ -390,7 +452,7 @@ public void ValidateObjectsCalledWithEntryWithoutValueReturnsWithError() List functions = [SyntaxValidationFunctions.EntryWithoutValue]; // Act - feedback = entryValidationMethod?.Invoke(null, [entries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportEntryFeedback(entries, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.EntryWithoutValue, feedback.First().Key.Rule); @@ -404,7 +466,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithIncompliantLanguagesRe List functions = [SyntaxValidationFunctions.IncompliantLanguage]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf] ) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(1, feedback); Assert.HasCount(2, feedback.First().Value); @@ -419,7 +481,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithUnrecommendedKeywordNa List functions = [SyntaxValidationFunctions.KeywordContainsUnderscore, SyntaxValidationFunctions.KeywordIsNotInUpperCase]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf] ) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(2, feedback); Assert.IsTrue(feedback.ContainsKey(new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.KeywordIsNotInUpperCase))); @@ -434,7 +496,7 @@ public void ValidateObjectsCalledWithLongKeywordReturnsWithWarnings() List functions = [SyntaxValidationFunctions.KeywordIsExcessivelyLong]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.KeywordExcessivelyLong, feedback.First().Key.Rule); @@ -453,7 +515,7 @@ public void GetValueTypeFromStringCalledWithValidListsValuesCorrectValueType(str List functions = [SyntaxValidationFunctions.InvalidValueFormat]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Utils.Validation.ValueType? valueType = getValueTypeFromStringMethod?.Invoke(null, [keyValuePairs[0].KeyValuePair.Value, PxFileConfiguration.Default]) as Utils.Validation.ValueType?; // Assert @@ -484,7 +546,7 @@ public void CorrectlyDefinedRangeAndSeriesTimeValuesReturnCorrectValueType(strin List functions = [SyntaxValidationFunctions.InvalidValueFormat]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Utils.Validation.ValueType? valueType = getValueTypeFromStringMethod?.Invoke(null, [keyValuePairs[0].KeyValuePair.Value, PxFileConfiguration.Default]) as Utils.Validation.ValueType?; // Assert @@ -514,7 +576,7 @@ public void IncorrectlyDefinedRangeAndSeriesTimeValuesReturnWithErrors(string ti List functions = [SyntaxValidationFunctions.InvalidValueFormat]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Utils.Validation.ValueType? valueType = getValueTypeFromStringMethod?.Invoke(null, [keyValuePairs.First().KeyValuePair.Value, PxFileConfiguration.Default]) as Utils.Validation.ValueType?; // Assert diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.cs index 5ab3f849..8d6c18c2 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.cs @@ -56,41 +56,9 @@ public sealed partial class ContentValidator( /// object that contains the feedback gathered during the validation process. public ContentValidationResult Validate() { - IEnumerable contentValidationEntryFunctions = DefaultContentValidationEntryFunctions; - IEnumerable contentValidationFindKeywordFunctions = DefaultContentValidationFindKeywordFunctions; - - if (customContentValidationFunctions is not null) - { - contentValidationEntryFunctions = contentValidationEntryFunctions.Concat(customContentValidationFunctions.CustomContentValidationEntryFunctions); - contentValidationFindKeywordFunctions = contentValidationFindKeywordFunctions.Concat(customContentValidationFunctions.CustomContentValidationFindKeywordFunctions); - } - - ValidationFeedback feedbackItems = []; - - foreach (ContentValidationFindKeywordValidator findingFunction in contentValidationFindKeywordFunctions) - { - ValidationFeedback? feedback = findingFunction(entries, this); - if (feedback is not null) - { - feedbackItems.AddRange(feedback); - } - } - foreach (ContentValidationEntryValidator entryFunction in contentValidationEntryFunctions) - { - foreach (ValidationStructuredEntry entry in entries) - { - ValidationFeedback? feedback = entryFunction(entry, this); - if (feedback is not null) - { - feedbackItems.AddRange(feedback); - } - } - } - int lengthOfDataRows = _headingDimensionNames is not null ? GetProductOfDimensionValues(_headingDimensionNames) : 0; - int amountOfDataRows = _stubDimensionNames is not null ? GetProductOfDimensionValues(_stubDimensionNames) : 0; - ResetFields(); - - return new ContentValidationResult(feedbackItems, lengthOfDataRows, amountOfDataRows); + ValidationFeedbackSink sink = new(); + ContentValidationOutput output = ValidateIntoSink(sink); + return new ContentValidationResult(sink.ToFeedback(), output.DataRowLength, output.DataRowAmount); } /// diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index 8120df85..25e1c78d 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -51,36 +51,9 @@ public ValidationResult Validate( Encoding? encoding = null, IFileSystem? fileSystem = null) { - fileSystem ??= new LocalFileSystem(); - if (encoding is null) - { - long originalPosition = stream.Position; - encoding = fileSystem.GetEncoding(stream); - stream.Position = originalPosition; - } - - SetValidationParameters(encoding, filename); - - ValidationFeedback validationFeedbacks = []; - long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); - if (dataStartIndex == -1) - { - KeyValuePair feedback = - new(new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.StartOfDataSectionNotFound), - new(filename, 0, 0)); - validationFeedbacks.Add(feedback); - - return new(validationFeedbacks); - } - - stream.Position = dataStartIndex; - ValidationFeedback dataStreamFeedbacks = ValidateDataStream(stream); - validationFeedbacks.AddRange(dataStreamFeedbacks); - - ResetValidator(); - - return new(validationFeedbacks); + ValidationFeedbackSink sink = new(); + ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new ValidationResult(sink.ToFeedback()); } /// @@ -175,35 +148,9 @@ public async Task ValidateAsync( IFileSystem? fileSystem = null, CancellationToken cancellationToken = default) { - fileSystem ??= new LocalFileSystem(); - if (encoding is null) - { - long originalPosition = stream.Position; - encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); - stream.Position = originalPosition; - } - SetValidationParameters(encoding, filename); - - ValidationFeedback validationFeedbacks = []; - long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); - if (dataStartIndex == -1) - { - KeyValuePair feedback = - new(new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.StartOfDataSectionNotFound), - new(filename, 0, 0)); - validationFeedbacks.Add(feedback); - - return new(validationFeedbacks); - } - stream.Position = dataStartIndex; - ValidationFeedback dataStreamFeedbacks = await Task.Factory.StartNew(() => - ValidateDataStream(stream, cancellationToken), cancellationToken); - validationFeedbacks.AddRange(dataStreamFeedbacks); - - ResetValidator(); - - return new(validationFeedbacks); + ValidationFeedbackSink sink = new(); + await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new ValidationResult(sink.ToFeedback()); } internal async Task ValidateIntoSinkAsync( diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs index 4de1fec5..91a6a584 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs @@ -36,39 +36,9 @@ public SyntaxValidationResult Validate( Encoding? encoding = null, IFileSystem? fileSystem = null) { - fileSystem ??= new LocalFileSystem(); - if (encoding is null) - { - long originalPosition = stream.Position; - encoding = fileSystem.GetEncoding(stream); - stream.Position = originalPosition; - } - - SyntaxValidationFunctions validationFunctions = new(); - IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; - IEnumerable keyValueValidationFunctions = validationFunctions.DefaultKeyValueValidationFunctions; - IEnumerable structuredValidationFunctions = validationFunctions.DefaultStructuredValidationFunctions; - - if (customValidationFunctions is not null) - { - stringValidationFunctions = stringValidationFunctions.Concat(customValidationFunctions.CustomStringValidationFunctions); - keyValueValidationFunctions = keyValueValidationFunctions.Concat(customValidationFunctions.CustomKeyValueValidationFunctions); - structuredValidationFunctions = structuredValidationFunctions.Concat(customValidationFunctions.CustomStructuredValidationFunctions); - } - - conf ??= PxFileConfiguration.Default; - ResetDataSectionPosition(); - _dataSectionStartStreamPosition = StreamUtilities.FindDataStartPosition(stream, conf, _bufferSize); - - ValidationFeedback validationFeedbacks = []; - List stringEntries = BuildValidationEntries(stream, encoding, conf, filename, _bufferSize); - validationFeedbacks.AddRange(ValidateEntries(stringEntries, stringValidationFunctions, conf)); - List keyValuePairs = BuildKeyValuePairs(stringEntries, conf); - validationFeedbacks.AddRange(ValidateKeyValuePairs(keyValuePairs, keyValueValidationFunctions, conf)); - List structuredEntries = BuildValidationStructureEntries(keyValuePairs, conf); - validationFeedbacks.AddRange(ValidateStructs(structuredEntries, structuredValidationFunctions, conf)); - - return new SyntaxValidationResult(validationFeedbacks, structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); + ValidationFeedbackSink sink = new(); + SyntaxValidationOutput output = ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new SyntaxValidationResult(sink.ToFeedback(), output.StructuredEntries, output.DataStartRow, output.DataStartStreamPosition); } /// @@ -150,38 +120,9 @@ public async Task ValidateAsync( IFileSystem? fileSystem = null, CancellationToken cancellationToken = default) { - fileSystem ??= new LocalFileSystem(); - if (encoding is null) - { - long originalPosition = stream.Position; - encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); - stream.Position = originalPosition; - } - - SyntaxValidationFunctions validationFunctions = new(); - IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; - IEnumerable keyValueValidationFunctions = validationFunctions.DefaultKeyValueValidationFunctions; - IEnumerable structuredValidationFunctions = validationFunctions.DefaultStructuredValidationFunctions; - - if (customValidationFunctions is not null) - { - stringValidationFunctions = stringValidationFunctions.Concat(customValidationFunctions.CustomStringValidationFunctions); - keyValueValidationFunctions = keyValueValidationFunctions.Concat(customValidationFunctions.CustomKeyValueValidationFunctions); - structuredValidationFunctions = structuredValidationFunctions.Concat(customValidationFunctions.CustomStructuredValidationFunctions); - } - - conf ??= PxFileConfiguration.Default; - ResetDataSectionPosition(); - _dataSectionStartStreamPosition = await StreamUtilities.FindDataStartPositionAsync(stream, conf, _bufferSize, cancellationToken); - ValidationFeedback validationFeedbacks = []; - List entries = await BuildValidationEntriesAsync(stream, encoding, conf, filename, _bufferSize, cancellationToken); - validationFeedbacks.AddRange(ValidateEntries(entries, stringValidationFunctions, conf)); - List keyValuePairs = BuildKeyValuePairs(entries, conf); - validationFeedbacks.AddRange(ValidateKeyValuePairs(keyValuePairs, keyValueValidationFunctions, conf)); - List structuredEntries = BuildValidationStructureEntries(keyValuePairs, conf); - validationFeedbacks.AddRange(ValidateStructs(structuredEntries, structuredValidationFunctions, conf)); - - return new SyntaxValidationResult(validationFeedbacks, structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); + ValidationFeedbackSink sink = new(); + SyntaxValidationOutput output = await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new SyntaxValidationResult(sink.ToFeedback(), output.StructuredEntries, output.DataStartRow, output.DataStartStreamPosition); } /// @@ -283,23 +224,6 @@ public static bool IsEndOfMetadataSection(char currentCharacter, PxFileConfigura return false; } - private static ValidationFeedback ValidateEntries(IEnumerable entries, IEnumerable validationFunctions, PxFileConfiguration syntaxConf) - { - ValidationFeedback validationFeedback = []; - foreach (ValidationEntry entry in entries) - { - foreach (EntryValidationFunction function in validationFunctions) - { - KeyValuePair? feedback = function(entry, syntaxConf); - if (feedback is not null) - { - validationFeedback.Add((KeyValuePair )feedback); - } - } - } - return validationFeedback; - } - private static void ReportEntryFeedback( IEnumerable entries, IEnumerable validationFunctions, @@ -319,26 +243,6 @@ private static void ReportEntryFeedback( } } - private static ValidationFeedback ValidateKeyValuePairs( - IEnumerable kvpObjects, - IEnumerable validationFunctions, - PxFileConfiguration syntaxConf) - { - ValidationFeedback validationFeedback = []; - foreach (ValidationKeyValuePair kvpObject in kvpObjects) - { - foreach (KeyValuePairValidationFunction function in validationFunctions) - { - KeyValuePair? feedback = function(kvpObject, syntaxConf); - if (feedback is not null) - { - validationFeedback.Add((KeyValuePair)feedback); - } - } - } - return validationFeedback; - } - private static void ReportKeyValuePairFeedback( IEnumerable kvpObjects, IEnumerable validationFunctions, @@ -358,26 +262,6 @@ private static void ReportKeyValuePairFeedback( } } - private static ValidationFeedback ValidateStructs( - IEnumerable structuredEntries, - IEnumerable validationFunctions, - PxFileConfiguration syntaxConf) - { - ValidationFeedback validationFeedback = []; - foreach (ValidationStructuredEntry structuredEntry in structuredEntries) - { - foreach (StructuredValidationFunction function in validationFunctions) - { - KeyValuePair? feedback = function(structuredEntry, syntaxConf); - if (feedback is not null) - { - validationFeedback.Add((KeyValuePair)feedback); - } - } - } - return validationFeedback; - } - private static void ReportStructuredFeedback( IEnumerable structuredEntries, IEnumerable validationFunctions,