diff --git a/Px.Utils.TestingApp/Commands/Benchmark.cs b/Px.Utils.TestingApp/Commands/Benchmark.cs index 4c7b2b0..5c2c14a 100644 --- a/Px.Utils.TestingApp/Commands/Benchmark.cs +++ b/Px.Utils.TestingApp/Commands/Benchmark.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Reflection; +using Px.Utils.Validation; namespace Px.Utils.TestingApp.Commands { @@ -77,12 +78,18 @@ internal BenchmarkResult(string name, IReadOnlyList iterationTimesMs) /// protected Func[] BenchmarkFunctionsAsync { get; set; } = []; + /// + /// Feedback retention options used by validation benchmark commands. + /// + protected ValidationOptions ValidationOptions { get; private set; } = new(); + private static readonly string[] iterFlags = ["-i", "-iter"]; + private static readonly string[] feedbackLimitFlags = ["-l", "-limit"]; /// /// List of flags that can be used to provide parameters to the benchmark command. /// - protected List ParameterFlags { get; } = [iterFlags]; + protected List ParameterFlags { get; } = [iterFlags, feedbackLimitFlags]; internal List Results { get; } = []; private int processesCompleted; @@ -137,7 +144,8 @@ internal override async Task Run(bool batchMode, List? inputs = null) protected virtual void SetRunParameters() { - if (Parameters.Keys.Count == ParameterFlags.Count) + ValidationOptions = new(); + if (Parameters.Keys.All(key => Array.Exists(ParameterFlags.ToArray(), flag => flag.Contains(key)))) { foreach (string key in Parameters.Keys) { @@ -146,9 +154,9 @@ protected virtual void SetRunParameters() { Iterations = iterations; } - else if(!Array.Exists(ParameterFlags.ToArray(), flag => flag.Contains(key))) + else if (feedbackLimitFlags.Contains(key) && Parameters[key].Count == 1) { - throw new ArgumentException($"Invalid argument {key} {string.Join(' ', Parameters[key])}"); + ValidationOptions = ParseValidationOptions(Parameters[key][0]); } } } @@ -176,6 +184,16 @@ protected virtual void StartInteractiveMode() Iterations = value; } + private static ValidationOptions ParseValidationOptions(string value) + { + if (!int.TryParse(value, out int limit) || limit <= 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Feedback limit must be a positive integer."); + } + + return new ValidationOptions { MaxFeedbackItemsPerSignature = limit }; + } + /// /// Setup method for the benchmark. Called before running the benchmarks. Marked as virtual to allow for custom setup in derived classes. /// diff --git a/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs index 47c02ad..eaa96a0 100644 --- a/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/DataValidationBenchmark.cs @@ -17,17 +17,14 @@ internal sealed class DataValidationBenchmark : FileBenchmark internal override string Help => "Validates the px file data." + Environment.NewLine + "\t-r, -rows: How many data rows the validator should expect." + Environment.NewLine + - "\t-c, -cols: How many data colums the validator should expect."; + "\t-c, -cols: How many data colums the validator should expect." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Benchmarks the data validation capabilities of the DataValidator."; private long start; - private const string dataKeyword = "DATA"; - private Encoding encoding; - private const int readStartOffset = 3; - internal DataValidationBenchmark() { BenchmarkFunctions = [ValidateDataBenchmarks]; @@ -44,28 +41,28 @@ protected override async Task OneTimeBenchmarkSetupAsync() using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); PxFileMetadataReader reader = new(); encoding = reader.GetEncoding(stream); - start = StreamUtilities.FindKeywordPosition(stream, dataKeyword, PxFileConfiguration.Default); + start = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default); if (start == -1) { - throw new ArgumentException($"Could not find data keyword '{dataKeyword}'"); + throw new ArgumentException("Could not find the first data value after 'DATA='"); } } private void ValidateDataBenchmarks() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); - stream.Position = start + dataKeyword.Length + readStartOffset; // skip the '=' and linechange + stream.Position = start; DataValidator validator = new(expectedCols, expectedRows, 0); - validator.Validate(stream, TestFilePath, encoding); + validator.Validate(stream, TestFilePath, encoding, null, ValidationOptions); } private async Task ValidateDataBenchmarksAsync() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); - stream.Position = start + dataKeyword.Length + readStartOffset; // skip the '=' and linechange + stream.Position = start; DataValidator validator = new(expectedCols, expectedRows, 0); - await validator.ValidateAsync(stream, TestFilePath, encoding); + await validator.ValidateAsync(stream, TestFilePath, encoding, null, ValidationOptions); } protected override void SetRunParameters() diff --git a/Px.Utils.TestingApp/Commands/DatabaseValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/DatabaseValidationBenchmark.cs index b59d324..2efccab 100644 --- a/Px.Utils.TestingApp/Commands/DatabaseValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/DatabaseValidationBenchmark.cs @@ -4,7 +4,9 @@ namespace Px.Utils.TestingApp.Commands { internal sealed class DatabaseValidationBenchmark : Benchmark { - internal override string Help => "Validates a px path database."; + internal override string Help => + "Validates a px path database." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Validates a px path database."; private static readonly string[] directoryFlags = ["-d", "-directory"]; @@ -59,13 +61,13 @@ protected override async Task OneTimeBenchmarkSetupAsync() private void ValidationBenchmark() { if(validator is null) throw new InvalidOperationException("Validator not initialized."); - validator.Validate(); + validator.Validate(ValidationOptions); } private async Task ValidationBenchmarkAsync() { if(validator is null) throw new InvalidOperationException("Validator not initialized."); - await validator.ValidateAsync(); + await validator.ValidateAsync(ValidationOptions); } } } diff --git a/Px.Utils.TestingApp/Commands/MetadataContentValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/MetadataContentValidationBenchmark.cs index 87fa9ce..a2dd89f 100644 --- a/Px.Utils.TestingApp/Commands/MetadataContentValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/MetadataContentValidationBenchmark.cs @@ -9,7 +9,8 @@ internal sealed class MetadataContentValidationBenchmark : FileBenchmark internal override string Help => "Validates the contents of the Px file metadata given amount of times." + Environment.NewLine + "\t-f, -file: The path to the px file to read." + Environment.NewLine + - "\t-i, -iter: The number of iterations to run."; + "\t-i, -iter: The number of iterations to run." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Benchmarks the metadata content validation of Px.Utils/Validation/SyntaxValidator."; @@ -34,7 +35,7 @@ protected override async Task OneTimeBenchmarkSetupAsync() private void ValidateContentBenchmark() { ContentValidator validator = new(TestFilePath, Encoding.Default, [.. _entries]); - validator.Validate(); + validator.Validate(ValidationOptions); } } } diff --git a/Px.Utils.TestingApp/Commands/MetadataSyntaxValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/MetadataSyntaxValidationBenchmark.cs index e85df6c..cd3ccf5 100644 --- a/Px.Utils.TestingApp/Commands/MetadataSyntaxValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/MetadataSyntaxValidationBenchmark.cs @@ -9,7 +9,8 @@ internal sealed class MetadataSyntaxValidationBenchmark : FileBenchmark internal override string Help => "Validates the syntax of the Px file metadata given amount of times." + Environment.NewLine + "\t-f, -file: The path to the px file to read." + Environment.NewLine + - "\t-i, -iter: The number of iterations to run."; + "\t-i, -iter: The number of iterations to run." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Benchmarks the metadata syntax validation of Px.Utils/Validation/SyntaxValidator."; @@ -34,7 +35,7 @@ private void SyntaxValidationBenchmark() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); SyntaxValidator validator = new(); - validator.Validate(stream, TestFilePath, encoding); + validator.Validate(stream, TestFilePath, encoding, null, ValidationOptions); stream.Close(); } @@ -42,7 +43,7 @@ private async Task SyntaxValidationBenchmarkAsync() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); SyntaxValidator validator = new(); - await validator.ValidateAsync(stream, TestFilePath, encoding); + await validator.ValidateAsync(stream, TestFilePath, encoding, null, ValidationOptions); stream.Close(); } } diff --git a/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs b/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs index f8a175a..fbdfca8 100644 --- a/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs +++ b/Px.Utils.TestingApp/Commands/PxFileValidationBenchmark.cs @@ -8,7 +8,8 @@ namespace Px.Utils.TestingApp.Commands internal sealed class PxFileValidationBenchmark : FileBenchmark { internal override string Help => - "Runs through the whole px file validation process (metadata syntax- and contents-, data-) for the given file."; + "Runs through the whole px file validation process (metadata syntax- and contents-, data-) for the given file." + Environment.NewLine + + "\t-l, -limit: Feedback items retained per file, level, and rule; use a positive number. Defaults to 100."; internal override string Description => "Benchmarks the px file validation capabilities of the PxFileValidator."; @@ -42,14 +43,14 @@ private void ValidatePxFileBenchmarks() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); PxFileValidator validator = new(); - validator.Validate(stream, TestFilePath, encoding); + validator.Validate(stream, TestFilePath, encoding, null, ValidationOptions); } private async Task ValidatePxFileBenchmarksAsync() { using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read); PxFileValidator validator = new(); - await validator.ValidateAsync(stream, TestFilePath, encoding); + await validator.ValidateAsync(stream, TestFilePath, encoding, null, ValidationOptions); } } } diff --git a/Px.Utils.UnitTests/PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs b/Px.Utils.UnitTests/PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs index cafee34..9f6d389 100644 --- a/Px.Utils.UnitTests/PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs +++ b/Px.Utils.UnitTests/PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs @@ -1,4 +1,5 @@ using Px.Utils.Models.Metadata; +using Px.Utils.PxFile; using Px.Utils.PxFile.Data; using PxFileTests.Fixtures; using Px.Utils.Models.Data; @@ -12,6 +13,46 @@ public class DataReaderTests { private readonly double[] missingMarkers = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]; + [TestMethod] + public void ReadDoubleDataValuesQuotedDataKeywordAndWhitespaceWithSmallBufferReturnsCorrectValue() + { + // Arrange + byte[] data = Encoding.UTF8.GetBytes("TITLE=\"DATA=not-an-entry\";\nDATA=\r\n\t1;"); + using Stream stream = new MemoryStream(data); + using PxFileStreamDataReader reader = new(stream, null, 2); + DoubleDataValue[] targetBuffer = new DoubleDataValue[1]; + MatrixMetadata metadata = TestModelBuilder.BuildTestMetadata([1]); + + // Act + reader.ReadDoubleDataValues(targetBuffer, 0, metadata, metadata); + + // Assert + Assert.AreEqual(1.0, targetBuffer[0].UnsafeValue); + } + + [TestMethod] + public void ReadDoubleDataValuesExplicitDataStartMatchesAutomaticPositioning() + { + // Arrange + byte[] data = Encoding.UTF8.GetBytes("TITLE=\"DATA=not-an-entry\";\nDATA= 1;"); + using Stream positionStream = new MemoryStream(data); + long dataStart = StreamUtilities.FindDataStartPosition(positionStream, PxFileConfiguration.Default); + using Stream automaticStream = new MemoryStream(data); + using Stream explicitPositionStream = new MemoryStream(data); + using PxFileStreamDataReader automaticReader = new(automaticStream, null, 2); + using PxFileStreamDataReader explicitPositionReader = new(explicitPositionStream, dataStart, null, 2); + DoubleDataValue[] automaticBuffer = new DoubleDataValue[1]; + DoubleDataValue[] explicitPositionBuffer = new DoubleDataValue[1]; + MatrixMetadata metadata = TestModelBuilder.BuildTestMetadata([1]); + + // Act + automaticReader.ReadDoubleDataValues(automaticBuffer, 0, metadata, metadata); + explicitPositionReader.ReadDoubleDataValues(explicitPositionBuffer, 0, metadata, metadata); + + // Assert + CollectionAssert.AreEqual(automaticBuffer, explicitPositionBuffer); + } + [TestMethod] public void ReadDoubleDataValuesValidIntegersReturnsCorrectDoubleDataValues() { diff --git a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs index a8715c1..d311c9f 100644 --- a/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs +++ b/Px.Utils.UnitTests/PxFileTests/DataTests/StreamUtilitiesTests.cs @@ -1,4 +1,4 @@ -using PxFileTests.Fixtures; +using PxFileTests.Fixtures; using Px.Utils.PxFile; using Px.Utils.PxFile.Data; using System.Text; @@ -13,141 +13,254 @@ public class StreamUtilitiesTests */ [TestMethod] - public void FindKeywordTestDataKeywordAtStartOfStreamReturnsZero() + public void FindKeywordPositionKeywordSplitAcrossBuffersReturnsKeywordOffset() { // Arrange - byte[] data = Encoding.UTF8.GetBytes("DATA="); + string content = "TITLE=\"DATA=79\";\r\n VALUES=\"x\";\r\nDATA=1;"; + byte[] data = Encoding.UTF8.GetBytes(content); using Stream stream = new MemoryStream(data); + long expectedPosition = Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("DATA=", StringComparison.Ordinal)]); // Act - long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default); + long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default, 3); // Assert - Assert.AreEqual(0, position); + Assert.AreEqual(expectedPosition, position); + Assert.IsTrue(stream.Position > position); } [TestMethod] - public void FindKeywordTestTwoDataKeywordsReturnsNegative1() + public async Task FindKeywordPositionAsyncKeywordSplitAcrossBuffersReturnsKeywordOffset() { // Arrange - byte[] data = Encoding.UTF8.GetBytes("DATADATA="); + string content = "TITLE=\"DATA=79\";\r\n VALUES=\"x\";\r\nDATA=1;"; + byte[] data = Encoding.UTF8.GetBytes(content); using Stream stream = new MemoryStream(data); + long expectedPosition = Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("DATA=", StringComparison.Ordinal)]); // Act - long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default); + long position = await StreamUtilities.FindKeywordPositionAsync(stream, "DATA", PxFileConfiguration.Default, CancellationToken.None, 3); // Assert - Assert.AreEqual(-1, position); + Assert.AreEqual(expectedPosition, position); + Assert.IsTrue(stream.Position > position); } [TestMethod] - public void FindKeywordTestDataKeywordInTheMiddleOfStreamReturnsIndex() + public void FindKeywordPositionAtStartOfMetadataReturnsFirstValueOffset() { // Arrange - byte[] data = Encoding.UTF8.GetBytes("KEYWORD=\"foo\";\nDATA=123 345"); + string content = "METADATA=\"FOO\";\r\nDATA=1;"; + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + byte[] data = contentBytes; using Stream stream = new MemoryStream(data); + long expectedPosition = Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("METADATA=", StringComparison.Ordinal)]); + + // Act + long position = StreamUtilities.FindKeywordPosition(stream, "METADATA", PxFileConfiguration.Default, 2); + + // Assert + Assert.AreEqual(expectedPosition, position); + } + + [TestMethod] + public async Task FindKeywordPositionAtStreamOriginWithBomReturnsRawKeywordOffset() + { + // Arrange + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] data = [.. bom, .. Encoding.UTF8.GetBytes("DATA=1;")]; + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); // Act - long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default); + long synchronousPosition = StreamUtilities.FindKeywordPosition(synchronousStream, "DATA", PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindKeywordPositionAsync(asynchronousStream, "DATA", PxFileConfiguration.Default, TestContext.CancellationToken, 2); // Assert - Assert.AreEqual(15, position); + Assert.AreEqual(bom.Length, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.IsGreaterThan(synchronousPosition, synchronousStream.Position); + Assert.IsGreaterThan(asynchronousPosition, asynchronousStream.Position); } [TestMethod] - public void FindKeywordTestDataKeywordAtTheEndOfStreamReturnsIndex() + public void FindKeywordPositionAtStartOfMetadataWithBomReturnsFirstValueOffset() { // Arrange - byte[] data = Encoding.UTF8.GetBytes("DADADADATA="); + string content = "METADATA=\"FOO\";\r\nDATA=1;"; + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + byte[] data = [.. bom, .. contentBytes]; using Stream stream = new MemoryStream(data); + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("METADATA=", StringComparison.Ordinal)]); // Act - long position = StreamUtilities.FindKeywordPosition(stream, "DATA", PxFileConfiguration.Default); + long position = StreamUtilities.FindKeywordPosition(stream, "METADATA", PxFileConfiguration.Default, 2); - // Assert - Assert.AreEqual(-1, position); + // Assert + Assert.AreEqual(expectedPosition, position); } [TestMethod] - public void FindKeywordTestDKeywordInTheMiddleOfStreamReturnsIndex() + [DataRow("1")] + [DataRow("-1")] + [DataRow(".5")] + [DataRow("0")] + [DataRow("0.0")] + [DataRow("0.5")] + [DataRow("123.456")] + [DataRow("\".\"")] + [DataRow("\"..\"")] + public void FindDataStartPositionDataWithBomMultibyteMetadataAndWhitespaceReturnsFirstValueOffset(string firstValue) { // Arrange - byte[] data = Encoding.UTF8.GetBytes("FFFFFF;D=AAAA"); + string content = "TITLE=\"DATA=79_20180101;\";\r\nVALUES=\"Ää, Öö\";\r\nDATA=\t \r\n" + firstValue + " 2;"; + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + byte[] data = [.. bom, .. contentBytes]; using Stream stream = new MemoryStream(data); + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..(content.LastIndexOf("DATA=", StringComparison.Ordinal) + 5 + "\t \r\n".Length)]); // Act - long position = StreamUtilities.FindKeywordPosition(stream, "D", PxFileConfiguration.Default); + long position = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 3); // Assert - Assert.AreEqual(7, position); + Assert.AreEqual(expectedPosition, position); + Assert.AreEqual(0, stream.Position); + Assert.AreEqual(firstValue[0], (char)data[(int)position]); } [TestMethod] - public void FindKeywordTestDATAKeywordInTheMiddleOfUtfFixtureStreamReturnsIndex() + public async Task FindDataStartPositionAsyncDataSplitAcrossBuffersReturnsSameOffset() { - string keyword = "DATA"; - // Arrange - byte[] data = Encoding.UTF8.GetBytes(MinimalPx.MINIMAL_UTF8_N); + string content = "TITLE=\"Åland\";\nDATA=\r\n\t\".\" 2;"; + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + byte[] data = [.. bom, .. contentBytes]; using Stream stream = new MemoryStream(data); + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..content.IndexOf('"', content.IndexOf("DATA=", StringComparison.Ordinal) + 5)]); // Act - long position = StreamUtilities.FindKeywordPosition(stream, keyword, PxFileConfiguration.Default); - string result = Encoding.ASCII.GetString(data, (int)position, keyword.Length); + long synchronousPosition = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionAsync(stream, PxFileConfiguration.Default, 2, System.Threading.CancellationToken.None); // Assert - Assert.AreEqual(keyword, result); + Assert.AreEqual(expectedPosition, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.AreEqual(0, stream.Position); } [TestMethod] - public void FindKeywordTestDATAKeywordInTheMiddleOfAsciiFixtureStreamReturnsIndex() + public async Task FindDataStartPositionAtStreamOriginWithBomReturnsFirstValueOffset() { - string keyword = "DATA"; - // Arrange - byte[] data = Encoding.UTF8.GetBytes(MinimalPx.MINIMAL_ISO_8859_15_N); + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] data = [.. bom, .. Encoding.UTF8.GetBytes("DATA=\r\n\t1;")]; + long expectedPosition = bom.Length + "DATA=\r\n\t".Length; using Stream stream = new MemoryStream(data); // Act - long position = StreamUtilities.FindKeywordPosition(stream, keyword, PxFileConfiguration.Default); - string result = Encoding.ASCII.GetString(data, (int)position, keyword.Length); + long synchronousPosition = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionAsync(stream, PxFileConfiguration.Default, 2, TestContext.CancellationToken); // Assert - Assert.AreEqual(keyword, result); + Assert.AreEqual(expectedPosition, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.AreEqual(0, stream.Position); } [TestMethod] - public void FindKeywordTestDATAKeywordInTheMiddleOfAsciiFixtureStreamShortBufferSplitsKeywordReturnsIndex() + public async Task FindKeywordPositionQuotedKeywordWithBomAndMultibyteMetadataReturnsRawTopLevelOffset() { - string keyword = "DATA"; + // Arrange + string content = "TITLE=\"DATA=not-an-entry\";\nVALUES=\"Ää\";\nDATA=1;"; + byte[] data = [.. Encoding.UTF8.GetPreamble(), .. Encoding.UTF8.GetBytes(content)]; + long expectedPosition = Encoding.UTF8.GetPreamble().Length + Encoding.UTF8.GetByteCount(content[..content.LastIndexOf("DATA=", StringComparison.Ordinal)]); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + + // Act + long synchronousPosition = StreamUtilities.FindKeywordPosition(synchronousStream, "DATA", PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindKeywordPositionAsync(asynchronousStream, "DATA", PxFileConfiguration.Default, CancellationToken.None, 2); + // Assert + Assert.AreEqual(expectedPosition, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.IsGreaterThan(synchronousPosition, synchronousStream.Position); + Assert.IsGreaterThan(asynchronousPosition, asynchronousStream.Position); + } + + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task FindDataStartPositionUncheckedQuotedKeywordAndWhitespaceReturnsOffsetAndAdvancesStream(bool includesUtf8Bom) + { // Arrange - byte[] data = Encoding.UTF8.GetBytes(MinimalPx.MINIMAL_ISO_8859_15_N); - using Stream stream = new MemoryStream(data); + string content = "TITLE=\"DATA=not-an-entry\";\nVALUES=\"Ää\";\nDATA=\r\n\t1 2;"; + byte[] bom = includesUtf8Bom ? Encoding.UTF8.GetPreamble() : []; + byte[] data = [.. bom, .. Encoding.UTF8.GetBytes(content)]; + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..content.IndexOf('1')]); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); // Act - long position = StreamUtilities.FindKeywordPosition(stream, keyword, PxFileConfiguration.Default, 3); - string result = Encoding.ASCII.GetString(data, (int)position, keyword.Length); + long synchronousPosition = StreamUtilities.FindDataStartPositionUnchecked(synchronousStream, PxFileConfiguration.Default, 2); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionUncheckedAsync(asynchronousStream, PxFileConfiguration.Default, 2, TestContext.CancellationToken); // Assert - Assert.AreEqual(keyword, result); + Assert.AreEqual(expectedPosition, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.IsGreaterThan(synchronousPosition, synchronousStream.Position); + Assert.IsGreaterThan(asynchronousPosition, asynchronousStream.Position); } [TestMethod] - public void FindKeywordTestDATAKeywordInTheMiddleOfDataFinderFixtureStreamReturnsIndex() + public async Task FindDataStartPositionUncheckedMissingDataReturnsNegative1() { - string keyword = "DATA"; + // Arrange + byte[] data = Encoding.UTF8.GetBytes("TITLE=\"value\";"); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + + // Act + long synchronousPosition = StreamUtilities.FindDataStartPositionUnchecked(synchronousStream, PxFileConfiguration.Default, 1); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionUncheckedAsync(asynchronousStream, PxFileConfiguration.Default, 1, TestContext.CancellationToken); + // Assert + Assert.AreEqual(-1, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + } + + [TestMethod] + public void FindDataStartPositionDataWithoutValueReturnsNegative1() + { // Arrange - byte[] data = Encoding.UTF8.GetBytes(MinimalPx.MINIMAL_UTF8_N_FOR_DATA_FINDER); - using Stream stream = new MemoryStream(data); + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes("TITLE=\"foo\";\nDATA=\r\n\t ")); // Act - long position = StreamUtilities.FindKeywordPosition(stream, keyword, PxFileConfiguration.Default); - string result = Encoding.ASCII.GetString(data, (int)position -1, keyword.Length+2); + long position = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 2); // Assert - Assert.AreEqual('\n' + keyword + '=', result); + Assert.AreEqual(-1, position); } + + [TestMethod] + [DataRow("DATA=;")] + [DataRow("DATA= ;")] + [DataRow("DATA=\r\n\t;")] + public async Task FindDataStartPositionEmptyDataEntryReturnsNegative1(string content) + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + + long synchronousPosition = StreamUtilities.FindDataStartPosition(stream, PxFileConfiguration.Default, 1); + long asynchronousPosition = await StreamUtilities.FindDataStartPositionAsync(stream, PxFileConfiguration.Default, 1); + + Assert.AreEqual(-1, synchronousPosition); + Assert.AreEqual(synchronousPosition, asynchronousPosition); + Assert.AreEqual(0, stream.Position); + } + + public TestContext TestContext { get; set; } } } diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index 3fcc692..b003d3f 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -49,6 +49,44 @@ public void ValidatePxFileContentCalledWithMinimalStructuredEntryReturnsValidRes Assert.HasCount(0, feedback.FeedbackItems); } + [TestMethod] + public void ValidateWithLimitRepeatedCustomContentFeedbackRetainsOnlyConfiguredCount() + { + const int limit = 2; + ContentValidationEntryValidator entryValidator = static (entry, _) => new(new( + new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.ValueIsNotInUpperCase), + new(entry.File, entry.KeyStartLineIndex))); + CustomContentValidationFunctions functions = new([], [entryValidator]); + ValidationStructuredEntry[] entries = [ + new("content.px", new("ONE", null, null, null), "1", 1, [], 0, null), + new("content.px", new("TWO", null, null, null), "2", 2, [], 0, null), + new("content.px", new("THREE", null, null, null), "3", 3, [], 0, null)]; + ContentValidator validator = new("content.px", Encoding.UTF8, entries, functions); + + ContentValidationResult result = validator.Validate(new ValidationOptions { MaxFeedbackItemsPerSignature = limit }); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.ValueIsNotInUpperCase); + Assert.HasCount(limit, result.FeedbackItems[key]); + Assert.Contains("Feedback limit of 2 instances", result.FeedbackItems[key][^1].AdditionalInfo!); + } + + [TestMethod] + public void ValidateDefaultOverloadRepeatedCustomContentFeedbackRetainsDefaultFeedbackCount() + { + static ValidationFeedback? entryValidator(ValidationStructuredEntry entry, ContentValidator _) => new(new( + new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.ValueIsNotInUpperCase), + new(entry.File, entry.KeyStartLineIndex))); + CustomContentValidationFunctions functions = new([], [entryValidator]); + ValidationStructuredEntry[] entries = [.. Enumerable.Range(1, 101).Select(index => new ValidationStructuredEntry("content.px", new($"KEY{index}", null, null, null), "1", index, [], 0, null))]; + ContentValidator validator = new("content.px", Encoding.UTF8, entries, functions); + + ContentValidationResult result = validator.Validate(); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.ValueIsNotInUpperCase); + Assert.HasCount(100, result.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", result.FeedbackItems[key][^1].AdditionalInfo!); + } + [TestMethod] public void ValidateCalledWithSharedDimensionNameAcrossLanguagesCalculatesRowCountsFromDefaultLanguage() { diff --git a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs index 66846a9..0b2cff2 100644 --- a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs +++ b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs @@ -3,6 +3,7 @@ using Px.Utils.UnitTests.Validation.Fixtures; using Px.Utils.Validation; using Px.Utils.Validation.DataValidation; +using Px.Utils.PxFile; namespace Px.Utils.UnitTests.Validation.DataValidationTests { @@ -18,8 +19,8 @@ public class DataValidationTest [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS, 0, 0)] [DataRow(DataStreamContents.SIMPLE_INVALID_DATA, 7, 12)] - [DataRow(DataStreamContents.NO_DATA, 2, 6, false)] - [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 2, 6, false)] + [DataRow(DataStreamContents.NO_DATA, 1, 1, false)] + [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 1, 1, false)] [DataRow(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE, 0, 0)] [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE, 0, 0)] [DataRow(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE, 0, 0)] @@ -45,13 +46,66 @@ public void ValidateDataReturnsExpectedErrorCount( Assert.AreEqual(expectedTotalErrorCount, actualErrorCount); } + [TestMethod] + public async Task ValidateAndValidateAsyncCustomDataKeywordAtStreamOriginFindDataSection() + { + PxFileConfiguration configuration = PxFileConfiguration.Default; + configuration.Tokens.KeyWords.Data = "VALUES"; + byte[] data = Encoding.UTF8.GetBytes("TITLE=\"test\";VALUES=1;"); + DataValidator validator = new(1, 0, 0, configuration); + + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + ValidationResult synchronousResult = validator.Validate(synchronousStream, "custom.px", Encoding.UTF8); + ValidationResult asynchronousResult = await validator.ValidateAsync(asynchronousStream, "custom.px", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound); + Assert.IsFalse(synchronousResult.FeedbackItems.ContainsKey(key)); + Assert.IsFalse(asynchronousResult.FeedbackItems.ContainsKey(key)); + } + + [TestMethod] + public void ValidateWithLimitLargeInvalidDataRetainsOnlyConfiguredFeedbackCount() + { + const int limit = 3; + string invalidData = string.Concat(Enumerable.Repeat("! ", 200)) + ";"; + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes("DATA=" + invalidData)); + DataValidator validator = new(0, 0, 0); + + ValidationResult result = validator.Validate(stream, "invalid.px", Encoding.UTF8, null, new ValidationOptions { MaxFeedbackItemsPerSignature = limit }); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + Assert.HasCount(limit, result.FeedbackItems[key]); + Assert.Contains("Feedback limit of 3 instances", result.FeedbackItems[key][^1].AdditionalInfo!); + } + + [TestMethod] + public async Task ValidateDefaultOverloadsLargeInvalidDataRetainDefaultFeedbackCount() + { + string invalidData = string.Concat(Enumerable.Repeat("! ", 200)) + ";"; + byte[] data = Encoding.UTF8.GetBytes("DATA=" + invalidData); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + DataValidator synchronousValidator = new(0, 0, 0); + DataValidator asynchronousValidator = new(0, 0, 0); + + ValidationResult synchronousResult = synchronousValidator.Validate(synchronousStream, "invalid.px", Encoding.UTF8); + ValidationResult asynchronousResult = await asynchronousValidator.ValidateAsync(asynchronousStream, "invalid.px", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + Assert.HasCount(100, synchronousResult.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", synchronousResult.FeedbackItems[key][^1].AdditionalInfo!); + Assert.HasCount(100, asynchronousResult.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", asynchronousResult.FeedbackItems[key][^1].AdditionalInfo!); + } + [TestMethod] [DataRow(DataStreamContents.SIMPLE_VALID_DATA, 0, 0)] [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS, 0, 0)] [DataRow(DataStreamContents.SIMPLE_INVALID_DATA, 7, 12)] - [DataRow(DataStreamContents.NO_DATA, 2, 6, false)] - [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 2, 6, false)] + [DataRow(DataStreamContents.NO_DATA, 1, 1, false)] + [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 1, 1, false)] [DataRow(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE, 0, 0)] [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE, 0, 0)] [DataRow(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE, 0, 0)] diff --git a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs index 15f16d8..90df93e 100644 --- a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs @@ -23,15 +23,54 @@ public class StreamSyntaxValidationTests public void Initialize() { entryValidationMethod = typeof(SyntaxValidator) - .GetMethod("ValidateEntries", BindingFlags.NonPublic | BindingFlags.Static); + .GetMethod("ReportEntryFeedback", BindingFlags.NonPublic | BindingFlags.Static); kvpValidationMethod = typeof(SyntaxValidator) - .GetMethod("ValidateKeyValuePairs", BindingFlags.NonPublic | BindingFlags.Static); + .GetMethod("ReportKeyValuePairFeedback", BindingFlags.NonPublic | BindingFlags.Static); structuredValidationMethod = typeof(SyntaxValidator) - .GetMethod("ValidateStructs", BindingFlags.NonPublic | BindingFlags.Static); + .GetMethod("ReportStructuredFeedback", BindingFlags.NonPublic | BindingFlags.Static); getValueTypeFromStringMethod = typeof(SyntaxValidationUtilityMethods) .GetMethod("GetValueTypeFromString", BindingFlags.NonPublic | BindingFlags.Static); } + private ValidationFeedback ReportEntryFeedback( + IEnumerable entries, + IEnumerable functions) + { + object sink = CreateFeedbackSink(); + entryValidationMethod!.Invoke(null, [entries, functions, conf, sink]); + return GetFeedback(sink); + } + + private ValidationFeedback ReportKeyValuePairFeedback( + IEnumerable keyValuePairs, + IEnumerable functions) + { + object sink = CreateFeedbackSink(); + kvpValidationMethod!.Invoke(null, [keyValuePairs, functions, conf, sink]); + return GetFeedback(sink); + } + + private ValidationFeedback ReportStructuredFeedback( + IEnumerable structuredEntries, + IEnumerable functions) + { + object sink = CreateFeedbackSink(); + structuredValidationMethod!.Invoke(null, [structuredEntries, functions, conf, sink]); + return GetFeedback(sink); + } + + private static object CreateFeedbackSink() + { + Type sinkType = typeof(SyntaxValidator).Assembly.GetType("Px.Utils.Validation.ValidationFeedbackSink")!; + return Activator.CreateInstance(sinkType, [null])!; + } + + private static ValidationFeedback GetFeedback(object sink) + { + MethodInfo toFeedbackMethod = sink.GetType().GetMethod("ToFeedback")!; + return (ValidationFeedback)toFeedbackMethod.Invoke(sink, null)!; + } + [TestMethod] public void ValidatePxFileSyntaxCalledWithMininalUtf8ReturnsValidResult() { @@ -53,6 +92,73 @@ public void ValidatePxFileSyntaxCalledWithMininalUtf8ReturnsValidResult() Assert.HasCount(0, feedback); } + [TestMethod] + public async Task ValidateAndValidateAsyncDataWithBomAndMultibyteMetadataReturnSameRawByteOffset() + { + // Arrange + string content = "TITLE=\"Ää Öö\";\r\nDATA=\r\n\t2635 2;"; + byte[] bom = Encoding.UTF8.GetPreamble(); + byte[] data = [.. bom, .. Encoding.UTF8.GetBytes(content)]; + long expectedPosition = bom.Length + Encoding.UTF8.GetByteCount(content[..content.IndexOf("2635", StringComparison.Ordinal)]); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + SyntaxValidator synchronousValidator = new(); + SyntaxValidator asynchronousValidator = new(); + + // Act + SyntaxValidationResult synchronousResult = synchronousValidator.Validate(synchronousStream, "foo", Encoding.UTF8); + SyntaxValidationResult asynchronousResult = await asynchronousValidator.ValidateAsync(asynchronousStream, "foo", Encoding.UTF8, cancellationToken: System.Threading.CancellationToken.None); + + // Assert + Assert.AreEqual(expectedPosition, synchronousResult.DataStartStreamPosition); + Assert.AreEqual(synchronousResult.DataStartStreamPosition, asynchronousResult.DataStartStreamPosition); + Assert.AreEqual(1, synchronousResult.DataStartRow); + Assert.AreEqual(synchronousResult.DataStartRow, asynchronousResult.DataStartRow); + synchronousStream.Position = synchronousResult.DataStartStreamPosition; + Assert.AreEqual('2', (char)synchronousStream.ReadByte()); + } + + [TestMethod] + public void ValidateWithLimitRepeatedCustomSyntaxFeedbackRetainsOnlyConfiguredCount() + { + const int limit = 2; + List entryFunctions = [static (entry, _) => new( + new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.MultipleEntriesOnOneLine), + new(entry.File, entry.KeyStartLineIndex))]; + CustomSyntaxValidationFunctions functions = new(entryFunctions, [], []); + SyntaxValidator validator = new(customValidationFunctions: functions); + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes("A=1;B=2;C=3;D=4;DATA=1;")); + + SyntaxValidationResult result = validator.Validate(stream, "syntax.px", Encoding.UTF8, null, new ValidationOptions { MaxFeedbackItemsPerSignature = limit }); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.MultipleEntriesOnOneLine); + Assert.HasCount(limit, result.FeedbackItems[key]); + Assert.Contains("Feedback limit of 2 instances", result.FeedbackItems[key][^1].AdditionalInfo); + } + + [TestMethod] + public async Task ValidateDefaultOverloadsRepeatedCustomSyntaxFeedbackRetainDefaultFeedbackCount() + { + List entryFunctions = [static (entry, _) => new( + new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.MultipleEntriesOnOneLine), + new(entry.File, entry.KeyStartLineIndex))]; + CustomSyntaxValidationFunctions functions = new(entryFunctions, [], []); + byte[] data = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Range(1, 101).Select(index => $"A{index}=1;")) + "DATA=1;"); + using Stream synchronousStream = new MemoryStream(data); + using Stream asynchronousStream = new MemoryStream(data); + SyntaxValidator synchronousValidator = new(customValidationFunctions: functions); + SyntaxValidator asynchronousValidator = new(customValidationFunctions: functions); + + SyntaxValidationResult synchronousResult = synchronousValidator.Validate(synchronousStream, "syntax.px", Encoding.UTF8); + SyntaxValidationResult asynchronousResult = await asynchronousValidator.ValidateAsync(asynchronousStream, "syntax.px", Encoding.UTF8); + + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.MultipleEntriesOnOneLine); + Assert.HasCount(100, synchronousResult.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", synchronousResult.FeedbackItems[key][^1].AdditionalInfo!); + Assert.HasCount(100, asynchronousResult.FeedbackItems[key]); + Assert.Contains("Feedback limit of 100 instances", asynchronousResult.FeedbackItems[key][^1].AdditionalInfo!); + } + [TestMethod] public void ValidateObjectsCalledWithMultipleEntriesInSingleLineReturnsWithWarnings() { @@ -61,7 +167,7 @@ public void ValidateObjectsCalledWithMultipleEntriesInSingleLineReturnsWithWarni List functions = [SyntaxValidationFunctions.MultipleEntriesOnLine]; // Act - feedback = entryValidationMethod?.Invoke(null, [entries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportEntryFeedback(entries, functions); Assert.HasCount(1, feedback); Assert.HasCount(2, feedback.First().Value); @@ -131,7 +237,7 @@ public void ValidateObjectsCalledWithKvpWithMultipleLangParamsReturnsWithError() List functions = [SyntaxValidationFunctions.MoreThanOneLanguageParameter]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.MoreThanOneLanguageParameterSection, feedback.First().Key.Rule); @@ -146,7 +252,7 @@ public void ValidateObjectsCalledWithKvpWithMultipleSpecifierParamsSReturnsWithE List functions = [SyntaxValidationFunctions.MoreThanOneSpecifierParameter]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.MoreThanOneSpecifierParameterSection, feedback.First().Key.Rule); @@ -160,7 +266,7 @@ public void ValidateObjectsCalledWithKvpWithWrongOrderAndMissingKeywordReturnsWi List functions = [SyntaxValidationFunctions.WrongKeyOrderOrMissingKeyword]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(2, feedback); Assert.IsTrue(feedback.ContainsKey(new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.KeyHasWrongOrder))); @@ -182,7 +288,7 @@ public void ValidateObjectsCalledWithKvpWithInvalidSpecifiersReturnsWithErrors() ValidationFeedbackKey notEnclosedFeedbackKey = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.SpecifierPartNotEnclosed); // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(3, feedback); Assert.IsTrue(feedback.ContainsKey(missingDelimeterFeedbackKey)); @@ -199,7 +305,7 @@ public void ValidateObjectsCalledWithKvpWithIllegalSymbolsInLanguageParamReturns List functions = [SyntaxValidationFunctions.IllegalSymbolsInLanguageParamSection]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.HasCount(3, feedback.First().Value); @@ -214,7 +320,7 @@ public void ValidateObjectsCalledWithKvpWithIllegalSymbolsInSpecifierParamReturn List functions = [SyntaxValidationFunctions.IllegalCharactersInSpecifierSection]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.HasCount(3, feedback.First().Value); @@ -229,7 +335,7 @@ public void ValidateObjectsCalledWithKvpWithBadValuesReturnsErrors() List functions = [SyntaxValidationFunctions.InvalidValueFormat]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.HasCount(4, feedback.First().Value); @@ -244,7 +350,7 @@ public void ValidateObjectsCalledWithKvpWithExcessListValueWhitespaceReturnsWith List functions = [SyntaxValidationFunctions.ExcessWhitespaceInValue]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.ExcessWhitespaceInValue, feedback.First().Key.Rule); @@ -258,7 +364,7 @@ public void ValidateObjectsCalledWithKvpWithExcessKeyWhitespaceReturnsWithWarnin List functions = [SyntaxValidationFunctions.KeyContainsExcessWhiteSpace]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.KeyContainsExcessWhiteSpace, feedback.First().Key.Rule); @@ -272,7 +378,7 @@ public void ValidateObjectsCalledWithKvpWithShortMultilineValueReturnsWithWarnin List functions = [SyntaxValidationFunctions.ExcessNewLinesInValue]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Assert.HasCount(1, feedback); Assert.HasCount(2, feedback.First().Value); @@ -289,7 +395,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithInvalidKeywordsReturns ValidationFeedbackKey illegalCharactersFeedbackKey = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.IllegalCharactersInKeyword); // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(2, feedback); Assert.IsTrue(feedback.ContainsKey(startWithletterFeedbackKey)); @@ -305,7 +411,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithValidLanguagesReturnsW List functions = [SyntaxValidationFunctions.IllegalCharactersInLanguageParameter]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(0, feedback); } @@ -318,7 +424,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithInvalidLanguagesReturn List functions = [SyntaxValidationFunctions.IllegalCharactersInLanguageParameter]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.IllegalCharactersInLanguageSection, feedback.First().Key.Rule); @@ -332,7 +438,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithIllegalCharactersInSpe List functions = [SyntaxValidationFunctions.IllegalCharactersInSpecifierParts]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.IllegalCharactersInSpecifierPart, feedback.First().Key.Rule); @@ -346,7 +452,7 @@ public void ValidateObjectsCalledWithEntryWithoutValueReturnsWithError() List functions = [SyntaxValidationFunctions.EntryWithoutValue]; // Act - feedback = entryValidationMethod?.Invoke(null, [entries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportEntryFeedback(entries, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.EntryWithoutValue, feedback.First().Key.Rule); @@ -360,7 +466,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithIncompliantLanguagesRe List functions = [SyntaxValidationFunctions.IncompliantLanguage]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf] ) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(1, feedback); Assert.HasCount(2, feedback.First().Value); @@ -375,7 +481,7 @@ public void ValidateObjectsCalledWithStructuredEntriesWithUnrecommendedKeywordNa List functions = [SyntaxValidationFunctions.KeywordContainsUnderscore, SyntaxValidationFunctions.KeywordIsNotInUpperCase]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf] ) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(2, feedback); Assert.IsTrue(feedback.ContainsKey(new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.KeywordIsNotInUpperCase))); @@ -390,7 +496,7 @@ public void ValidateObjectsCalledWithLongKeywordReturnsWithWarnings() List functions = [SyntaxValidationFunctions.KeywordIsExcessivelyLong]; // Act - feedback = structuredValidationMethod?.Invoke(null, [structuredEntries, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportStructuredFeedback(structuredEntries, functions); Assert.HasCount(1, feedback); Assert.AreEqual(ValidationFeedbackRule.KeywordExcessivelyLong, feedback.First().Key.Rule); @@ -409,7 +515,7 @@ public void GetValueTypeFromStringCalledWithValidListsValuesCorrectValueType(str List functions = [SyntaxValidationFunctions.InvalidValueFormat]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Utils.Validation.ValueType? valueType = getValueTypeFromStringMethod?.Invoke(null, [keyValuePairs[0].KeyValuePair.Value, PxFileConfiguration.Default]) as Utils.Validation.ValueType?; // Assert @@ -440,7 +546,7 @@ public void CorrectlyDefinedRangeAndSeriesTimeValuesReturnCorrectValueType(strin List functions = [SyntaxValidationFunctions.InvalidValueFormat]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Utils.Validation.ValueType? valueType = getValueTypeFromStringMethod?.Invoke(null, [keyValuePairs[0].KeyValuePair.Value, PxFileConfiguration.Default]) as Utils.Validation.ValueType?; // Assert @@ -470,7 +576,7 @@ public void IncorrectlyDefinedRangeAndSeriesTimeValuesReturnWithErrors(string ti List functions = [SyntaxValidationFunctions.InvalidValueFormat]; // Act - feedback = kvpValidationMethod?.Invoke(null, [keyValuePairs, functions, conf]) as ValidationFeedback ?? []; + feedback = ReportKeyValuePairFeedback(keyValuePairs, functions); Utils.Validation.ValueType? valueType = getValueTypeFromStringMethod?.Invoke(null, [keyValuePairs.First().KeyValuePair.Value, PxFileConfiguration.Default]) as Utils.Validation.ValueType?; // Assert diff --git a/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs b/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs new file mode 100644 index 0000000..04a3ad4 --- /dev/null +++ b/Px.Utils.UnitTests/Validation/ValidationFeedbackSinkTests.cs @@ -0,0 +1,67 @@ +using Px.Utils.Validation; + +namespace Px.Utils.UnitTests.Validation +{ + [TestClass] + public class ValidationFeedbackSinkTests + { + [TestMethod] + public void ReportMatchingFeedbackBeyondLimitRetainsLimitAndAnnotatesFinalItem() + { + ValidationFeedbackSink sink = new(new ValidationOptions { MaxFeedbackItemsPerSignature = 2 }); + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + + sink.Report(key, new ValidationFeedbackValue("file.px", 1)); + sink.Report(key, new ValidationFeedbackValue("file.px", 2, additionalInfo: "Original information.")); + sink.Report(key, new ValidationFeedbackValue("file.px", 3)); + sink.Report(key, new ValidationFeedbackValue("file.px", 4)); + + ValidationFeedback feedback = sink.ToFeedback(); + List values = feedback[key]; + + Assert.HasCount(2, values); + Assert.Contains("Original information.", values[1].AdditionalInfo); + Assert.Contains("Feedback limit of 2 instances", values[1].AdditionalInfo); + } + + [TestMethod] + public void ReportUnlimitedFeedbackRetainsAllItems() + { + ValidationFeedbackSink sink = new(ValidationOptions.Unlimited); + ValidationFeedbackKey key = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + + for (int line = 1; line <= 101; line++) + { + sink.Report(key, new ValidationFeedbackValue("file.px", line)); + } + + List values = sink.ToFeedback()[key]; + Assert.HasCount(101, values); + Assert.DoesNotContain(value => value.AdditionalInfo?.Contains("Feedback limit", StringComparison.Ordinal) == true, values); + } + + [TestMethod] + public void ReportDifferentSignaturesRetainsSeparateLimits() + { + ValidationFeedbackSink sink = new(new ValidationOptions { MaxFeedbackItemsPerSignature = 1 }); + ValidationFeedbackKey errorKey = new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + ValidationFeedbackKey warningKey = new(ValidationFeedbackLevel.Warning, ValidationFeedbackRule.DataValidationFeedbackInvalidChar); + + sink.Report(errorKey, new ValidationFeedbackValue("first.px")); + sink.Report(errorKey, new ValidationFeedbackValue("first.px")); + sink.Report(errorKey, new ValidationFeedbackValue("second.px")); + sink.Report(warningKey, new ValidationFeedbackValue("first.px")); + + ValidationFeedback feedback = sink.ToFeedback(); + + Assert.AreEqual(2, feedback[errorKey].Select(value => value.Filename).Distinct(StringComparer.Ordinal).Count()); + Assert.HasCount(1, feedback[warningKey]); + } + + [TestMethod] + public void ValidationOptionsNonPositiveLimitThrows() + { + Assert.ThrowsExactly(() => new ValidationFeedbackSink(new ValidationOptions { MaxFeedbackItemsPerSignature = 0 })); + } + } +} diff --git a/Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs b/Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs index a41ba4a..52249d7 100644 --- a/Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs +++ b/Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs @@ -51,7 +51,7 @@ public MatrixMetadata Build(IEnumerable> metadataIn /// /// Builds a object from a given set of metadata entries. /// - /// A of key-value pairs representing the metadata entries in the Px-File format. + /// A of key-value pairs representing the metadata entries in the Px-File format. /// A object constructed from the input metadata entries. public MatrixMetadata Build(IReadOnlyDictionary metadataInput) { diff --git a/Px.Utils/Operations/DivisionMatrixFunctionExtensions.cs b/Px.Utils/Operations/DivisionMatrixFunctionExtensions.cs index db377e2..1875fb6 100644 --- a/Px.Utils/Operations/DivisionMatrixFunctionExtensions.cs +++ b/Px.Utils/Operations/DivisionMatrixFunctionExtensions.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models; +using Px.Utils.Models; using Px.Utils.Models.Metadata; using System.Numerics; @@ -13,7 +13,7 @@ public static class DivisionMatrixFunctionExtensions /// Divides all datapoints defined by the with a given constant. /// /// Type of the data values in the matrix, must implement - /// and + /// and /// The source matrix for the operation. /// Defines the datapoints to be divided. /// The constant used to divide the datapoints. @@ -28,7 +28,7 @@ public static Matrix DivideSubsetByConstant(this Matrix inp /// Asyncronously divides all datapoints defined by the with a given constant. /// /// Type of the data values in the matrix, must implement - /// and + /// and /// The source matrix for the operation. /// Defines the datapoints to be divided. /// The constant used to divide the datapoints. @@ -43,7 +43,7 @@ public static async Task> DivideSubsetByConstantAsync(this /// Asyncronously divides all datapoints defined by the with a given constant. /// /// Type of the data values in the matrix, must implement - /// and + /// and /// A tasks that produces the source matrix for the operation. /// Defines the datapoints to be divided. /// The constant used to divide the datapoints. @@ -58,7 +58,7 @@ public static async Task> DivideSubsetByConstantAsync(this /// Divides datapoints defined by by the values of datapoints defined by the . /// /// Type of the data values in the matrix, must implement - /// and + /// and /// The source matrix for the operation. /// The datapoints defined by these values will be divided. /// The set of datapoints defined by this dimension value @@ -74,7 +74,7 @@ public static Matrix DivideSubsetBySelectedValue(this Matrix by the values of datapoints defined by the . /// /// Type of the data values in the matrix, must implement - /// and + /// and /// The source matrix for the operation. /// The datapoints defined by these values will be divided. /// The set of datapoints defined by this dimension value @@ -90,7 +90,7 @@ public static async Task> DivideSubsetBySelectedValueAsync( /// Asyncronously divides datapoints defined by by the values of datapoints defined by the . /// /// Type of the data values in the matrix, must implement - /// and + /// and /// A tasks that produces the source matrix for the operation. /// The datapoints defined by these values will be divided. /// The set of datapoints defined by this dimension value diff --git a/Px.Utils/Operations/MultiplicationMatrixFunction.cs b/Px.Utils/Operations/MultiplicationMatrixFunction.cs index ce2685a..97bd169 100644 --- a/Px.Utils/Operations/MultiplicationMatrixFunction.cs +++ b/Px.Utils/Operations/MultiplicationMatrixFunction.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models; +using Px.Utils.Models; using Px.Utils.Models.Metadata; using Px.Utils.Models.Metadata.Dimensions; using System.Numerics; @@ -18,7 +18,8 @@ public static class MultiplicationMatrixFunctionExtensions /// The source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. - /// Also defines which s are included in the sum. + /// Also defines which s are included in the product. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the operation. public static Matrix MultiplyToNewValue(this Matrix input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) where TData : IMultiplyOperators, IMultiplicativeIdentity @@ -26,7 +27,7 @@ public static Matrix MultiplyToNewValue(this Matrix input, return input.ApplyOverDimension(newValue, multiplicationMap, Multiply, TData.MultiplicativeIdentity, insertIndex); } - + /// /// Asynchronously multiplies the values defined in the together and places the product in the new value which will be added to the dimension. /// @@ -35,7 +36,8 @@ public static Matrix MultiplyToNewValue(this Matrix input, /// The source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. - /// Also defines which s are included in the sum. + /// Also defines which s are included in the product. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the operation. public async static Task> MultiplyToNewValueAsync(this Matrix input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) where TData : IMultiplyOperators, IMultiplicativeIdentity @@ -51,7 +53,8 @@ public async static Task> MultiplyToNewValueAsync(this Matr /// The source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the products are calculated. - /// Also defines which s are included in the sum. + /// Also defines which s are included in the product. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the operation. public async static Task> MultiplyToNewValueAsync(this Task> input, DimensionValue newValue, IDimensionMap multiplicationMap, int insertIndex = -1) where TData : IMultiplyOperators, IMultiplicativeIdentity diff --git a/Px.Utils/Operations/SumMatrixFunctionExtensions.cs b/Px.Utils/Operations/SumMatrixFunctionExtensions.cs index eaaa35c..0cd4622 100644 --- a/Px.Utils/Operations/SumMatrixFunctionExtensions.cs +++ b/Px.Utils/Operations/SumMatrixFunctionExtensions.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models; +using Px.Utils.Models; using Px.Utils.Models.Metadata; using Px.Utils.Models.Metadata.Dimensions; using System.Numerics; @@ -18,8 +18,9 @@ public static class SumMatrixFunctionExtensions /// and /// The source matrix for the operation /// This value will be added to the dimension defined by the - /// Defines the relative to which the sums are calculated. + /// Defines the relative to which the sums are calculated. /// Also defines which s are included in the sum. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// A new object that contais the results of the additions. public static Matrix SumToNewValue(this Matrix input, DimensionValue newValue, IDimensionMap sumMap, int insertIndex = -1) where TData : IAdditionOperators, IAdditiveIdentity @@ -35,7 +36,8 @@ public static Matrix SumToNewValue(this Matrix input, Dimen /// and /// The source matrix for the operation /// This value will be added to the dimension defined by the - /// Defines the relative to which the sums are calculated. + /// Defines the relative to which the sums are calculated. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// Also defines which s are included in the sum. /// A new object that contais the results of the additions. public async static Task> SumToNewValueAsync(this Matrix input, DimensionValue newValue, IDimensionMap sumMap, int insetIndex = -1) @@ -52,6 +54,7 @@ public async static Task> SumToNewValueAsync(this MatrixThe source matrix for the operation /// This value will be added to the dimension defined by the /// Defines the relative to which the sums are calculated. + /// The index at which to insert the new value. Defaults to -1, which appends the value to the end. /// Also defines which s are included in the sum. /// A new object that contais the results of the additions. public async static Task> SumToNewValueAsync(this Task> input, DimensionValue newValue, IDimensionMap sumMap, int insetIndex = -1) diff --git a/Px.Utils/Px.Utils.csproj b/Px.Utils/Px.Utils.csproj index 2e85547..146ca66 100644 --- a/Px.Utils/Px.Utils.csproj +++ b/Px.Utils/Px.Utils.csproj @@ -2,7 +2,7 @@ Px.Utils - 1.5.0 + 1.6.0 net10.0 enable enable diff --git a/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs b/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs index 9651e5b..80de91e 100644 --- a/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs +++ b/Px.Utils/PxFile/Data/PxFileStreamDataReader.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models.Data.DataValue; +using Px.Utils.Models.Data.DataValue; using Px.Utils.Models.Metadata; using Px.Utils.Models.Metadata.ExtensionMethods; @@ -25,6 +25,7 @@ public sealed class PxFileStreamDataReader : IPxFileStreamDataReader, IDisposabl /// /// Px file stream /// Px file syntax configuration + /// The size of the buffer used for reading from the stream. public PxFileStreamDataReader(Stream stream, PxFileConfiguration? conf = null, int readBufferSize = 4096) { _stream = stream; @@ -36,8 +37,9 @@ public PxFileStreamDataReader(Stream stream, PxFileConfiguration? conf = null, i /// Constructor that allows specifying the position of the data section in the file. /// /// Px file stream - /// Position of the first data point in the file + /// Absolute raw byte offset of the first non-whitespace data value after DATA=. /// Px file syntax configuration + /// The size of the buffer used for reading from the stream. public PxFileStreamDataReader(Stream stream, long dataStart, PxFileConfiguration? conf = null, int readBufferSize = 4096) { _stream = stream; @@ -244,7 +246,8 @@ await Task.Factory.StartNew(() => /// /// The buffer to store the read values. /// The starting index in the buffer to begin storing the read values. - /// Provides the indexes where the data will be read. + /// Map defining the data to be read. Must be a submap of the map. + /// Map defining the complete data set. /// A to observe while waiting for the task to complete. public async Task ReadDecimalDataValuesAsync(DecimalDataValue[] buffer, int offset, IMatrixMap target, IMatrixMap complete, CancellationToken cancellationToken) { @@ -272,25 +275,23 @@ public void Dispose() private void SetReaderPositionIfZero() { if (_stream.Position != 0) return; - string dataKeyword = _conf.Tokens.KeyWords.Data; - long start = StreamUtilities.FindKeywordPosition(_stream, dataKeyword, _conf); + long start = StreamUtilities.FindDataStartPositionUnchecked(_stream, _conf, _readBufferSize); if (start == -1) { - throw new ArgumentException($"Could not find data keyword '{dataKeyword}'"); + throw new ArgumentException($"Could not find the first data value after '{_conf.Tokens.KeyWords.Data}='"); } - _stream.Position = start + dataKeyword.Length + 1; // +1 to skip the '=' + _stream.Position = start; } private async Task SetReaderPositionIfZeroAsync(CancellationToken? cancellationToken = null) { if (_stream.Position != 0) return; - string dataKeyword = _conf.Tokens.KeyWords.Data; - long start = await StreamUtilities.FindKeywordPositionAsync(_stream, dataKeyword, _conf, cancellationToken); + long start = await StreamUtilities.FindDataStartPositionUncheckedAsync(_stream, _conf, _readBufferSize, cancellationToken ?? CancellationToken.None); if (start == -1) { - throw new ArgumentException($"Could not find data keyword '{dataKeyword}'"); + throw new ArgumentException($"Could not find the first data value after '{_conf.Tokens.KeyWords.Data}='"); } - _stream.Position = start + dataKeyword.Length + 1; // +1 to skip the '=' + _stream.Position = start; } private void ReadItemsFromStreamByCoordinate(T[] buffer, int offset, IMatrixMap target, IMatrixMap complete, Func readItem, CancellationToken? token = null) diff --git a/Px.Utils/PxFile/Data/StreamUtilities.cs b/Px.Utils/PxFile/Data/StreamUtilities.cs index adcc974..720ed74 100644 --- a/Px.Utils/PxFile/Data/StreamUtilities.cs +++ b/Px.Utils/PxFile/Data/StreamUtilities.cs @@ -1,4 +1,5 @@ -using System.Text; +using System.Runtime.CompilerServices; +using System.Text; namespace Px.Utils.PxFile.Data { @@ -8,76 +9,491 @@ namespace Px.Utils.PxFile.Data public static class StreamUtilities { /// - /// Finds the position of the first occurrence of a specified keyword in a given stream. + /// Finds the absolute raw byte offset of the first occurrence of a keyword at the start of a top-level PX entry. /// - /// The stream to search in. + /// The PX file stream to search from its current position. /// The keyword to search for. - /// A configuration object that contains symbols used in the px file syntax. + /// A configuration object that contains PX syntax symbols. /// The size of the buffer to use when reading from the stream. Defaults to 4096. - /// The position of the keyword in the stream if found, otherwise -1. + /// The absolute raw byte offset of the keyword, or -1 when the keyword cannot be found. public static long FindKeywordPosition(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize = 4096) { - return FindKeywordPostionImpl(stream, keyword, conf, bufferSize); + return FindKeywordPositionImpl(stream, keyword, conf, bufferSize); } /// - /// Asynchronously finds the position of the first occurrence of a specified keyword in a given stream. + /// Asynchronously finds the absolute raw byte offset of the first occurrence of a keyword at the start of a top-level PX entry. /// - /// The stream to search in. + /// The PX file stream to search from its current position. /// The keyword to search for. - /// A configuration object that contains symbols used in the px file syntax. - /// A token that can be used to cancel the operation. Defaults to None. + /// A configuration object that contains PX syntax symbols. + /// A token that can be used to cancel the operation. /// The size of the buffer to use when reading from the stream. Defaults to 4096. - /// The task result contains the position of the keyword in the stream if found, otherwise -1. - public async static Task FindKeywordPositionAsync(Stream stream, string keyword, PxFileConfiguration conf, CancellationToken? cToken = null, int bufferSize = 4096) + /// The absolute raw byte offset of the keyword, or -1 when the keyword cannot be found. + public static Task FindKeywordPositionAsync(Stream stream, string keyword, PxFileConfiguration conf, CancellationToken? cancellationToken = null, int bufferSize = 4096) { - return await Task.Factory.StartNew( - () => FindKeywordPostionImpl(stream, keyword, conf, bufferSize, cToken), cToken - ?? CancellationToken.None - ); + return FindKeywordPositionImplAsync(stream, keyword, conf, bufferSize, cancellationToken ?? CancellationToken.None); } - private static long FindKeywordPostionImpl(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize = 4096, CancellationToken? cancellationToken = null) + /// + /// Finds the absolute raw byte offset of the first non-whitespace data value after a top-level DATA entry. + /// The stream position is restored before this method returns. Returns -1 when the DATA entry or its first value cannot be found. + /// + /// The seekable PX file stream to search from its origin. + /// A configuration object that contains the DATA keyword and PX syntax symbols. + /// The size of the buffer to use when reading from the stream. Defaults to 4096. + /// The absolute raw byte offset of the first data value, or -1. + public static long FindDataStartPosition(Stream stream, PxFileConfiguration conf, int bufferSize = 4096) + { + long originalPosition = stream.Position; + try + { + stream.Position = 0; + return FindDataStartPositionImpl(stream, conf, bufferSize); + } + finally + { + stream.Position = originalPosition; + } + } + + /// + /// Finds the first non-whitespace data value after a top-level DATA entry in already validated PX input. + /// The stream is read from its current position and remains advanced. + /// + /// The PX file stream to search from its current position. + /// A configuration object that contains the DATA keyword and PX syntax symbols. + /// The size of the buffer to use when reading from the stream. Defaults to 4096. + /// The absolute raw byte offset of the first data value, or -1. + internal static long FindDataStartPositionUnchecked(Stream stream, PxFileConfiguration conf, int bufferSize = 4096) + { + return FindDataStartPositionUncheckedImpl(stream, conf, bufferSize); + } + + /// + /// Asynchronously finds the first non-whitespace data value after a top-level DATA entry in already validated PX input. + /// The stream is read from its current position and remains advanced. + /// + /// The PX file stream to search from its current position. + /// A configuration object that contains the DATA keyword and PX syntax symbols. + /// The size of the buffer to use when reading from the stream. Defaults to 4096. + /// A token that can be used to cancel the operation. + /// The absolute raw byte offset of the first data value, or -1. + internal static Task FindDataStartPositionUncheckedAsync(Stream stream, PxFileConfiguration conf, int bufferSize = 4096, CancellationToken cancellationToken = default) + { + return FindDataStartPositionUncheckedImplAsync(stream, conf, bufferSize, cancellationToken); + } + + /// + /// Asynchronously finds the absolute raw byte offset of the first non-whitespace data value after a top-level DATA entry. + /// The stream position is restored before this method returns. Returns -1 when the DATA entry or its first value cannot be found. + /// + /// The seekable PX file stream to search from its origin. + /// A configuration object that contains the DATA keyword and PX syntax symbols. + /// A token that can be used to cancel the operation. + /// The size of the buffer to use when reading from the stream. Defaults to 4096. + /// The absolute raw byte offset of the first data value, or -1. + public static async Task FindDataStartPositionAsync(Stream stream, PxFileConfiguration conf, int bufferSize = 4096, CancellationToken cancellationToken = default) + { + long originalPosition = stream.Position; + try + { + stream.Position = 0; + return await FindDataStartPositionImplAsync(stream, conf, bufferSize, cancellationToken); + } + finally + { + stream.Position = originalPosition; + } + } + + private static long FindKeywordPositionImpl(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize) + { + SkipUtf8BomAtStreamOrigin(stream); + byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); + byte[] buffer = new byte[bufferSize]; + TopLevelKeywordSearchState state = new( + keywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.Key.StringDelimeter); + + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + { + long bufferStart = stream.Position - bytesRead; + long keywordPosition = FindKeywordPosition(buffer.AsSpan(0, bytesRead), bufferStart, ref state); + if (keywordPosition >= 0) + { + return keywordPosition; + } + } + + return -1; + } + + private static long FindDataStartPositionUncheckedImpl(Stream stream, PxFileConfiguration conf, int bufferSize) + { + byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data + conf.Symbols.KeywordSeparator); + byte[] buffer = new byte[bufferSize]; + UncheckedDataStartSearchState state = new(dataKeywordBytes, (byte)conf.Symbols.EntrySeparator, (byte)conf.Symbols.Key.StringDelimeter); + + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + { + long bufferStart = stream.Position - bytesRead; + if (TryFindDataStartPositionUnchecked(buffer.AsSpan(0, bytesRead), bufferStart, ref state, out long dataStartPosition)) + { + return dataStartPosition; + } + } + + return -1; + } + + private static async Task FindDataStartPositionUncheckedImplAsync(Stream stream, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) { - char entrySeparator = conf.Symbols.EntrySeparator; + byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data + conf.Symbols.KeywordSeparator); + byte[] buffer = new byte[bufferSize]; + UncheckedDataStartSearchState state = new(dataKeywordBytes, (byte)conf.Symbols.EntrySeparator, (byte)conf.Symbols.Key.StringDelimeter); + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) + { + long bufferStart = stream.Position - bytesRead; + if (TryFindDataStartPositionUnchecked(buffer.AsSpan(0, bytesRead), bufferStart, ref state, out long dataStartPosition)) + { + return dataStartPosition; + } + } + + return -1; + } + + private static async Task FindKeywordPositionImplAsync(Stream stream, string keyword, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) + { + await SkipUtf8BomAtStreamOriginAsync(stream, cancellationToken); byte[] keywordBytes = Encoding.ASCII.GetBytes(keyword + conf.Symbols.KeywordSeparator); byte[] buffer = new byte[bufferSize]; + TopLevelKeywordSearchState state = new( + keywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.Key.StringDelimeter); + + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) + { + long bufferStart = stream.Position - bytesRead; + long keywordPosition = FindKeywordPosition(buffer.AsSpan(0, bytesRead), bufferStart, ref state); + if (keywordPosition >= 0) + { + return keywordPosition; + } + } - long read; - int keywordIndex = 0; - int lastKeyIndex = keywordBytes.Length - 1; - bool searchMode = true; + return -1; + } - do + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static long FindKeywordPosition(ReadOnlySpan buffer, long bufferStart, ref TopLevelKeywordSearchState state) + { + for (int i = 0; i < buffer.Length; i++) { - cancellationToken?.ThrowIfCancellationRequested(); - read = stream.Read(buffer, 0, bufferSize); + if (TryProcessTopLevelKeywordByte(buffer[i], ref state)) + { + return bufferStart + i - state.KeywordBytes.Length + 1; + } + } + + return -1; + } + + private static long FindDataStartPositionImpl(Stream stream, PxFileConfiguration conf, int bufferSize) + { + SkipUtf8BomAtStreamOrigin(stream); + byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); + byte[] buffer = new byte[bufferSize]; + DataStartSearchState state = new( + dataKeywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.KeywordSeparator, + (byte)conf.Symbols.Key.StringDelimeter); + + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + { + long bufferStart = stream.Position - bytesRead; + if (TryFindDataStartPosition( + buffer.AsSpan(0, bytesRead), + bufferStart, + ref state, + out long dataStartPosition)) + { + return dataStartPosition; + } + } + + return -1; + } + + private static async Task FindDataStartPositionImplAsync(Stream stream, PxFileConfiguration conf, int bufferSize, CancellationToken cancellationToken) + { + await SkipUtf8BomAtStreamOriginAsync(stream, cancellationToken); + byte[] dataKeywordBytes = Encoding.ASCII.GetBytes(conf.Tokens.KeyWords.Data); + byte[] buffer = new byte[bufferSize]; + DataStartSearchState state = new( + dataKeywordBytes, + (byte)conf.Symbols.EntrySeparator, + (byte)conf.Symbols.KeywordSeparator, + (byte)conf.Symbols.Key.StringDelimeter); - for (int i = 0; i < read; i++) + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) + { + long bufferStart = stream.Position - bytesRead; + if (TryFindDataStartPosition( + buffer.AsSpan(0, bytesRead), + bufferStart, + ref state, + out long dataStartPosition)) { - if (searchMode && !CharacterConstants.WhitespaceCharacters.Contains((char)buffer[i])) + return dataStartPosition; + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryFindDataStartPosition( + ReadOnlySpan buffer, + long bufferStart, + ref DataStartSearchState state, + out long dataStartPosition) + { + for (int i = 0; i < buffer.Length; i++) + { + if (TryProcessDataStartByte(buffer[i], ref state)) + { + dataStartPosition = bufferStart + i; + return true; + } + if (state.IsDataEntryEmpty) + { + dataStartPosition = -1; + return true; + } + } + + dataStartPosition = -1; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryProcessDataStartByte(byte currentByte, ref DataStartSearchState state) + { + if (state.IsAfterDataKeyword) + { + if (currentByte == state.EntrySeparator) + { + state.IsDataEntryEmpty = true; + return false; + } + return !IsWhitespace(currentByte); + } + if (state.IsInString) + { + state.IsInString = currentByte != state.StringDelimiter; + return false; + } + if (currentByte == state.StringDelimiter) + { + state.IsInString = true; + return false; + } + if (currentByte == state.EntrySeparator) + { + state.KeywordSearchState.Reset(); + return false; + } + if (TryProcessEntryKeywordByte(currentByte, ref state.KeywordSearchState)) + { + state.IsAfterDataKeyword = true; + } + + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryFindDataStartPositionUnchecked(ReadOnlySpan buffer, long bufferStart, ref UncheckedDataStartSearchState state, out long dataStartPosition) + { + for (int i = 0; i < buffer.Length; i++) + { + byte currentByte = buffer[i]; + if (state.IsAfterDataKeyword) + { + if (currentByte == state.EntrySeparator) { - if (buffer[i] == keywordBytes[keywordIndex]) - { - if (keywordIndex == lastKeyIndex) return stream.Position - read + i - keyword.Length; - else keywordIndex++; - } - else - { - searchMode = false; - keywordIndex = 0; - } + dataStartPosition = -1; + return true; } - else if (buffer[i] == entrySeparator) + if (!IsWhitespace(currentByte)) { - searchMode = true; + dataStartPosition = bufferStart + i; + return true; } + + continue; + } + + if (TryProcessTopLevelKeywordByte(currentByte, ref state.KeywordSearchState)) + { + state.IsAfterDataKeyword = true; } } - while (read > 0); - return -1; + dataStartPosition = -1; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryProcessEntryKeywordByte(byte currentByte, ref EntryKeywordSearchState state) + { + if (currentByte == state.EntrySeparator) + { + state.Reset(); + return false; + } + if (!state.IsAtEntryStart || IsWhitespace(currentByte)) + { + return false; + } + if (state.MatchedKeywordBytes < state.KeywordBytes.Length && currentByte == state.KeywordBytes[state.MatchedKeywordBytes]) + { + state.MatchedKeywordBytes++; + return state.MatchedKeywordBytes == state.KeywordBytes.Length; + } + + state.IsAtEntryStart = false; + state.MatchedKeywordBytes = 0; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryProcessTopLevelKeywordByte(byte currentByte, ref TopLevelKeywordSearchState state) + { + if (state.IsInString) + { + state.IsInString = currentByte != state.StringDelimiter; + return false; + } + if (currentByte == state.StringDelimiter) + { + state.IsInString = true; + return false; + } + if (currentByte == state.EntrySeparator) + { + state.Reset(); + return false; + } + if (!state.IsAtEntryStart || IsWhitespace(currentByte)) + { + return false; + } + if (state.MatchedKeywordBytes < state.KeywordBytes.Length && currentByte == state.KeywordBytes[state.MatchedKeywordBytes]) + { + state.MatchedKeywordBytes++; + return state.MatchedKeywordBytes == state.KeywordBytes.Length; + } + + state.IsAtEntryStart = false; + state.MatchedKeywordBytes = 0; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsWhitespace(byte value) + { + return value is CharacterConstants.SPACE or CharacterConstants.HORIZONTALTAB or CharacterConstants.CARRIAGERETURN or CharacterConstants.LINEFEED; + } + + private static void SkipUtf8BomAtStreamOrigin(Stream stream) + { + if (!stream.CanSeek || stream.Position != 0 || stream.Length < Encoding.UTF8.Preamble.Length) + { + return; + } + + Span prefix = stackalloc byte[Encoding.UTF8.Preamble.Length]; + int bytesRead = stream.Read(prefix); + if (bytesRead != prefix.Length || !prefix.SequenceEqual(Encoding.UTF8.Preamble)) + { + stream.Position = 0; + } + } + + private static async Task SkipUtf8BomAtStreamOriginAsync(Stream stream, CancellationToken cancellationToken) + { + if (!stream.CanSeek || stream.Position != 0 || stream.Length < Encoding.UTF8.Preamble.Length) + { + return; + } + + byte[] prefix = new byte[Encoding.UTF8.Preamble.Length]; + int bytesRead = await stream.ReadAsync(prefix, cancellationToken); + if (bytesRead != prefix.Length || !prefix.AsSpan().SequenceEqual(Encoding.UTF8.Preamble)) + { + stream.Position = 0; + } + } + + private struct DataStartSearchState(byte[] dataKeywordBytes, byte entrySeparator, byte keywordSeparator, byte stringDelimiter) + { + public readonly byte EntrySeparator = entrySeparator; + public readonly byte StringDelimiter = stringDelimiter; + public EntryKeywordSearchState KeywordSearchState = new([.. dataKeywordBytes, keywordSeparator], entrySeparator); + public bool IsInString; + public bool IsAfterDataKeyword; + public bool IsDataEntryEmpty; + } + + private struct UncheckedDataStartSearchState(byte[] dataKeywordBytes, byte entrySeparator, byte stringDelimiter) + { + public readonly byte EntrySeparator = entrySeparator; + public TopLevelKeywordSearchState KeywordSearchState = new(dataKeywordBytes, entrySeparator, stringDelimiter); + public bool IsAfterDataKeyword; + } + + private struct EntryKeywordSearchState(byte[] keywordBytes, byte entrySeparator) + { + public readonly byte[] KeywordBytes = keywordBytes; + public readonly byte EntrySeparator = entrySeparator; + public int MatchedKeywordBytes; + public bool IsAtEntryStart = true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Reset() + { + MatchedKeywordBytes = 0; + IsAtEntryStart = true; + } + } + + private struct TopLevelKeywordSearchState(byte[] keywordBytes, byte entrySeparator, byte stringDelimiter) + { + public readonly byte[] KeywordBytes = keywordBytes; + public readonly byte EntrySeparator = entrySeparator; + public readonly byte StringDelimiter = stringDelimiter; + public int MatchedKeywordBytes; + public bool IsAtEntryStart = true; + public bool IsInString; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Reset() + { + MatchedKeywordBytes = 0; + IsAtEntryStart = true; + } } } } diff --git a/Px.Utils/PxFile/Metadata/IPxFileMetadataReader.cs b/Px.Utils/PxFile/Metadata/IPxFileMetadataReader.cs index f31cca6..3bdc9e6 100644 --- a/Px.Utils/PxFile/Metadata/IPxFileMetadataReader.cs +++ b/Px.Utils/PxFile/Metadata/IPxFileMetadataReader.cs @@ -1,4 +1,4 @@ -using System.Text; +using System.Text; namespace Px.Utils.PxFile.Metadata { @@ -13,11 +13,12 @@ public interface IPxFileMetadataReader /// The stream from which to determine the encoding. /// The determined encoding of the stream. Encoding GetEncoding(Stream stream); - + /// /// Asynchronously determines the encoding of the provided stream based on the Byte Order Mark (BOM) or the CODEPAGE keyword in the metadata. /// /// The stream from which to determine the encoding. + /// A token that can be used to cancel the operation. /// The determined encoding of the stream. Task GetEncodingAsync(Stream stream, CancellationToken cancellationToken = default); @@ -38,6 +39,7 @@ public interface IPxFileMetadataReader /// The stream from which to read the metadata. /// The encoding to use when reading the stream. /// The size of the buffer to use when reading the stream. If not specified, the default buffer size is used. + /// A token that can be used to cancel the operation. /// An of key-value pairs representing the metadata entries in the file. IAsyncEnumerable> ReadMetadataAsync( Stream stream, @@ -63,6 +65,7 @@ IAsyncEnumerable> ReadMetadataAsync( /// The stream from which to read the metadata. /// The encoding to use when reading the stream. /// The size of the buffer to use when reading the stream. If not specified, the default buffer size is used. + /// A token that can be used to cancel the operation. /// A dictionary containing the metadata entries in the file. Task> ReadMetadataToDictionaryAsync( Stream stream, diff --git a/Px.Utils/Validation/ContentValidation/ContentValidationOutput.cs b/Px.Utils/Validation/ContentValidation/ContentValidationOutput.cs new file mode 100644 index 0000000..79644a4 --- /dev/null +++ b/Px.Utils/Validation/ContentValidation/ContentValidationOutput.cs @@ -0,0 +1,4 @@ +namespace Px.Utils.Validation.ContentValidation +{ + internal readonly record struct ContentValidationOutput(int DataRowLength, int DataRowAmount); +} \ No newline at end of file diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.UtilityMethods.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.UtilityMethods.cs index 6a2aaaa..abd916c 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.UtilityMethods.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.UtilityMethods.cs @@ -1,4 +1,4 @@ -using Px.Utils.PxFile; +using Px.Utils.PxFile; using Px.Utils.Validation.SyntaxValidation; namespace Px.Utils.Validation.ContentValidation @@ -154,6 +154,7 @@ [.. values.Select(v => SyntaxValidationUtilityMethods.CleanString(v, conf))] /// /// Keywords to search for that are language specific /// Language agnostic keywords + /// Recommended keywords that should be present in the Px file /// Structured entries of Px file metadata to be searched from /// Object that provides information of the ongoing content validation process /// KeyValuePair that contains the processed language as key and the dimension name as value diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index 5f0ab0a..5ca8905 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -25,7 +25,7 @@ public sealed partial class ContentValidator /// /// Validates that given entry does not contain specifiers if it is not allowed to have them based on keyword. /// - /// Px file metadata entries in an array of objects + /// Px file metadata entry represented by a object /// object that stores information that is gathered during the validation process /// Key value pair containing information about the rule violation is returned if an unexpected specifier is detected. public static ValidationFeedback? ValidateUnexpectedSpecifiers(ValidationStructuredEntry entry, ContentValidator validator) diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.cs index 87c4e27..8d6c18c 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.cs @@ -55,6 +55,25 @@ public sealed partial class ContentValidator( /// /// object that contains the feedback gathered during the validation process. public ContentValidationResult Validate() + { + ValidationFeedbackSink sink = new(); + ContentValidationOutput output = ValidateIntoSink(sink); + return new ContentValidationResult(sink.ToFeedback(), output.DataRowLength, output.DataRowAmount); + } + + /// + /// Validates contents of PX file metadata using the specified feedback retention options. + /// + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The content validation result with retained feedback and calculated data dimensions. + public ContentValidationResult Validate(ValidationOptions options) + { + ValidationFeedbackSink sink = new(options); + ContentValidationOutput output = ValidateIntoSink(sink); + return new ContentValidationResult(sink.ToFeedback(), output.DataRowLength, output.DataRowAmount); + } + + internal ContentValidationOutput ValidateIntoSink(ValidationFeedbackSink sink) { IEnumerable contentValidationEntryFunctions = DefaultContentValidationEntryFunctions; IEnumerable contentValidationFindKeywordFunctions = DefaultContentValidationFindKeywordFunctions; @@ -65,14 +84,12 @@ public ContentValidationResult Validate() contentValidationFindKeywordFunctions = contentValidationFindKeywordFunctions.Concat(customContentValidationFunctions.CustomContentValidationFindKeywordFunctions); } - ValidationFeedback feedbackItems = []; - foreach (ContentValidationFindKeywordValidator findingFunction in contentValidationFindKeywordFunctions) { ValidationFeedback? feedback = findingFunction(entries, this); if (feedback is not null) { - feedbackItems.AddRange(feedback); + sink.ReportRange(feedback); } } foreach (ContentValidationEntryValidator entryFunction in contentValidationEntryFunctions) @@ -82,7 +99,7 @@ public ContentValidationResult Validate() ValidationFeedback? feedback = entryFunction(entry, this); if (feedback is not null) { - feedbackItems.AddRange(feedback); + sink.ReportRange(feedback); } } } @@ -90,7 +107,7 @@ public ContentValidationResult Validate() int amountOfDataRows = _stubDimensionNames is not null ? GetProductOfDimensionValues(_stubDimensionNames) : 0; ResetFields(); - return new ContentValidationResult(feedbackItems, lengthOfDataRows, amountOfDataRows); + return new ContentValidationOutput(lengthOfDataRows, amountOfDataRows); } #region Interface implementation diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index f726cec..25e1c78 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -1,6 +1,7 @@ using System.Runtime.CompilerServices; using System.Text; using Px.Utils.PxFile; +using Px.Utils.PxFile.Data; using Px.Utils.Validation.DatabaseValidation; namespace Px.Utils.Validation.DataValidation @@ -15,7 +16,6 @@ namespace Px.Utils.Validation.DataValidation public class DataValidator(int rowLen, int numOfRows, int startRow, PxFileConfiguration? conf = null) : IPxFileStreamValidator, IPxFileStreamValidatorAsync { private const int _streamBufferSize = 4096; - private static ReadOnlySpan MissingValueStartBytes => [CharacterConstants.QUOTATIONMARK, (byte)'-', (byte)'.']; private readonly PxFileConfiguration _conf = conf ?? PxFileConfiguration.Default; @@ -50,75 +50,139 @@ public ValidationResult Validate( string filename, Encoding? encoding = null, IFileSystem? fileSystem = null) + { + ValidationFeedbackSink sink = new(); + ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new ValidationResult(sink.ToFeedback()); + } + + /// + /// Validates data using the specified feedback retention options. + /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The data validation result with retained feedback. + public ValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options) + { + ValidationFeedbackSink sink = new(options); + ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new ValidationResult(sink.ToFeedback()); + } + + internal void ValidateIntoSink( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink) { fileSystem ??= new LocalFileSystem(); - encoding ??= fileSystem.GetEncoding(stream); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = fileSystem.GetEncoding(stream); + stream.Position = originalPosition; + } SetValidationParameters(encoding, filename); - ValidationFeedback validationFeedbacks = []; - int dataStartIndex = GetStreamIndexOfFirstDataValue(stream, ref validationFeedbacks); + long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); if (dataStartIndex == -1) { - KeyValuePair feedback = - new(new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.StartOfDataSectionNotFound), - new(filename, 0, 0)); - validationFeedbacks.Add(feedback); - - return new (validationFeedbacks); + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), + new(filename, 0, 0))); + ResetValidator(); + return; } - stream.Position = dataStartIndex; - ValidationFeedback dataStreamFeedbacks = ValidateDataStream(stream); - validationFeedbacks.AddRange(dataStreamFeedbacks); + stream.Position = dataStartIndex; + ValidateDataStream(stream, sink); ResetValidator(); + } - return new (validationFeedbacks); + /// + /// Asynchronously validates data using the specified feedback retention options. + /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// A token that cancels the operation. + /// A task that produces the data validation result with retained feedback. + public async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options, + CancellationToken cancellationToken = default) + { + ValidationFeedbackSink sink = new(options); + await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new ValidationResult(sink.ToFeedback()); } /// /// Validates the data in the specified stream asynchronously. /// Assumes that the stream is at the start of the data section (after 'DATA='-keyword) at the first data item. - /// + /// /// Px file stream to be validated - /// Encoding of the stream /// Name of the file being validated. If not provided, validator tries to find the encoding. + /// Encoding of the stream. /// File system used for file operations. If not provided, default file system is used. - /// Cancellation token for cancelling the validation process - /// object that contains a collection of - /// validation feedback key value pairs representing the feedback for the data validation. - /// + /// Cancellation token for cancelling the validation process. + /// A task that produces validation feedback for the data section. public async Task ValidateAsync( Stream stream, string filename, Encoding? encoding = null, IFileSystem? fileSystem = null, CancellationToken cancellationToken = default) + { + ValidationFeedbackSink sink = new(); + await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new ValidationResult(sink.ToFeedback()); + } + + internal async Task ValidateIntoSinkAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink, + CancellationToken cancellationToken = default) { fileSystem ??= new LocalFileSystem(); - encoding ??= await fileSystem.GetEncodingAsync(stream, cancellationToken); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } SetValidationParameters(encoding, filename); - - ValidationFeedback validationFeedbacks = []; - int dataStartIndex = GetStreamIndexOfFirstDataValue(stream, ref validationFeedbacks); + + long dataStartIndex = GetStreamIndexOfFirstDataValue(stream); if (dataStartIndex == -1) { - KeyValuePair feedback = - new(new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.StartOfDataSectionNotFound), - new(filename, 0, 0)); - validationFeedbacks.Add(feedback); - - return new (validationFeedbacks); + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), + new(filename, 0, 0))); + ResetValidator(); + return; } - stream.Position = dataStartIndex; - ValidationFeedback dataStreamFeedbacks = await Task.Factory.StartNew(() => - ValidateDataStream(stream, cancellationToken), cancellationToken); - validationFeedbacks.AddRange(dataStreamFeedbacks); + stream.Position = dataStartIndex; + await Task.Factory.StartNew(() => ValidateDataStream(stream, sink, cancellationToken), cancellationToken); ResetValidator(); - - return new (validationFeedbacks); } private void SetValidationParameters(Encoding encoding, string filename) @@ -185,6 +249,52 @@ private ValidationFeedback ValidateDataStream(Stream stream, CancellationToken? return validationFeedbacks; } + private void ValidateDataStream(Stream stream, ValidationFeedbackSink sink, CancellationToken? cancellationToken = null) + { + byte endOfData = (byte)_conf.Symbols.EntrySeparator; + _currentEntry = new(_streamBufferSize); + byte[] buffer = new byte[_streamBufferSize]; + int bytesRead = 0; + + do + { + cancellationToken?.ThrowIfCancellationRequested(); + for (int i = 0; i < bytesRead; i++) + { + byte currentByte = buffer[i]; + _currentCharacterType = currentByte switch + { + CharacterConstants.SPACE or CharacterConstants.HORIZONTALTAB => EntryType.DataItemSeparator, + CharacterConstants.LINEFEED or CharacterConstants.CARRIAGERETURN => EntryType.LineSeparator, + >= CharacterConstants.QUOTATIONMARK and not CharacterConstants.SEMICOLON => EntryType.DataItem, + _ when currentByte == endOfData => EntryType.EndOfData, + _ => EntryType.Unknown + }; + if (_currentCharacterType != _currentEntryType) + { + HandleEntryTypeChange(sink); + if (_currentCharacterType != EntryType.DataItemSeparator) + { + HandleNonSeparatorType(sink); + } + _currentEntryType = _currentCharacterType; + _currentEntry.Clear(); + } + + _currentEntry.Add(currentByte); + _charPosition++; + } + } + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0); + + if (numOfRows != _lineNumber - 1) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidRowCount), + new(_filename, _lineNumber + startRow, _charPosition, $" Expected {numOfRows} rows, got {_lineNumber - 1} rows."))); + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void HandleEntryTypeChange(ref ValidationFeedback validationFeedbacks) { @@ -210,7 +320,7 @@ private void HandleEntryTypeChange(ref ValidationFeedback validationFeedbacks) KeyValuePair? feedback = validator.Validate( _currentEntry, _currentEntryType, - _encoding, + _encoding, _lineNumber + startRow, _charPosition, _filename); @@ -222,6 +332,35 @@ private void HandleEntryTypeChange(ref ValidationFeedback validationFeedbacks) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void HandleEntryTypeChange(ValidationFeedbackSink sink) + { + if (_currentEntryType == EntryType.Unknown && (_lineNumber > 1 || _charPosition > 0)) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidChar), + new(_filename, _lineNumber + startRow, _charPosition))); + return; + } + + List validators = _currentEntryType switch + { + EntryType.DataItemSeparator => _dataSeparatorValidators, + EntryType.DataItem => _currentEntry[^1] > '.' ? _dataNumValidators : _dataStringValidators, + _ => _commonValidators + }; + + foreach (IDataValidator validator in validators) + { + KeyValuePair? feedback = validator.Validate( + _currentEntry, _currentEntryType, _encoding, _lineNumber + startRow, _charPosition, _filename); + if (feedback is not null) + { + sink.Report(feedback.Value); + } + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void HandleNonSeparatorType(ref ValidationFeedback validationFeedbacks) { @@ -234,7 +373,7 @@ private void HandleNonSeparatorType(ref ValidationFeedback validationFeedbacks) if (_currentRowLength != rowLen) { validationFeedbacks.Add(new( - new (ValidationFeedbackLevel.Error, + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidRowLength), new(_filename, _lineNumber + startRow, _charPosition, $"Expected {rowLen}, got row length of {_currentRowLength}.")) @@ -246,6 +385,27 @@ private void HandleNonSeparatorType(ref ValidationFeedback validationFeedbacks) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void HandleNonSeparatorType(ValidationFeedbackSink sink) + { + if (_currentCharacterType == EntryType.DataItem) + { + _currentRowLength++; + } + else if (_currentCharacterType == EntryType.LineSeparator) + { + if (_currentRowLength != rowLen) + { + sink.Report(new( + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.DataValidationFeedbackInvalidRowLength), + new(_filename, _lineNumber + startRow, _charPosition, $"Expected {rowLen}, got row length of {_currentRowLength}."))); + } + _lineNumber++; + _currentRowLength = 0; + _charPosition = 0; + } + } + private void ResetValidator() { _commonValidators.Clear(); @@ -259,40 +419,24 @@ private void ResetValidator() _currentRowLength = 0; } - private int GetStreamIndexOfFirstDataValue(Stream stream, ref ValidationFeedback feedbacks) + private long GetStreamIndexOfFirstDataValue(Stream stream) { - byte[] buffer = new byte[_streamBufferSize]; - int bytesRead; - do + if (stream.Position == 0) { - bytesRead = stream.Read(buffer, 0, buffer.Length); - for (int i = 0; i < bytesRead; i++) + return StreamUtilities.FindDataStartPosition(stream, _conf, _streamBufferSize); + } + + int currentByte; + while ((currentByte = stream.ReadByte()) != -1) + { + if (currentByte is not CharacterConstants.SPACE and not CharacterConstants.HORIZONTALTAB and not CharacterConstants.CARRIAGERETURN and not CharacterConstants.LINEFEED) { - byte currentByte = buffer[i]; - char currentChar = (char)currentByte; - if (IsDataValueStartByte(currentByte)) - { - return (int)stream.Position - bytesRead + i; - } - else if (!CharacterConstants.WhitespaceCharacters.Contains(currentChar)) - { - feedbacks.Add(new( - new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.DataValidationFeedbackInvalidChar), - new(_filename, _lineNumber + startRow, _charPosition)) - ); - } + return stream.Position - 1; } - } while (bytesRead > 0); + } return -1; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsDataValueStartByte(byte currentByte) - { - return currentByte >= CharacterConstants.Zero && currentByte <= CharacterConstants.Nine || MissingValueStartBytes.Contains(currentByte); - } } /// diff --git a/Px.Utils/Validation/DataValidation/DataValidatorFunctions.cs b/Px.Utils/Validation/DataValidation/DataValidatorFunctions.cs index 727b5a7..42c9eaf 100644 --- a/Px.Utils/Validation/DataValidation/DataValidatorFunctions.cs +++ b/Px.Utils/Validation/DataValidation/DataValidatorFunctions.cs @@ -1,4 +1,4 @@ -using Px.Utils.PxFile; +using Px.Utils.PxFile; using System.Globalization; using System.Text; @@ -32,6 +32,7 @@ public class DataStringValidator : IDataValidator /// Encoding format of the Px file. /// Line number for the validation item. /// Represents the position relative to the line for the validation item. + /// The name of the file being validated. /// Key value pair containing information about the rule violation if the entry is not a missing value string sequence, otherwise null. public KeyValuePair? Validate(List entry, EntryType entryType, Encoding encoding, int lineNumber, int charPos, string filename) { @@ -63,6 +64,7 @@ public class DataNumberValidator : IDataValidator /// Encoding format of the Px file. /// Line number for the validation item. /// Represents the position relative to the line for the validation item. + /// The name of the file being validated. /// Key value pair containing information about the rule violation if the entry is not a valid number, otherwise null. public KeyValuePair? Validate(List entry, EntryType entryType, Encoding encoding, int lineNumber, int charPos, string filename) { @@ -145,6 +147,7 @@ public class DataSeparatorValidator : IDataValidator /// Encoding format of the Px file. /// Line number for the validation item. /// Represents the position relative to the line for the validation item. + /// The name of the file being validated. /// Key value pair containing information about the rule violation if the entry is not a valid item separator, otherwise null. public KeyValuePair? Validate(List entry, EntryType entryType, Encoding encoding, int lineNumber, int charPos, string filename) { @@ -188,7 +191,7 @@ public class DataStructureValidator : IDataValidator /// Encoding format of the Px file. /// Line number for the validation item. /// Represents the position relative to the line for the validation item. - /// Reference to a list of feedback items to which any validation feedback is added to. + /// The name of the file being validated. /// Key value pair containing information about the rule violation if the entry sequence is invalid. Otherwise null. public KeyValuePair? Validate(List entry, EntryType entryType, Encoding encoding, int lineNumber, int charPos, string filename) { diff --git a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs index 3a16495..59c6eec 100644 --- a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs +++ b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs @@ -38,8 +38,16 @@ public class DatabaseValidator( /// /// object that contains feedback gathered during the validation process. public ValidationResult Validate() + => Validate(new ValidationOptions()); + + /// + /// Runs database validation using the specified feedback retention options. + /// + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The database validation result with retained feedback. + public ValidationResult Validate(ValidationOptions options) { - ValidationFeedback feedbacks = []; + ValidationFeedbackSink sink = new(options); ConcurrentBag pxFiles = []; ConcurrentBag aliasFiles = []; List fileTasks = []; @@ -49,9 +57,9 @@ public ValidationResult Validate() { fileTasks.Add(Task.Run(() => { - (DatabaseFileInfo? file, ValidationFeedback feedback) = ProcessPxFile(fileName); + (DatabaseFileInfo? file, ValidationFeedback feedback) = ProcessPxFile(fileName, sink); if (file != null) pxFiles.Add(file); - feedbacks.AddRange(feedback); + sink.ReportRange(feedback); })); } @@ -66,8 +74,8 @@ public ValidationResult Validate() } Task.WaitAll([.. fileTasks]); - feedbacks.AddRange(ValidateDatabaseContents(pxFiles, aliasFiles)); - return new (feedbacks); + sink.ReportRange(ValidateDatabaseContents(pxFiles, aliasFiles)); + return new(sink.ToFeedback()); } /// @@ -76,8 +84,17 @@ public ValidationResult Validate() /// Optional cancellation token /// object that contains feedback gathered during the validation process. public async Task ValidateAsync(CancellationToken cancellationToken = default) + => await ValidateAsync(new ValidationOptions(), cancellationToken); + + /// + /// Runs database validation asynchronously using the specified feedback retention options. + /// + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// A token that cancels the operation. + /// A task that produces the database validation result with retained feedback. + public async Task ValidateAsync(ValidationOptions options, CancellationToken cancellationToken = default) { - ValidationFeedback feedbacks = []; + ValidationFeedbackSink sink = new(options); ConcurrentBag pxFiles = []; ConcurrentBag aliasFiles = []; List fileTasks = []; @@ -87,9 +104,9 @@ public async Task ValidateAsync(CancellationToken cancellation { fileTasks.Add(Task.Run(async () => { - (DatabaseFileInfo? file, ValidationFeedback feedback) = await ProcessPxFileAsync(fileName, cancellationToken); + (DatabaseFileInfo? file, ValidationFeedback feedback) = await ProcessPxFileAsync(fileName, sink, cancellationToken); if (file != null) pxFiles.Add(file); - feedbacks.AddRange(feedback); + sink.ReportRange(feedback); }, cancellationToken)); } @@ -104,11 +121,11 @@ public async Task ValidateAsync(CancellationToken cancellation } await Task.WhenAll(fileTasks); - feedbacks.AddRange(ValidateDatabaseContents(pxFiles, aliasFiles)); - return new (feedbacks); + sink.ReportRange(ValidateDatabaseContents(pxFiles, aliasFiles)); + return new(sink.ToFeedback()); } - private (DatabaseFileInfo?, ValidationFeedback) ProcessPxFile(string fileName) + private (DatabaseFileInfo?, ValidationFeedback) ProcessPxFile(string fileName, ValidationFeedbackSink sink) { ValidationFeedback feedbacks = []; using Stream stream = _fileSystem.GetFileStream(fileName); @@ -120,7 +137,7 @@ public async Task ValidateAsync(CancellationToken cancellation } stream.Position = 0; PxFileValidator validator = new(_conf); - feedbacks.AddRange(validator.Validate(stream, fileName, fileInfo.Encoding).FeedbackItems); + validator.ValidateIntoSink(stream, fileName, fileInfo.Encoding, null, sink); return (fileInfo, feedbacks); } @@ -130,7 +147,7 @@ private DatabaseFileInfo ProcessAliasFile(string fileName) return GetAliasFileInfo(fileName, stream); } - private async Task<(DatabaseFileInfo?, ValidationFeedback)> ProcessPxFileAsync(string fileName, CancellationToken cancellationToken) + private async Task<(DatabaseFileInfo?, ValidationFeedback)> ProcessPxFileAsync(string fileName, ValidationFeedbackSink sink, CancellationToken cancellationToken) { ValidationFeedback feedbacks = []; using Stream stream = _fileSystem.GetFileStream(fileName); @@ -142,8 +159,7 @@ private DatabaseFileInfo ProcessAliasFile(string fileName) } stream.Position = 0; PxFileValidator validator = new(_conf); - ValidationResult result = await validator.ValidateAsync(stream, fileName, fileInfo.Encoding, cancellationToken: cancellationToken); - feedbacks.AddRange(result.FeedbackItems); + await validator.ValidateIntoSinkAsync(stream, fileName, fileInfo.Encoding, null, sink, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); return (fileInfo, feedbacks); } @@ -413,21 +429,55 @@ private async Task GetAliasFileInfoAsync(string filename, Stre } } + /// + /// Represents a database validation item. + /// + /// The path of the database validation item. public class DatabaseValidationItem(string path) { + /// + /// Gets the path of the file or directory. + /// public string Path { get; } = path; } + /// + /// Represents a px file or alias file within a px file database for validation purposes. + /// + /// Name of the file. + /// Path of the file's directory. + /// Languages associated with the file. + /// Encoding of the file. public class DatabaseFileInfo(string name, string location, string[] languages, Encoding encoding) : DatabaseValidationItem(name) { + /// + /// Gets the name of the file. + /// public string Name { get; } = name; + /// + /// Gets the path of the file's directory. + /// public string Location { get; } = location; + /// + /// Gets the languages associated with the file. + /// public string[] Languages { get; } = languages; + /// + /// Gets the encoding of the file. + /// public Encoding Encoding { get; } = encoding; } + /// + /// Validator interface for validating a px file database including px files, alias files, and directory structures. + /// public interface IDatabaseValidator { + /// + /// Validates a given and returns a feedback entry if validation fails, or if validation passes. + /// + /// The database validation item to validate. + /// A key-value pair representing the validation feedback if validation fails, or if validation passes. public KeyValuePair? Validate(DatabaseValidationItem item); } } diff --git a/Px.Utils/Validation/DatabaseValidation/IFileSystem.cs b/Px.Utils/Validation/DatabaseValidation/IFileSystem.cs index 6439403..31fad9a 100644 --- a/Px.Utils/Validation/DatabaseValidation/IFileSystem.cs +++ b/Px.Utils/Validation/DatabaseValidation/IFileSystem.cs @@ -1,4 +1,4 @@ -using Px.Utils.PxFile.Metadata; +using Px.Utils.PxFile.Metadata; using System.Diagnostics.CodeAnalysis; using System.Text; @@ -9,12 +9,49 @@ namespace Px.Utils.Validation.DatabaseValidation /// public interface IFileSystem { + /// + /// Enumerates all files in the specified directory and its subdirectories that match the given search pattern. + /// + /// Path to the directory to search. + /// Pattern to match against the names of files in the directory. This parameter can contain a combination of literal and wildcard characters. + /// Enumerable collection of file paths that match the search pattern. public IEnumerable EnumerateFiles(string path, string searchPattern); + /// + /// Gets a read-only stream for the specified file path. + /// + /// Path to the file to open. + /// Stream for reading the specified file. public Stream GetFileStream(string path); + /// + /// Enumerates all directories in the specified path and its subdirectories. + /// + /// Path to the directory to search. + /// Enumerable collection of directory paths. public IEnumerable EnumerateDirectories(string path); + /// + /// Gets the file name and extension of the specified path string. + /// + /// Path string from which to get the file name and extension. + /// Name and extension of the specified path string. public string GetFileName(string path); + /// + /// Gets the directory name of the specified path string. + /// + /// Path string from which to get the directory name. + /// Name of the directory from the specified path string. public string GetDirectoryName(string path); + /// + /// Gets the encoding of the specified stream. This method may read from the stream to determine its encoding, so the stream position may be changed after calling this method. + /// + /// Stream from which to determine the encoding. + /// Encoding of the specified stream. public Encoding GetEncoding(Stream stream); + /// + /// Asynchronously gets the encoding of the specified stream. This method may read from the stream to determine its encoding, so the stream position may be changed after calling this method. + /// + /// Stream from which to determine the encoding. + /// Token to monitor for cancellation requests. + /// Task representing the asynchronous operation, with the encoding of the specified stream as the result. public Task GetEncodingAsync(Stream stream, CancellationToken cancellationToken); } } diff --git a/Px.Utils/Validation/DatabaseValidation/LocalFileSystem.cs b/Px.Utils/Validation/DatabaseValidation/LocalFileSystem.cs index 0e08c63..9c8914a 100644 --- a/Px.Utils/Validation/DatabaseValidation/LocalFileSystem.cs +++ b/Px.Utils/Validation/DatabaseValidation/LocalFileSystem.cs @@ -1,49 +1,56 @@ -using Px.Utils.PxFile.Metadata; +using Px.Utils.PxFile.Metadata; using System.Diagnostics.CodeAnalysis; using System.Text; namespace Px.Utils.Validation.DatabaseValidation { - // Excluded from code coverage because it is a wrapper around the file system and testing IO operations is not feasible. - [ExcludeFromCodeCoverage] /// /// Default file system used for database validation process. Contains default implementations of numerous IO operations /// and a function for determining a file's encoding format. /// + // Excluded from code coverage because it is a wrapper around the file system and testing IO operations is not feasible. + [ExcludeFromCodeCoverage] public class LocalFileSystem : IFileSystem { + /// public IEnumerable EnumerateFiles(string path, string searchPattern) { return Directory.EnumerateFiles(path, searchPattern, SearchOption.AllDirectories); } + /// public Stream GetFileStream(string path) { return new FileStream(path, FileMode.Open, FileAccess.Read); } + /// public IEnumerable EnumerateDirectories(string path) { return Directory.EnumerateDirectories(path, "*", SearchOption.AllDirectories); } + /// public string GetFileName(string path) { return Path.GetFileName(path); } + /// public string GetDirectoryName(string path) { string? directory = Path.GetDirectoryName(path); return directory ?? throw new ArgumentException("Path does not contain a directory."); } + /// public Encoding GetEncoding(Stream stream) { PxFileMetadataReader reader = new(); return reader.GetEncoding(stream); } + /// public async Task GetEncodingAsync(Stream stream, CancellationToken cancellationToken) { PxFileMetadataReader reader = new(); diff --git a/Px.Utils/Validation/PxFileValidator.cs b/Px.Utils/Validation/PxFileValidator.cs index aae28c0..70a4988 100644 --- a/Px.Utils/Validation/PxFileValidator.cs +++ b/Px.Utils/Validation/PxFileValidator.cs @@ -1,4 +1,4 @@ -using Px.Utils.PxFile; +using Px.Utils.PxFile; using Px.Utils.Validation.ContentValidation; using Px.Utils.Validation.DatabaseValidation; using Px.Utils.Validation.DataValidation; @@ -67,44 +67,81 @@ public ValidationResult Validate( Encoding? encoding = null, IFileSystem? fileSystem = null) { - encoding ??= new LocalFileSystem().GetEncoding(stream); + ValidationFeedbackSink sink = new(); + ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new ValidationResult(sink.ToFeedback()); + } + + /// + /// Validates the PX file using the specified feedback retention options. + /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The complete PX file validation result with retained feedback. + public ValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options) + { + ValidationFeedbackSink sink = new(options); + ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new ValidationResult(sink.ToFeedback()); + } + + internal void ValidateIntoSink( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink) + { + fileSystem ??= new LocalFileSystem(); + + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = fileSystem.GetEncoding(stream); + stream.Position = originalPosition; + } + conf ??= PxFileConfiguration.Default; - ValidationFeedback feedbacks = []; SyntaxValidator syntaxValidator = new(conf, _customSyntaxValidationFunctions); - SyntaxValidationResult syntaxValidationResult = syntaxValidator.Validate(stream, filename, encoding, fileSystem); - feedbacks.AddRange(syntaxValidationResult.FeedbackItems); + SyntaxValidationOutput syntaxValidationOutput = syntaxValidator.ValidateIntoSink(stream, filename, encoding, fileSystem, sink); - ContentValidator contentValidator = new(filename, encoding, [.. syntaxValidationResult.Result], _customContentValidationFunctions, conf); - ContentValidationResult contentValidationResult = contentValidator.Validate(); - feedbacks.AddRange(contentValidationResult.FeedbackItems); + ContentValidator contentValidator = new(filename, encoding, [.. syntaxValidationOutput.StructuredEntries], _customContentValidationFunctions, conf); + ContentValidationOutput contentValidationOutput = contentValidator.ValidateIntoSink(sink); - if (syntaxValidationResult.DataStartStreamPosition == -1) + if (syntaxValidationOutput.DataStartStreamPosition == -1) { - feedbacks.Add(new( + sink.Report(new KeyValuePair( new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), new(filename, 0, 0) )); - return new (feedbacks); + return; } - stream.Position = syntaxValidationResult.DataStartStreamPosition; + stream.Position = syntaxValidationOutput.DataStartStreamPosition; DataValidator dataValidator = new( - contentValidationResult.DataRowLength, - contentValidationResult.DataRowAmount, - syntaxValidationResult.DataStartRow, + contentValidationOutput.DataRowLength, + contentValidationOutput.DataRowAmount, + syntaxValidationOutput.DataStartRow, conf); - ValidationResult dataValidationResult = dataValidator.Validate(stream, filename, encoding, fileSystem); - feedbacks.AddRange(dataValidationResult.FeedbackItems); + dataValidator.ValidateIntoSink(stream, filename, encoding, fileSystem, sink); if (_customStreamValidators is not null) { foreach (IPxFileStreamValidator customValidator in _customStreamValidators) { ValidationResult customValidationResult = customValidator.Validate(stream, filename, encoding, fileSystem); - feedbacks.AddRange(customValidationResult.FeedbackItems); + sink.ReportRange(customValidationResult.FeedbackItems); } } if (_customValidators is not null) @@ -112,11 +149,10 @@ public ValidationResult Validate( foreach (IValidator customValidator in _customValidators) { ValidationResult customValidationResult = customValidator.Validate(); - feedbacks.AddRange(customValidationResult.FeedbackItems); + sink.ReportRange(customValidationResult.FeedbackItems); } } stream.Close(); - return new ValidationResult(feedbacks); } /// @@ -136,45 +172,85 @@ public async Task ValidateAsync( IFileSystem? fileSystem = null, CancellationToken cancellationToken = default) { - encoding ??= await new LocalFileSystem().GetEncodingAsync(stream, cancellationToken); + ValidationFeedbackSink sink = new(); + await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new ValidationResult(sink.ToFeedback()); + } + + /// + /// Asynchronously validates the PX file using the specified feedback retention options. + /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// A token that cancels the operation. + /// A task that produces the complete PX file validation result with retained feedback. + public async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options, + CancellationToken cancellationToken = default) + { + ValidationFeedbackSink sink = new(options); + await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new ValidationResult(sink.ToFeedback()); + } + + internal async Task ValidateIntoSinkAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink, + CancellationToken cancellationToken = default) + { + fileSystem ??= new LocalFileSystem(); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } + conf ??= PxFileConfiguration.Default; - ValidationFeedback feedbacks = []; SyntaxValidator syntaxValidator = new(conf, _customSyntaxValidationFunctions); - SyntaxValidationResult syntaxValidationResult = await syntaxValidator.ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); - feedbacks.AddRange(syntaxValidationResult.FeedbackItems); + SyntaxValidationOutput syntaxValidationOutput = await syntaxValidator.ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); - ContentValidator contentValidator = new(filename, encoding, [..syntaxValidationResult.Result], _customContentValidationFunctions, conf); - ContentValidationResult contentValidationResult = contentValidator.Validate(); - feedbacks.AddRange(contentValidationResult.FeedbackItems); + ContentValidator contentValidator = new(filename, encoding, [..syntaxValidationOutput.StructuredEntries], _customContentValidationFunctions, conf); + ContentValidationOutput contentValidationOutput = contentValidator.ValidateIntoSink(sink); - if (syntaxValidationResult.DataStartStreamPosition == -1) + if (syntaxValidationOutput.DataStartStreamPosition == -1) { - feedbacks.Add(new( + sink.Report(new KeyValuePair( new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.StartOfDataSectionNotFound), new(filename, 0, 0) )); - return new (feedbacks); + stream.Close(); + return; } - stream.Position = syntaxValidationResult.DataStartStreamPosition; + stream.Position = syntaxValidationOutput.DataStartStreamPosition; DataValidator dataValidator = new( - contentValidationResult.DataRowLength, - contentValidationResult.DataRowAmount, - syntaxValidationResult.DataStartRow, + contentValidationOutput.DataRowLength, + contentValidationOutput.DataRowAmount, + syntaxValidationOutput.DataStartRow, conf); - ValidationResult dataValidationResult = await dataValidator.ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); - feedbacks.AddRange(dataValidationResult.FeedbackItems); + await dataValidator.ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); if (_customStreamAsyncValidators is not null) { foreach (IPxFileStreamValidatorAsync customValidator in _customStreamAsyncValidators) { ValidationResult customValidationResult = await customValidator.ValidateAsync(stream, filename, encoding, fileSystem, cancellationToken); - feedbacks.AddRange(customValidationResult.FeedbackItems); + sink.ReportRange(customValidationResult.FeedbackItems); } } if (_customAsyncValidators is not null) @@ -182,11 +258,10 @@ public async Task ValidateAsync( foreach (IValidatorAsync customValidator in _customAsyncValidators) { ValidationResult customValidationResult = await customValidator.ValidateAsync(cancellationToken); - feedbacks.AddRange(customValidationResult.FeedbackItems); + sink.ReportRange(customValidationResult.FeedbackItems); } } stream.Close(); - return new ValidationResult(feedbacks); } } } diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationOutput.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationOutput.cs new file mode 100644 index 0000000..9bcb58d --- /dev/null +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationOutput.cs @@ -0,0 +1,7 @@ +namespace Px.Utils.Validation.SyntaxValidation +{ + internal sealed record SyntaxValidationOutput( + List StructuredEntries, + int DataStartRow, + long DataStartStreamPosition); +} \ No newline at end of file diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationResult.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationResult.cs index cbb17da..dc03775 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationResult.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationResult.cs @@ -1,4 +1,4 @@ -namespace Px.Utils.Validation.SyntaxValidation +namespace Px.Utils.Validation.SyntaxValidation { /// /// Represents the result of a syntax validation operation. This struct contains a validation report and a list of structured validation entries. @@ -6,14 +6,24 @@ /// A dictionary of amd objects produced by the syntax validation operation. /// A list of objects produced by the syntax validation operation. /// The row number where the data section starts in the file. - /// The stream position where the data section starts in the file. - public class SyntaxValidationResult(ValidationFeedback feedbacks, List result, int dataStartRow, int dataStartStreamPosition) : ValidationResult(feedbacks) + /// The absolute raw byte offset of the first non-whitespace data value after the DATA keyword, + /// suitable for direct assignment to ; otherwise -1 when the DATA keyword or a value cannot be found. + public class SyntaxValidationResult(ValidationFeedback feedbacks, List result, int dataStartRow, long dataStartStreamPosition) : ValidationResult(feedbacks) { /// /// Gets the list of objects produced by the syntax validation operation. /// public List Result { get; } = result; + + /// + /// Gets the row number where the data section starts in the file. + /// public int DataStartRow { get; } = dataStartRow; + + /// + /// Gets the absolute raw byte offset of the first non-whitespace data value after the DATA keyword. + /// The value is suitable for direct assignment to and is -1 when the DATA keyword or a value cannot be found. + /// public long DataStartStreamPosition { get; } = dataStartStreamPosition; } } diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs index 035b050..fb4fe05 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs @@ -44,6 +44,7 @@ public static class SyntaxValidationUtilityMethods /// /// The input string to extract from /// Symbol that starts enclosement + /// Symbol that encloses a string /// Optional symbol that closes the enclosement. If none given, startSymbol is used for both starting and ending the enclosement /// Returns an object that contains the extracted sections, /// the string that remains after the operation and starting indexes of extracted sections diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs index 5528fb9..91a6a58 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidator.cs @@ -1,4 +1,5 @@ -using Px.Utils.PxFile; +using Px.Utils.PxFile; +using Px.Utils.PxFile.Data; using Px.Utils.Validation.DatabaseValidation; using System.Runtime.CompilerServices; using System.Text; @@ -8,10 +9,9 @@ namespace Px.Utils.Validation.SyntaxValidation /// /// Provides methods for validating the syntax of a PX file. Validation can be done using both synchronous and asynchronous methods. /// Additionally custom validation functions can be provided to be used during validation. + /// /// Object that stores syntax specific symbols and tokens for the PX file /// Object that contains any optional additional validation functions - /// his is required if multiple validations are executed for the same stream. - /// public class SyntaxValidator( PxFileConfiguration? conf = null, CustomSyntaxValidationFunctions? customValidationFunctions = null) @@ -19,7 +19,7 @@ public class SyntaxValidator( { private const int _bufferSize = 4096; private int _dataSectionStartRow = -1; - private int _dataSectionStartStreamPosition = -1; + private long _dataSectionStartStreamPosition = -1; /// /// Validates the syntax of a PX file's metadata. @@ -35,9 +35,47 @@ public SyntaxValidationResult Validate( string filename, Encoding? encoding = null, IFileSystem? fileSystem = null) + { + ValidationFeedbackSink sink = new(); + SyntaxValidationOutput output = ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new SyntaxValidationResult(sink.ToFeedback(), output.StructuredEntries, output.DataStartRow, output.DataStartStreamPosition); + } + + /// + /// Validates the syntax of a PX file's metadata using the specified feedback retention options. + /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// The syntax result with retained validation feedback and parsed metadata entries. + public SyntaxValidationResult Validate( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options) + { + ValidationFeedbackSink sink = new(options); + SyntaxValidationOutput output = ValidateIntoSink(stream, filename, encoding, fileSystem, sink); + return new SyntaxValidationResult(sink.ToFeedback(), output.StructuredEntries, output.DataStartRow, output.DataStartStreamPosition); + } + + internal SyntaxValidationOutput ValidateIntoSink( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink) { fileSystem ??= new LocalFileSystem(); - encoding ??= fileSystem.GetEncoding(stream); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = fileSystem.GetEncoding(stream); + stream.Position = originalPosition; + } SyntaxValidationFunctions validationFunctions = new(); IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; @@ -52,16 +90,17 @@ public SyntaxValidationResult Validate( } conf ??= PxFileConfiguration.Default; + ResetDataSectionPosition(); + _dataSectionStartStreamPosition = StreamUtilities.FindDataStartPosition(stream, conf, _bufferSize); - ValidationFeedback validationFeedbacks = []; List stringEntries = BuildValidationEntries(stream, encoding, conf, filename, _bufferSize); - validationFeedbacks.AddRange(ValidateEntries(stringEntries, stringValidationFunctions, conf)); + ReportEntryFeedback(stringEntries, stringValidationFunctions, conf, sink); List keyValuePairs = BuildKeyValuePairs(stringEntries, conf); - validationFeedbacks.AddRange(ValidateKeyValuePairs(keyValuePairs, keyValueValidationFunctions, conf)); + ReportKeyValuePairFeedback(keyValuePairs, keyValueValidationFunctions, conf, sink); List structuredEntries = BuildValidationStructureEntries(keyValuePairs, conf); - validationFeedbacks.AddRange(ValidateStructs(structuredEntries, structuredValidationFunctions, conf)); + ReportStructuredFeedback(structuredEntries, structuredValidationFunctions, conf, sink); - return new SyntaxValidationResult(validationFeedbacks, structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); + return new SyntaxValidationOutput(structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); } /// @@ -80,9 +119,50 @@ public async Task ValidateAsync( Encoding? encoding = null, IFileSystem? fileSystem = null, CancellationToken cancellationToken = default) + { + ValidationFeedbackSink sink = new(); + SyntaxValidationOutput output = await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new SyntaxValidationResult(sink.ToFeedback(), output.StructuredEntries, output.DataStartRow, output.DataStartStreamPosition); + } + + /// + /// Asynchronously validates the syntax of a PX file's metadata using the specified feedback retention options. + /// + /// The PX file stream to validate. + /// The name used in reported feedback. + /// The PX file encoding, or to detect it. + /// The file system used for encoding detection, or for the default. + /// Feedback retention options. A positive limit applies per filename, level, and rule; limit retains all feedback. + /// A token that cancels the operation. + /// A task that produces the syntax result with retained validation feedback and parsed metadata entries. + public async Task ValidateAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationOptions options, + CancellationToken cancellationToken = default) + { + ValidationFeedbackSink sink = new(options); + SyntaxValidationOutput output = await ValidateIntoSinkAsync(stream, filename, encoding, fileSystem, sink, cancellationToken); + return new SyntaxValidationResult(sink.ToFeedback(), output.StructuredEntries, output.DataStartRow, output.DataStartStreamPosition); + } + + internal async Task ValidateIntoSinkAsync( + Stream stream, + string filename, + Encoding? encoding, + IFileSystem? fileSystem, + ValidationFeedbackSink sink, + CancellationToken cancellationToken = default) { fileSystem ??= new LocalFileSystem(); - encoding ??= await fileSystem.GetEncodingAsync(stream, cancellationToken); + if (encoding is null) + { + long originalPosition = stream.Position; + encoding = await fileSystem.GetEncodingAsync(stream, cancellationToken); + stream.Position = originalPosition; + } SyntaxValidationFunctions validationFunctions = new(); IEnumerable stringValidationFunctions = validationFunctions.DefaultStringValidationFunctions; @@ -97,15 +177,16 @@ public async Task ValidateAsync( } conf ??= PxFileConfiguration.Default; - ValidationFeedback validationFeedbacks = []; + ResetDataSectionPosition(); + _dataSectionStartStreamPosition = await StreamUtilities.FindDataStartPositionAsync(stream, conf, _bufferSize, cancellationToken); List entries = await BuildValidationEntriesAsync(stream, encoding, conf, filename, _bufferSize, cancellationToken); - validationFeedbacks.AddRange(ValidateEntries(entries, stringValidationFunctions, conf)); + ReportEntryFeedback(entries, stringValidationFunctions, conf, sink); List keyValuePairs = BuildKeyValuePairs(entries, conf); - validationFeedbacks.AddRange(ValidateKeyValuePairs(keyValuePairs, keyValueValidationFunctions, conf)); + ReportKeyValuePairFeedback(keyValuePairs, keyValueValidationFunctions, conf, sink); List structuredEntries = BuildValidationStructureEntries(keyValuePairs, conf); - validationFeedbacks.AddRange(ValidateStructs(structuredEntries, structuredValidationFunctions, conf)); + ReportStructuredFeedback(structuredEntries, structuredValidationFunctions, conf, sink); - return new SyntaxValidationResult(validationFeedbacks, structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); + return new SyntaxValidationOutput(structuredEntries, _dataSectionStartRow, _dataSectionStartStreamPosition); } #region Interface implementation @@ -143,9 +224,12 @@ public static bool IsEndOfMetadataSection(char currentCharacter, PxFileConfigura return false; } - private static ValidationFeedback ValidateEntries(IEnumerable entries, IEnumerable validationFunctions, PxFileConfiguration syntaxConf) + private static void ReportEntryFeedback( + IEnumerable entries, + IEnumerable validationFunctions, + PxFileConfiguration syntaxConf, + ValidationFeedbackSink sink) { - ValidationFeedback validationFeedback = []; foreach (ValidationEntry entry in entries) { foreach (EntryValidationFunction function in validationFunctions) @@ -153,19 +237,18 @@ private static ValidationFeedback ValidateEntries(IEnumerable e KeyValuePair? feedback = function(entry, syntaxConf); if (feedback is not null) { - validationFeedback.Add((KeyValuePair )feedback); + sink.Report(feedback.Value); } } } - return validationFeedback; } - private static ValidationFeedback ValidateKeyValuePairs( + private static void ReportKeyValuePairFeedback( IEnumerable kvpObjects, IEnumerable validationFunctions, - PxFileConfiguration syntaxConf) + PxFileConfiguration syntaxConf, + ValidationFeedbackSink sink) { - ValidationFeedback validationFeedback = []; foreach (ValidationKeyValuePair kvpObject in kvpObjects) { foreach (KeyValuePairValidationFunction function in validationFunctions) @@ -173,19 +256,18 @@ private static ValidationFeedback ValidateKeyValuePairs( KeyValuePair? feedback = function(kvpObject, syntaxConf); if (feedback is not null) { - validationFeedback.Add((KeyValuePair)feedback); + sink.Report(feedback.Value); } } } - return validationFeedback; } - private static ValidationFeedback ValidateStructs( - IEnumerable structuredEntries, + private static void ReportStructuredFeedback( + IEnumerable structuredEntries, IEnumerable validationFunctions, - PxFileConfiguration syntaxConf) + PxFileConfiguration syntaxConf, + ValidationFeedbackSink sink) { - ValidationFeedback validationFeedback = []; foreach (ValidationStructuredEntry structuredEntry in structuredEntries) { foreach (StructuredValidationFunction function in validationFunctions) @@ -193,11 +275,10 @@ private static ValidationFeedback ValidateStructs( KeyValuePair? feedback = function(structuredEntry, syntaxConf); if (feedback is not null) { - validationFeedback.Add((KeyValuePair)feedback); + sink.Report(feedback.Value); } } } - return validationFeedback; } private static List BuildKeyValuePairs(List validationEntries, PxFileConfiguration syntaxConf) @@ -251,8 +332,6 @@ private List BuildValidationEntries(Stream stream, Encoding enc if (IsEndOfMetadataSection(buffer[i], syntaxConf, entryBuilder, isProcessingString)) { _dataSectionStartRow = lineChangeIndexes.Count; - // This here should find the actual start of the data section, after line changes, spaces and whatnot. - _dataSectionStartStreamPosition = characterIndex + 1; return entries; } UpdateLineAndCharacter(buffer[i], syntaxConf, ref characterIndex, ref lineChangeIndexes, ref isProcessingString); @@ -311,7 +390,6 @@ private async Task> BuildValidationEntriesAsync( if (IsEndOfMetadataSection(buffer[i], syntaxConf, entryBuilder, isProcessingString)) { _dataSectionStartRow = lineChangeIndexes.Count; - _dataSectionStartStreamPosition = characterIndex + 1; return entries; } UpdateLineAndCharacter(buffer[i], syntaxConf, ref characterIndex, ref lineChangeIndexes, ref isProcessingString); @@ -335,6 +413,12 @@ private async Task> BuildValidationEntriesAsync( return entries; } + private void ResetDataSectionPosition() + { + _dataSectionStartRow = -1; + _dataSectionStartStreamPosition = -1; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void UpdateLineAndCharacter(char currentCharacter, PxFileConfiguration syntaxConf, ref int characterIndex, ref List linebreakIndexes, ref bool isProcessingString) { diff --git a/Px.Utils/Validation/SyntaxValidation/ValidationStructuredEntry.cs b/Px.Utils/Validation/SyntaxValidation/ValidationStructuredEntry.cs index a6e4878..32f3d9b 100644 --- a/Px.Utils/Validation/SyntaxValidation/ValidationStructuredEntry.cs +++ b/Px.Utils/Validation/SyntaxValidation/ValidationStructuredEntry.cs @@ -1,4 +1,4 @@ -namespace Px.Utils.Validation.SyntaxValidation +namespace Px.Utils.Validation.SyntaxValidation { /// /// Represents a key for a . A key consists of a keyword and two optional language and specifier strings. @@ -50,6 +50,7 @@ public ValidationStructuredEntryKey(string keyword, string? language = null, str /// Index of the line where the entry starts. /// Character indexes of the line changes in the entry starting from the entry start. /// Index of the first character of the value in the entry. + /// Value type of the value part of the entry, if found. public class ValidationStructuredEntry( string file, ValidationStructuredEntryKey key, diff --git a/Px.Utils/Validation/ValidationFeedbackSink.cs b/Px.Utils/Validation/ValidationFeedbackSink.cs new file mode 100644 index 0000000..1947563 --- /dev/null +++ b/Px.Utils/Validation/ValidationFeedbackSink.cs @@ -0,0 +1,104 @@ +using System.Collections.Concurrent; + +namespace Px.Utils.Validation +{ + internal sealed class ValidationFeedbackSink + { + private const int DEFAULT_MAX_FEEDBACK_ITEMS_PER_SIGNATURE = 100; + private readonly ConcurrentDictionary _buckets = new(ValidationFeedbackSignatureComparer.Instance); + private readonly int? _maxFeedbackItemsPerSignature; + + public ValidationFeedbackSink(ValidationOptions? options = null) + { + _maxFeedbackItemsPerSignature = options is null + ? DEFAULT_MAX_FEEDBACK_ITEMS_PER_SIGNATURE + : options.MaxFeedbackItemsPerSignature; + if (_maxFeedbackItemsPerSignature is <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), "The maximum number of feedback items per signature must be positive or unlimited."); + } + } + + public void Report(KeyValuePair feedback) + => Report(feedback.Key, feedback.Value); + + public void Report(ValidationFeedbackKey key, ValidationFeedbackValue value) + { + ValidationFeedbackSignature signature = new(value.Filename, key.Level, key.Rule); + FeedbackBucket bucket = _buckets.GetOrAdd(signature, static _ => new FeedbackBucket()); + + lock (bucket) + { + if (_maxFeedbackItemsPerSignature is null || bucket.Values.Count < _maxFeedbackItemsPerSignature.Value) + { + bucket.Values.Add(value); + return; + } + + if (!bucket.IsTruncated) + { + ValidationFeedbackValue finalValue = bucket.Values[^1]; + string truncationNote = $"Feedback limit of {_maxFeedbackItemsPerSignature.Value} instances for this file, level, and rule was reached. " + + $"Additional instances were detected but not logged."; + string additionalInfo = string.IsNullOrEmpty(finalValue.AdditionalInfo) + ? truncationNote + : $"{finalValue.AdditionalInfo}{Environment.NewLine}{truncationNote}"; + bucket.Values[^1] = new ValidationFeedbackValue(finalValue.Filename, finalValue.Line, finalValue.Character, additionalInfo); + bucket.IsTruncated = true; + } + } + } + + public void ReportRange(ValidationFeedback feedback) + { + foreach (KeyValuePair> feedbackGroup in feedback) + { + foreach (ValidationFeedbackValue value in feedbackGroup.Value) + { + Report(feedbackGroup.Key, value); + } + } + } + + public ValidationFeedback ToFeedback() + { + ValidationFeedback feedback = []; + foreach (KeyValuePair pair in _buckets) + { + lock (pair.Value) + { + ValidationFeedbackKey key = new(pair.Key.Level, pair.Key.Rule); + if (!feedback.TryGetValue(key, out List? values)) + { + values = []; + feedback[key] = values; + } + + values.AddRange(pair.Value.Values); + } + } + + return feedback; + } + + private sealed class FeedbackBucket + { + public List Values { get; } = []; + + public bool IsTruncated { get; set; } + } + } + + internal readonly record struct ValidationFeedbackSignature(string Filename, ValidationFeedbackLevel Level, ValidationFeedbackRule Rule); + + internal sealed class ValidationFeedbackSignatureComparer : IEqualityComparer + { + public static ValidationFeedbackSignatureComparer Instance { get; } = new(); + + public bool Equals(ValidationFeedbackSignature x, ValidationFeedbackSignature y) + => x.Level == y.Level && x.Rule == y.Rule && StringComparer.Ordinal.Equals(x.Filename, y.Filename); + + public int GetHashCode(ValidationFeedbackSignature obj) + => HashCode.Combine(StringComparer.Ordinal.GetHashCode(obj.Filename), obj.Level, obj.Rule); + } +} diff --git a/Px.Utils/Validation/ValidationOptions.cs b/Px.Utils/Validation/ValidationOptions.cs new file mode 100644 index 0000000..3f530a2 --- /dev/null +++ b/Px.Utils/Validation/ValidationOptions.cs @@ -0,0 +1,18 @@ +namespace Px.Utils.Validation +{ + /// + /// Configures validation feedback retention. + /// + public sealed class ValidationOptions + { + /// + /// Gets a configuration that retains every feedback item. + /// + public static ValidationOptions Unlimited { get; } = new() { MaxFeedbackItemsPerSignature = null }; + + /// + /// Gets or initializes the maximum number of feedback items retained for each filename, level, and rule signature. A null value retains all feedback items. + /// + public int? MaxFeedbackItemsPerSignature { get; init; } = 100; + } +} diff --git a/docs/PXFILE_SPECIFICATION.md b/docs/PXFILE_SPECIFICATION.md index dc75d5b..24f6c3b 100644 --- a/docs/PXFILE_SPECIFICATION.md +++ b/docs/PXFILE_SPECIFICATION.md @@ -2,6 +2,12 @@ Purpose of this document is to describe the syntax and content requirements for the pxfiles supported by this library. These requirements do not necessarily apply to other implementations of the pxfile format. +## Supported text encodings + +Supported PX file encodings are ASCII-compatible single-byte encodings, including ANSI code pages declared through `CODEPAGE`, and UTF-8. UTF-8 files may include a BOM or omit it, and metadata strings may contain multibyte UTF-8 characters. + +UTF-16 and UTF-32 are not supported, regardless of byte order or BOM presence. + ## Syntax ### Entries diff --git a/docs/README.md b/docs/README.md index 3114422..084dfc6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -41,6 +41,12 @@ The read pipeline consists of the following components: Reading the metadata, bu Each of these components can be used separately and replaced with custom implementations. Especially if the contents of your px-files do not follow the standard px-file format, you might need to implement your own metadata builder. +### Supported text encodings + +Px.Utils supports ASCII-compatible single-byte encodings, including ANSI code pages specified through `CODEPAGE`, and UTF-8. UTF-8 files may be provided with or without a BOM. UTF-8 metadata may contain multibyte characters. + +UTF-16 and UTF-32, in either byte order and with or without a BOM, are not supported for full PX file reading or validation. + #### PxFileMetadataReader : IPxFileMetadataReader ```ReadMetadata(Stream stream, Encoding encoding)``` reads the metadata entries from the provided stream as a IEnumerable of ```KeyValuePair``` representing the keys and values of the entries. **This method does not perform any validation on the metadata entries.** @@ -57,6 +63,10 @@ The entries need to be in the same key-value format as the output of the ```PxFi **IMPORTANT!** The target map must have the same order as the complete file map. This is for performance reasons, we do not want to move back and forth in the file or generate a second indexer for placing the data in the buffer. +When the reader is created at stream position `0`, it finds the first non-whitespace data value after the top-level `DATA=` entry automatically. This also works with a UTF-8 BOM and multibyte metadata. The overload that accepts `dataStart` expects the absolute raw byte offset of that first value; use `StreamUtilities.FindDataStartPosition()` or `FindDataStartPositionAsync()` to obtain the offset from a seekable stream. Both helpers restore the original stream position and return `-1` when no data value is found, including an explicitly empty entry such as `DATA=;`. + +`StreamUtilities.FindKeywordPosition()` and `FindKeywordPositionAsync()` locate a specified keyword at the start of a top-level PX entry and return that keyword's raw byte offset. Use `FindDataStartPosition` or `FindDataStartPositionAsync` when locating the first data value after `DATA=`. Unlike the DATA-specific helpers, the generic keyword methods search from the stream's current position and leave it advanced after reading. + ### Metadata example ```csharp // Read meta @@ -225,6 +235,14 @@ Validator classes implement either ```IPxFileStreamValidator``` or ```IPxFileStr - encoding (Encoding, optional): Encoding of the px file. Default is Encoding.Default - fileSystem (IFileSystem, optional): Object that defines the file system used for the validation process. Default file called LocalFileSystem system is used if none provided. +#### Feedback retention +All concrete validators provide overloads that accept a `ValidationOptions` instance. By default, validation retains at most 100 feedback items for each filename, feedback level, and rule combination. When a limit is reached, the final retained item is annotated to indicate that additional matching feedback was detected but not logged. Set `MaxFeedbackItemsPerSignature` to a positive number to choose a limit, or use `ValidationOptions.Unlimited` to retain every item. + +```csharp +ValidationOptions options = new() { MaxFeedbackItemsPerSignature = 500 }; +ValidationResult result = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8, null, options); +``` + #### PxFileValidator : IPxFileStreamValidator, IPxFileStreamValidatorAsync ```PxFileValidator``` is a class that validates the whole px file including its data, metadata syntax and metadata contents. The class can be instantiated with the following parameters: - conf (PxFileConfiguration, optional): Object that contains px file configuration. @@ -236,6 +254,7 @@ Once the PxFileValidator object is instantiated, either the Validate or Validate PxFileValidator validator = new PxFileValidator(); ValidationResult result = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8); ValidationResult asyncResult = await validator.ValidateAsync(fileStream, "path/to/file.px", Encoding.UTF8, cancellationToken: cancellationToken); + ValidationResult limitedResult = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8, null, new ValidationOptions { MaxFeedbackItemsPerSignature = 500 }); ``` #### SyntaxValidator : IPxFileStreamValidator, IPxFileStreamValidatorAsync @@ -249,6 +268,7 @@ The class can be instantiated with the following parameters: SyntaxValidator validator = new SyntaxValidator(); SyntaxValidationResult result = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8); SyntaxValidationResult asyncResult = await validator.ValidateAsync(fileStream, "path/to/file.px", Encoding.UTF8, cancellationToken: cancellationToken); + SyntaxValidationResult limitedResult = validator.Validate(fileStream, "path/to/file.px", Encoding.UTF8, null, ValidationOptions.Unlimited); ``` #### ContentValidator : IValidator @@ -267,6 +287,7 @@ The class can be instantiated with the following parameters: SyntaxValidationResult syntaxResult = syntaxValidator.Validate(fileStream, "path/to/file.px", encoding); ContentValidator validator = new ContentValidator("path/to/file.px", encoding, syntaxResult.Result); ValidationResult result = validator.Validate(); + ValidationResult limitedResult = validator.Validate(new ValidationOptions { MaxFeedbackItemsPerSignature = 500 }); ``` #### DataValidator : IPxFileStreamValidator, IPxFileStreamValidatorAsync @@ -286,6 +307,7 @@ The class can be instantiated with the following parameters: ValidationResult contentResult = contentValidator.Validate(); DataValidator validator = new DataValidator(contentResult.DataRowLength, contentResult.DataRowAmount, syntaxResult.DataStartRow); ValidationResult result = validator.Validate(fileStream, "path/to/file.px", encoding); + ValidationResult limitedResult = validator.Validate(fileStream, "path/to/file.px", encoding, null, new ValidationOptions { MaxFeedbackItemsPerSignature = 500 }); ``` #### DatabaseValidator : IValidator, IValidatorAsync @@ -305,6 +327,7 @@ The database needs to contain alias files for each language used in the database DatabaseValidator validator = new DatabaseValidator("path/to/database"); ValidationResult result = validator.Validate(); ValidationResult asyncResult = await validator.ValidateAsync(cancellationToken); + ValidationResult limitedResult = validator.Validate(new ValidationOptions { MaxFeedbackItemsPerSignature = 500 }); ``` ### Computing diff --git a/docs/architecture.readers.md b/docs/architecture.readers.md index 635b2e8..aac5ae5 100644 --- a/docs/architecture.readers.md +++ b/docs/architecture.readers.md @@ -32,13 +32,15 @@ void ReadDecimalDataValues(DecimalDataValue[] buffer, int offset, IMatrixMap tar Async variants available. Implements `IDisposable`. Depends on: `PxFileConfiguration`. +When constructed at stream position `0`, `PxFileStreamDataReader` uses an internal trusted-input locator to find the first non-whitespace value following the top-level `DATA=` entry. This path retains quote-aware top-level matching but assumes that the PX syntax was validated before reading. The overload that accepts `dataStart` requires that value's absolute raw byte offset and avoids a metadata rescan; use a validated `SyntaxValidationResult.DataStartStreamPosition` when one is available. + ### Helpers | File | Purpose | |---|---| | `DataIndexer.cs` | Index mapping between source and target `IMatrixMap` | | `DataValueParsers.cs` | Parse raw data values from byte spans | -| `StreamUtilities.cs` | Stream helper methods | +| `StreamUtilities.cs` | Public `FindDataStartPosition` and async equivalent are validation-aware: they search from origin, return the first value after top-level `DATA=` as an absolute raw byte offset, and restore the original position of a seekable stream. Internal `FindDataStartPositionUnchecked` equivalents are for validated reader input, search from the current position, and leave the stream advanced. `FindKeywordPosition` and async equivalent locate quote-aware top-level entry keywords from the current position and leave the stream advanced. | ## Binary Data diff --git a/docs/architecture.testing.md b/docs/architecture.testing.md index b3305d6..fbcd94e 100644 --- a/docs/architecture.testing.md +++ b/docs/architecture.testing.md @@ -18,6 +18,7 @@ Framework: MSTest + Moq. Naming: `MethodNameStateUnderTestExpectedBehavior`. | `DataStringValueValidatorTests` | `Validation/DataValidationTests/DataStringValueValidatorTests.cs` | String data validation | | `DataSeparatorValidatorTest` | `Validation/DataValidationTests/DataSeparatorValidatorTest.cs` | Separator validation | | `DataStructureValidationTests` | `Validation/DataValidationTests/DataStructureValidationTests.cs` | Data structure validation | +| `ValidationFeedbackSinkTests` | `Validation/ValidationFeedbackSinkTests.cs` | Per-signature feedback limits, truncation annotations, unlimited retention, and invalid limits | | `DatabaseValidatorTests` | `Validation/DatabaseValidation/DatabaseValidatorTests.cs` | `DatabaseValidator` | | `DatabaseValidatorFunctionTests` | `Validation/DatabaseValidation/DatabaseValidatorFunctionTests.cs` | Database validator functions | | `PxFileValidationTests` | `Validation/PxFileValidationTests/PxFileValidationTests.cs` | `PxFileValidator` | @@ -34,6 +35,7 @@ Framework: MSTest + Moq. Naming: `MethodNameStateUnderTestExpectedBehavior`. | `MultiPartReadingTests` | `PxFileTests/DataTests/PxFileStreamDataReaderTests/MultiPartReadingTests.cs` | Multi-part reads | | `DataIndexerTests` | `PxFileTests/DataTests/DataIndexerTests.cs` | `DataIndexer` | | `DataValueParserTests` | `PxFileTests/DataTests/DataValueParserTests.cs` | `DataValueParsers` | +| `StreamUtilitiesTests` | `PxFileTests/DataTests/StreamUtilitiesTests.cs` | Byte-accurate `DATA=` start offsets with BOM, multibyte metadata, whitespace, buffer splits, and missing data | ### Model & Builder Tests diff --git a/docs/architecture.validation.md b/docs/architecture.validation.md index 84736d0..e2f0fe3 100644 --- a/docs/architecture.validation.md +++ b/docs/architecture.validation.md @@ -22,12 +22,16 @@ Validate(stream, filename, encoding?, fileSystem?) ValidateAsync(stream, filename, encoding?, fileSystem?, cancellationToken) ``` +Concrete validators also expose `ValidationOptions` overloads. `ValidationOptions.MaxFeedbackItemsPerSignature` defaults to `100` and limits retained feedback by filename, level, and rule; set it to `null` through `ValidationOptions.Unlimited` to retain all feedback. Validators report each discovered finding directly to `ValidationFeedbackSink`, so configured limits bound retained feedback during syntax, content, and data scans instead of only truncating completed results. When a limit is exceeded, the last retained item is annotated with a truncation notice. `ValidationFeedbackSink` applies this policy safely while database validation processes files concurrently. + ### SyntaxValidator Validates PX file metadata syntax (key-value structure, encoding, characters). File: `Validation/SyntaxValidation/SyntaxValidator.cs` Partial helpers: `SyntaxValidationFunctions.StringValidationFunctions.cs`, `KeyValueValidationFunctions.cs`, `StructuredValidationFunctions.cs` +Before parsing metadata, it locates the first non-whitespace value after the top-level `DATA=` entry using `StreamUtilities`. `SyntaxValidationResult.DataStartStreamPosition` is the resulting absolute raw byte offset, suitable for direct assignment to `Stream.Position`, or `-1` if no data value is found. An empty entry such as `DATA=;` is treated as missing data and produces error-level `StartOfDataSectionNotFound` feedback in file and standalone data validation. + ### ContentValidator Validates metadata content (required keys, language definitions, dimension consistency). @@ -41,6 +45,8 @@ Dimension type validation uses `PxFileConfiguration.TokenDefinitions.VariableTyp Validates data section (row counts, row lengths, value types, separators). File: `Validation/DataValidation/DataValidator.cs` +When invoked at the start of a stream, it uses its supplied `PxFileConfiguration` to locate the configured DATA keyword and syntax separators. + ### DatabaseValidator Validates entire PX database directory (all `.px` files, alias files, directory structure). @@ -93,6 +99,8 @@ Validation/ ├── IPxFileStreamValidator.cs -- IPxFileStreamValidator, IPxFileStreamValidatorAsync ├── IValidationResult.cs -- ValidationResult ├── ValidationFeedback.cs -- ValidationFeedbackKey, ValidationFeedbackValue, ValidationFeedback +├── ValidationOptions.cs -- Feedback retention configuration +├── ValidationFeedbackSink.cs -- Concurrent feedback retention and truncation ├── ValidationObject.cs -- Validation context ├── Enums.cs -- ValidationFeedbackLevel, ValidationFeedbackRule, ValueType ├── PxFileValidator.cs -- Orchestrator