diff --git a/DemUtility/DemUtility.csproj b/DemUtility/DemUtility.csproj index f4e5935..538cd2c 100644 --- a/DemUtility/DemUtility.csproj +++ b/DemUtility/DemUtility.csproj @@ -2,13 +2,14 @@ Exe - net8.0 + net10.0 enable dem - + + diff --git a/DemUtility/Program.cs b/DemUtility/Program.cs index cb3d5c2..37867ee 100644 --- a/DemUtility/Program.cs +++ b/DemUtility/Program.cs @@ -1,80 +1,131 @@ using System; using System.Collections.Generic; +using System.CommandLine; using System.IO; using System.IO.Compression; +using System.Linq; using System.Text.Json; using System.Threading.Tasks; -using CommandLine; using Pmad.Cartography; using Pmad.Cartography.Databases; using Pmad.Cartography.DataCells; +using Pmad.ProgressTracking; namespace DemUtility { - [Verb("repack", HelpText = "Create a copy of a DEM database with specified compresssion.")] - internal class RepackOptions + internal class Program { - [Option('s', "source", Required = true, HelpText = "Source directory.")] - public string? Source { get; set; } + static int Main(string[] args) + { + var rootCommand = new RootCommand("DEM database utility."); - [Option('t', "target", Required = true, HelpText = "Target directory.")] - public string? Target { get; set; } + // repack + var repackSourceOption = new Option("--source", "-s") { Description = "Source directory.", Required = true }; + var repackTargetOption = new Option("--target", "-t") { Description = "Target directory.", Required = true }; + var repackCompressionOption = new Option("--compression", "-c") { Description = "Compression to use: 'GZip', 'ZSTD', 'Brotli', or 'None' (ZSTD by default).", DefaultValueFactory = _ => Compression.ZSTD }; + var repackMaxCpuOption = new Option("--max-cpu", "-m") { Description = "Number of CPU Cores that can be used for process.", DefaultValueFactory = _ => -1 }; + var repackKeepOption = new Option("--keep", "-k") { Description = "Keep existing files." }; - [Option('c', "compression", Required = false, HelpText = "Compression to use: 'GZip', 'ZSTD', 'Brotli', or 'None' (ZSTD by default).")] - public Compression TargetCompression { get; set; } = Compression.ZSTD; + var repackCommand = new Command("repack", "Create a copy of a DEM database with specified compression."); + repackCommand.Options.Add(repackSourceOption); + repackCommand.Options.Add(repackTargetOption); + repackCommand.Options.Add(repackCompressionOption); + repackCommand.Options.Add(repackMaxCpuOption); + repackCommand.Options.Add(repackKeepOption); + repackCommand.SetAction(parseResult => + { + using var render = ConsoleProgessHelper.Create(); + return Repack( + parseResult.GetValue(repackSourceOption)!, + parseResult.GetValue(repackTargetOption)!, + parseResult.GetValue(repackCompressionOption), + parseResult.GetValue(repackMaxCpuOption), + parseResult.GetValue(repackKeepOption), + render); + }); - [Option('m', "max-cpu", Required = false, HelpText = "Number of CPU Cores that can be used for process.")] - public int MaxCPU { get; set; } = -1; + // index + var indexPathOption = new Option("--path", "-p") { Description = "Database directory.", Required = true }; - [Option('k', "keep", Required = false, HelpText = "Keep existing files.")] - public bool Keep { get; set; } - } + var indexCommand = new Command("index", "Build index."); + indexCommand.Options.Add(indexPathOption); + indexCommand.SetAction(async parseResult => + { + using var render = ConsoleProgessHelper.Create(); + return await Index(parseResult.GetValue(indexPathOption)!, render); + }); - [Verb("index", HelpText = "Build index.")] - internal class IndexOptions - { - [Option('p', "path", Required = true, HelpText = "Database directory.")] - public string? Source { get; set; } - } + // update-index + var updateIndexPathOption = new Option("--path", "-p") { Description = "Database directory.", Required = true }; - internal class Program - { - static int Main(string[] args) - { - return CommandLine.Parser.Default.ParseArguments(args) - .MapResult( - (RepackOptions opts) => Repack(opts), - (IndexOptions opts) => Index(opts), - errs => 1); + var updateIndexCommand = new Command("update-index", "Add missing SHA-256 checksums to an existing index file."); + updateIndexCommand.Options.Add(updateIndexPathOption); + updateIndexCommand.SetAction(async parseResult => + { + return await UpdateIndex(parseResult.GetValue(updateIndexPathOption)!); + }); + + // check + var checkPathOption = new Option("--path", "-p") { Description = "Local database directory." }; + var checkUrlOption = new Option("--url", "-u") { Description = "HTTP base URL of the database." }; + var checkVerifyOption = new Option("--verify", "-v") { Description = "Verify SHA-256 checksums of files (local or downloaded via HTTP)." }; + var checkSampleOption = new Option("--sample", "-n") { Description = "Number of randomly sampled cells to verify (verifies all cells when omitted)." }; + + var checkCommand = new Command("check", "Check a DEM database (local or HTTP) and report its content."); + checkCommand.Options.Add(checkPathOption); + checkCommand.Options.Add(checkUrlOption); + checkCommand.Options.Add(checkVerifyOption); + checkCommand.Options.Add(checkSampleOption); + checkCommand.SetAction(async parseResult => + { + var path = parseResult.GetValue(checkPathOption); + var url = parseResult.GetValue(checkUrlOption); + var verify = parseResult.GetValue(checkVerifyOption); + var sample = parseResult.GetValue(checkSampleOption); + if (path == null && url == null) + { + Console.Error.WriteLine("Either --path or --url must be specified."); + return 1; + } + if (path != null && url != null) + { + Console.Error.WriteLine("Only one of --path or --url can be specified."); + return 1; + } + if (sample.HasValue && sample.Value <= 0) + { + Console.Error.WriteLine("--sample must be a positive integer."); + return 1; + } + return await Check(path, url, verify, sample); + }); + + rootCommand.Subcommands.Add(repackCommand); + rootCommand.Subcommands.Add(indexCommand); + rootCommand.Subcommands.Add(updateIndexCommand); + rootCommand.Subcommands.Add(checkCommand); + + return rootCommand.Parse(args).Invoke(); } - private static int Index(IndexOptions opts) + private static async Task Index(string sourcePath, IProgressScope render) { - if (string.IsNullOrEmpty(opts.Source)) + var source = new DemFileSystemStorage(sourcePath); + DemDatabaseIndex index; + using (var progress = render.CreatePercent("Build index")) { - throw new ArgumentNullException(); + index = await source.BuildIndexAsync(progress).ConfigureAwait(false); } - var source = new DemFileSystemStorage(opts.Source); - var index = source.BuildIndex(); - using(var file = File.Create(Path.Combine(opts.Source, "index.json"))) + using (var file = File.Create(Path.Combine(sourcePath, "index.json"))) { JsonSerializer.Serialize(file, index); } return 0; } - private static int Repack(RepackOptions opts) + private static int Repack(string sourcePath, string targetPath, Compression targetCompression, int maxCPU, bool keep, IProgressScope render) { - if (string.IsNullOrEmpty(opts.Source)) - { - throw new ArgumentNullException(); - } - if (string.IsNullOrEmpty(opts.Target)) - { - throw new ArgumentNullException(); - } - - var files = Directory.GetFiles(opts.Source, "*.*", SearchOption.AllDirectories); + var files = Directory.GetFiles(sourcePath, "*.*", SearchOption.AllDirectories); var demFiles = new List(); var zipFiles = new List(); @@ -92,25 +143,24 @@ private static int Repack(RepackOptions opts) } var parallel = new ParallelOptions(); - if (opts.MaxCPU > 0) + if (maxCPU > 0) { - parallel.MaxDegreeOfParallelism = opts.MaxCPU; + parallel.MaxDegreeOfParallelism = maxCPU; } - Directory.CreateDirectory(opts.Target); + Directory.CreateDirectory(targetPath); if (demFiles.Count > 0) { - Console.WriteLine($"{demFiles.Count} DEM files to process."); - using (var report = new ProgressReport("DEM", demFiles.Count)) + using (var report = render.CreateInteger("DEM", demFiles.Count)) { Parallel.ForEach(demFiles, parallel, file => { - var filename = CompressionHelper.GetFileName(Path.GetFileName(file)) + CompressionHelper.GetExtension(opts.TargetCompression); - var target = Path.Combine(opts.Target, filename); - if (!opts.Keep || !File.Exists(target)) + var filename = CompressionHelper.GetFileName(Path.GetFileName(file)) + CompressionHelper.GetExtension(targetCompression); + var target = Path.Combine(targetPath, filename); + if (!keep || !File.Exists(target)) { - CompressionHelper.Write(target, opts.TargetCompression, + CompressionHelper.Write(target, targetCompression, output => CompressionHelper.Read(file, input => input.CopyTo(output))); } report.ReportOneDone(); @@ -120,24 +170,23 @@ private static int Repack(RepackOptions opts) if (zipFiles.Count > 0) { - Console.WriteLine($"{zipFiles.Count} ZIP files to scan."); - using (var report = new ProgressReport("ZIP", zipFiles.Count)) + using (var report = render.CreateInteger("ZIP", zipFiles.Count)) { Parallel.ForEach(zipFiles, parallel, file => { using (var archive = new ZipArchive(File.OpenRead(file), ZipArchiveMode.Read)) { - foreach(var entry in archive.Entries) + foreach (var entry in archive.Entries) { if (entry.Name.EndsWith("_DSM.tif", StringComparison.OrdinalIgnoreCase)) { - var filename = entry.Name + CompressionHelper.GetExtension(opts.TargetCompression); - var target = Path.Combine(opts.Target, filename); - if (!opts.Keep || !File.Exists(target)) + var filename = entry.Name + CompressionHelper.GetExtension(targetCompression); + var target = Path.Combine(targetPath, filename); + if (!keep || !File.Exists(target)) { using (var input = entry.Open()) { - CompressionHelper.Write(target, opts.TargetCompression, + CompressionHelper.Write(target, targetCompression, output => input.CopyTo(output)); } } @@ -150,5 +199,157 @@ private static int Repack(RepackOptions opts) } return 0; } + + private static async Task UpdateIndex(string sourcePath) + { + var indexFile = Path.Combine(sourcePath, "index.json"); + if (!File.Exists(indexFile)) + { + Console.Error.WriteLine($"No index.json found in '{sourcePath}'. Run 'index' first."); + return 1; + } + + DemDatabaseIndex index; + using (var file = File.OpenRead(indexFile)) + { + index = JsonSerializer.Deserialize(file)!; + } + + if (index.Cells.Count(c => c.Sha256 == null) == 0) + { + Console.WriteLine("All entries already have a SHA-256 checksum. Nothing to do."); + return 0; + } + + // Backup existing index + File.Copy(indexFile, indexFile + $"-{DateTime.Now:yyyyMMddHHmmss}.bak"); + + using var render = ConsoleProgessHelper.Create(); + var updated = new List(); + using (var report = render.CreateInteger("SHA-256", index.Cells.Count)) + { + foreach (var cell in index.Cells) + { + if (cell.Sha256 == null) + { + var fullPath = Path.Combine(sourcePath, cell.Path.Replace('/', Path.DirectorySeparatorChar)); + var hash = await Sha256Helper.ComputeHexAsync(fullPath).ConfigureAwait(false); + updated.Add(new DemDatabaseFileInfos(cell.Path, cell.Metadata, hash)); + } + else + { + updated.Add(cell); + } + report.ReportOneDone(); + } + } + + var newIndex = new DemDatabaseIndex(updated); + using (var file = File.Create(indexFile)) + { + JsonSerializer.Serialize(file, newIndex); + } + return 0; + } + private static async Task Check(string? localPath, string? url, bool verify, int? sample = null) + { + IDemStorage storage; + if (localPath != null) + { + if (!Directory.Exists(localPath)) + { + Console.Error.WriteLine($"Directory '{localPath}' does not exist."); + return 1; + } + storage = new DemFileSystemStorage(localPath); + } + else + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) + { + Console.Error.WriteLine($"Invalid URL '{url}'."); + return 1; + } + storage = new DemHttpStorage(uri); + } + + Console.WriteLine("Reading index..."); + DemDatabaseIndex index; + try + { + index = await storage.ReadIndex(); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Failed to read index: {ex.Message}"); + return 1; + } + + Console.WriteLine($"Cells : {index.Cells.Count}"); + + if (index.Cells.Count > 0) + { + var withSha256 = index.Cells.Count(c => c.Sha256 != null); + Console.WriteLine($"SHA-256 : {withSha256}/{index.Cells.Count}"); + + var minLat = index.Cells.Min(c => c.Metadata.Start.Latitude); + var maxLat = index.Cells.Max(c => c.Metadata.End.Latitude); + var minLon = index.Cells.Min(c => c.Metadata.Start.Longitude); + var maxLon = index.Cells.Max(c => c.Metadata.End.Longitude); + Console.WriteLine($"Coverage : Lat [{minLat:F4} ; {maxLat:F4}], Lon [{minLon:F4} ; {maxLon:F4}]"); + } + + if (verify) + { + Console.WriteLine("Verifying checksums..."); + int errors = 0; + int skipped = 0; + var cellsWithHash = index.Cells.Where(c => c.Sha256 != null).ToList(); + IEnumerable cellsToVerify; + if (sample.HasValue && sample.Value < cellsWithHash.Count) + { + var rng = new Random(); + cellsToVerify = cellsWithHash.OrderBy(_ => rng.Next()).Take(sample.Value).ToList(); + Console.WriteLine($" Sampling {sample.Value}/{cellsWithHash.Count} cells with a checksum."); + } + else + { + cellsToVerify = cellsWithHash; + } + skipped = index.Cells.Count - cellsWithHash.Count; + using (var render = ConsoleProgessHelper.Create()) + { + foreach (var cell in cellsToVerify.WithProgress(render, "Checksum")) + { + var actualHash = await storage.GetSha256Async(cell.Path).ConfigureAwait(false); + if (actualHash == null) + { + render.WriteLine($" MISSING: {cell.Path}"); + errors++; + } + else if (!string.Equals(actualHash, cell.Sha256, StringComparison.OrdinalIgnoreCase)) + { + render.WriteLine($" INVALID: Actual:'{actualHash}' Expected:'{cell.Sha256}'"); + errors++; + } + } + } + if (skipped > 0) + { + Console.WriteLine($" Skipped {skipped} cell(s) without a stored checksum."); + } + if (errors == 0) + { + Console.WriteLine($" All checksums OK."); + } + else + { + Console.Error.WriteLine($" {errors} error(s) found."); + return 1; + } + } + + return 0; + } } } \ No newline at end of file diff --git a/DemUtility/ProgressReport.cs b/DemUtility/ProgressReport.cs deleted file mode 100644 index 1bd34d1..0000000 --- a/DemUtility/ProgressReport.cs +++ /dev/null @@ -1,118 +0,0 @@ -using System; -using System.Diagnostics; -using System.Threading; - -namespace DemUtility -{ - internal class ProgressReport : IDisposable - { - private readonly string taskName; - private readonly int itemsToDo; - private readonly Stopwatch sw; - private readonly Stopwatch lastReport; - private readonly object locker = new object(); - private int lastDone = 0; - - public ProgressReport(string taskName, int itemsToDo = 1) - { - this.taskName = taskName; - this.itemsToDo = itemsToDo; - this.sw = Stopwatch.StartNew(); - this.lastReport = Stopwatch.StartNew(); - - Trace.WriteLine(string.Empty); - Trace.WriteLine($"Begin task {taskName}"); - Console.Write(taskName); - WritePercent(0); - } - - public int Total - { - get { return itemsToDo; } - } - - public void ReportOneDone() - { - Interlocked.Increment(ref lastDone); - DrawDone(lastDone); - } - - public void ReportItemsDone(int done) - { - lastDone = done; - DrawDone(done); - } - - private void DrawDone(int done) - { - if (lastReport.ElapsedMilliseconds > 500) - { - lock (locker) - { - if (lastReport.ElapsedMilliseconds > 500) - { - lastReport.Restart(); - WritePercent(done * 100.0 / itemsToDo); - - if (done > 0) - { - var milisecondsLeft = sw.ElapsedMilliseconds * (itemsToDo - done) / done; - if (milisecondsLeft > 120000d) - { - Console.Write($"{Math.Round(milisecondsLeft / 60000d)} min left"); - } - else - { - Console.Write($"{Math.Ceiling(milisecondsLeft / 1000d)} sec left"); - } - } - CleanEndOfLine(); - } - } - } - } - - private void WritePercent(double percent) - { - if (!Console.IsOutputRedirected) - { - var cols = Math.Max(0, Math.Min(20, (int)(percent / 5))); - Console.CursorLeft = 20; - Console.ForegroundColor = ConsoleColor.Green; - Console.Write(new string('#', cols)); - Console.ForegroundColor = ConsoleColor.Gray; - Console.Write(new string('-', 20 - cols)); - Console.Write(' '); - Console.Write($"{percent,6:0.00} % "); - } - } - - public void TaskDone() - { - if (Console.IsOutputRedirected) - { - Console.Write(' '); - } - sw.Stop(); - WritePercent(100d); - Console.Write($"Done in {Math.Ceiling(sw.ElapsedMilliseconds / 1000d)} sec"); - CleanEndOfLine(); - Console.WriteLine(); - Trace.WriteLine($"Task {taskName} took {sw.ElapsedMilliseconds} msec"); - Trace.Flush(); - } - - private static void CleanEndOfLine() - { - if (!Console.IsOutputRedirected) - { - Console.Write(new string(' ', Console.BufferWidth - Console.CursorLeft - 1)); - } - } - - public void Dispose() - { - TaskDone(); - } - } -} diff --git a/DemUtility/Properties/launchSettings.json b/DemUtility/Properties/launchSettings.json index 68aaa3a..16e899c 100644 --- a/DemUtility/Properties/launchSettings.json +++ b/DemUtility/Properties/launchSettings.json @@ -1,6 +1,6 @@ { "profiles": { - "Repack STRM": { + "Repack SRTM": { "commandName": "Project", "commandLineArgs": "repack -k -s E:\\Carto\\SRTMv3 -t c:\\temp\\SRTMv3 -m 10" }, @@ -8,13 +8,57 @@ "commandName": "Project", "commandLineArgs": "repack -k -s E:\\Carto\\AW3D30 -t c:\\temp\\AW3D30 -m 10" }, - "Index STRM": { + + "Index AW3D30": { "commandName": "Project", - "commandLineArgs": "index -p C:\\temp\\SRTMv3" + "commandLineArgs": "index -p D:\\dem\\AW3D30" }, - "Index AW3D30": { + "Index SRTM15Plus": { + "commandName": "Project", + "commandLineArgs": "index -p D:\\dem\\SRTM15Plus" + }, + "Index SRTM1": { + "commandName": "Project", + "commandLineArgs": "index -p D:\\dem\\SRTM1" + }, + + "Update Index AW3D30": { + "commandName": "Project", + "commandLineArgs": "update-index -p D:\\dem\\AW3D30" + }, + "Update Index SRTM15Plus": { + "commandName": "Project", + "commandLineArgs": "update-index -p D:\\dem\\SRTM15Plus" + }, + "Update Index SRTM1": { + "commandName": "Project", + "commandLineArgs": "update-index -p D:\\dem\\SRTM1" + }, + + "Check AW3D30": { + "commandName": "Project", + "commandLineArgs": "check -v -p D:\\dem\\AW3D30" + }, + "Check SRTM15Plus": { + "commandName": "Project", + "commandLineArgs": "check -v -p D:\\dem\\SRTM15Plus" + }, + "Check SRTM1": { + "commandName": "Project", + "commandLineArgs": "check -v -p D:\\dem\\SRTM1" + }, + + "Check CDN AW3D30": { + "commandName": "Project", + "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/AW3D30/ -n 100" + }, + "Check CDN SRTM15Plus": { + "commandName": "Project", + "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/SRTM15Plus/ -n 100" + }, + "Check CDN SRTM1": { "commandName": "Project", - "commandLineArgs": "index -p C:\\temp\\AW3D30" + "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/SRTM1/ -n 100" } } } \ No newline at end of file diff --git a/MapToolkit.Test/CollectionDefinitions.cs b/MapToolkit.Test/CollectionDefinitions.cs new file mode 100644 index 0000000..67dd774 --- /dev/null +++ b/MapToolkit.Test/CollectionDefinitions.cs @@ -0,0 +1,5 @@ +namespace Pmad.Cartography.Test +{ + [CollectionDefinition("Sequential", DisableParallelization = true)] + public class SequentialCollectionDefinition { } +} diff --git a/MapToolkit.Test/CompressionHelperTest.cs b/MapToolkit.Test/CompressionHelperTest.cs index f7327c2..8a67835 100644 --- a/MapToolkit.Test/CompressionHelperTest.cs +++ b/MapToolkit.Test/CompressionHelperTest.cs @@ -22,9 +22,13 @@ public void GetExtension_FromFilename() public void GetExtension_FromCompression() { Assert.Equal(".zst", CompressionHelper.GetExtension(Compression.ZSTD)); - Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZib)); + Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZip)); Assert.Equal(".bt", CompressionHelper.GetExtension(Compression.Brotli)); Assert.Equal(string.Empty, CompressionHelper.GetExtension(Compression.None)); + +#pragma warning disable CS0618 // Le type ou le membre est obsolète + Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZib)); +#pragma warning restore CS0618 // Le type ou le membre est obsolète } [Fact] @@ -40,7 +44,7 @@ public void GetFileName_RemovesExtension() [Fact] public void ReadSeekable_ReadsCompressedFile_Compressed() { - var filename = WriteCompressedFile(Compression.GZib); + var filename = WriteCompressedFile(Compression.GZip); var result = CompressionHelper.ReadSeekable(filename, stream => { using (var reader = new StreamReader(stream)) @@ -68,9 +72,9 @@ public void ReadSeekable_ReadsCompressedFile_None() } [Fact] - public void Read_ReadsCompressedFile_GZib() + public void Read_ReadsCompressedFile_GZip() { - var filename = WriteCompressedFile(Compression.GZib); + var filename = WriteCompressedFile(Compression.GZip); var result = CompressionHelper.Read(filename, stream => { using (var reader = new StreamReader(stream)) @@ -130,7 +134,7 @@ public void Read_ReadsCompressedFile_Brotli() [Fact] public void GetSize_ReturnsCorrectSize_GZip() { - var filename = WriteCompressedFile(Compression.GZib); + var filename = WriteCompressedFile(Compression.GZip); var size = CompressionHelper.GetSize(filename); Assert.Equal(12, size); // "test content" length File.Delete(filename); diff --git a/MapToolkit.Test/Databases/DemDatabaseEntryTest.cs b/MapToolkit.Test/Databases/DemDatabaseEntryTest.cs index f0c0c3c..7d624a2 100644 --- a/MapToolkit.Test/Databases/DemDatabaseEntryTest.cs +++ b/MapToolkit.Test/Databases/DemDatabaseEntryTest.cs @@ -40,7 +40,7 @@ public async Task DemDatabaseEntry_Load() var cache = new MemoryCache(new MemoryCacheOptions()); var dataCell = new Mock(); dataCell.Setup(d => d.SizeInBytes).Returns(100); - storage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell.Object); + storage.Setup(s => s.LoadAsync(It.IsAny(), null, default)).ReturnsAsync(dataCell.Object); var entry = new DemDatabaseEntry("test", metadata.Object); var result = await entry.Load(storage.Object, cache); diff --git a/MapToolkit.Test/Databases/DemDatabaseTest.cs b/MapToolkit.Test/Databases/DemDatabaseTest.cs index 09da847..abce7cb 100644 --- a/MapToolkit.Test/Databases/DemDatabaseTest.cs +++ b/MapToolkit.Test/Databases/DemDatabaseTest.cs @@ -39,7 +39,7 @@ public async Task GetDataCellsAsync_ShouldReturnDataCells() var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100)); var dataCell = new Mock().Object; mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List { entry })); - mockStorage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell); + mockStorage.Setup(s => s.LoadAsync(It.IsAny(), It.IsAny(), default)).ReturnsAsync(dataCell); // Act var result = await demDatabase.GetDataCellsAsync(start, end); @@ -60,7 +60,7 @@ public async Task CreateView_ShouldReturnDemDataView() var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100)); var dataCell = new DemDataCellPixelIsPoint(new Coordinates(0, 0), new Coordinates(1, 1), new float[100, 100]); mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List { entry })); - mockStorage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell); + mockStorage.Setup(s => s.LoadAsync(It.IsAny(), It.IsAny(), default)).ReturnsAsync(dataCell); // Act var result = await demDatabase.CreateView(start, end); @@ -119,7 +119,7 @@ public void GetElevation_ShouldReturnElevation() var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100)); var dataCell = new Mock().Object; mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List { entry })); - mockStorage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell); + mockStorage.Setup(s => s.LoadAsync(It.IsAny(), It.IsAny(), default)).ReturnsAsync(dataCell); // Act var result = demDatabase.GetElevation(coordinates, interpolation); @@ -140,7 +140,7 @@ public async Task GetElevationAsync_ShouldReturnElevation() var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100)); var dataCell = new DemDataCellPixelIsPoint(new Coordinates(0, 0), new Coordinates(1, 1), new float[100, 100]); mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List { entry })); - mockStorage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell); + mockStorage.Setup(s => s.LoadAsync(It.IsAny(), It.IsAny(), default)).ReturnsAsync(dataCell); // Act var result = await demDatabase.GetElevationAsync(coordinates, interpolation); diff --git a/MapToolkit.Test/Databases/DemFileSystemStorageTest.cs b/MapToolkit.Test/Databases/DemFileSystemStorageTest.cs new file mode 100644 index 0000000..6cd3ea3 --- /dev/null +++ b/MapToolkit.Test/Databases/DemFileSystemStorageTest.cs @@ -0,0 +1,88 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Threading.Tasks; +using Pmad.Cartography.Databases; +using Xunit; + +namespace Pmad.Cartography.Test.Databases +{ + public class DemFileSystemStorageTest + { + private static string CreateTempFile(byte[] content) + { + var path = Path.GetTempFileName(); + File.WriteAllBytes(path, content); + return path; + } + + private static string ComputeExpectedHex(byte[] content) + { + return Convert.ToHexString(SHA256.HashData(content)).ToLowerInvariant(); + } + + [Fact] + public async Task GetSha256Async_ShouldReturnCorrectHash_WhenFileExists() + { + // Arrange + var content = "hello dem"u8.ToArray(); + var file = CreateTempFile(content); + var basePath = Path.GetDirectoryName(file)!; + var fileName = Path.GetFileName(file); + var storage = new DemFileSystemStorage(basePath); + + try + { + // Act + var hash = await storage.GetSha256Async(fileName); + + // Assert + Assert.NotNull(hash); + Assert.Equal(ComputeExpectedHex(content), hash); + } + finally + { + File.Delete(file); + } + } + + [Fact] + public async Task GetSha256Async_ShouldReturnNull_WhenFileDoesNotExist() + { + // Arrange + var basePath = Path.GetTempPath(); + var storage = new DemFileSystemStorage(basePath); + + // Act + var hash = await storage.GetSha256Async("nonexistent_file_that_does_not_exist.hgt.zst"); + + // Assert + Assert.Null(hash); + } + + [Fact] + public async Task GetSha256Async_ShouldReturnLowercaseHex() + { + // Arrange + var content = "case check"u8.ToArray(); + var file = CreateTempFile(content); + var basePath = Path.GetDirectoryName(file)!; + var fileName = Path.GetFileName(file); + var storage = new DemFileSystemStorage(basePath); + + try + { + // Act + var hash = await storage.GetSha256Async(fileName); + + // Assert + Assert.NotNull(hash); + Assert.Equal(hash!.ToLowerInvariant(), hash); + } + finally + { + File.Delete(file); + } + } + } +} diff --git a/MapToolkit.Test/Databases/DemHttpStorageTest.cs b/MapToolkit.Test/Databases/DemHttpStorageTest.cs index 651a4ce..2046d92 100644 --- a/MapToolkit.Test/Databases/DemHttpStorageTest.cs +++ b/MapToolkit.Test/Databases/DemHttpStorageTest.cs @@ -1,25 +1,26 @@ using System; using System.IO; +using System.Linq; using System.Net.Http; +using System.Security.Cryptography; using System.Threading.Tasks; -using Xunit; using Pmad.Cartography.Databases; -using Pmad.Cartography.DataCells; -using System.Linq; namespace Pmad.Cartography.Test.Databases { public class DemHttpStorageTest { - private const string userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"; + private const string userAgent = "Mozilla/5.0 (Pmad-Cartography; UnitTests)"; private const string baseAddress = "https://cdn.dem.pmad.net/SRTM1/"; private const string samplePath = "N00E006.SRTMGL1.hgt.zst"; + private static string CreateUniqueTempDir() => Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + [Fact] public async Task Load_ShouldDownloadAndCacheFile() { // Arrange - var localCache = Path.Combine(Path.GetTempPath(), "dem_test_cache"); + var localCache = CreateUniqueTempDir(); var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) }; httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); var storage = new DemHttpStorage(localCache, httpClient); @@ -37,11 +38,83 @@ public async Task Load_ShouldDownloadAndCacheFile() Assert.True(File.Exists(cacheFile)); } + [Fact] + public async Task Load_ShouldUseCachedFile_OnSecondCall() + { + // Arrange + var localCache = CreateUniqueTempDir(); + var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) }; + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); + var storage = new DemHttpStorage(localCache, httpClient); + var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", samplePath); + + // Ensure file is already cached + await storage.Load(samplePath); + var firstWriteTime = File.GetLastWriteTimeUtc(cacheFile); + + // Act + await storage.Load(samplePath); + var secondWriteTime = File.GetLastWriteTimeUtc(cacheFile); + + // Assert: cached file was not re-written + Assert.Equal(firstWriteTime, secondWriteTime); + } + + [Fact] + public async Task Load_ShouldDeleteAndRedownload_WhenChecksumMismatch() + { + // Arrange + var localCache = CreateUniqueTempDir(); + var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) }; + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); + var storage = new DemHttpStorage(localCache, httpClient); + var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", samplePath); + + // Pre-create a corrupted cache file + Directory.CreateDirectory(Path.GetDirectoryName(cacheFile)!); + File.WriteAllBytes(cacheFile, new byte[] { 0x00, 0x01 }); + + // Act: loading with a wrong checksum should re-download, fail validation, delete the file and throw + await Assert.ThrowsAsync(() => + storage.LoadAsync(samplePath, "0000000000000000000000000000000000000000000000000000000000000000")); + + // The file should have been deleted after the checksum mismatch + Assert.False(File.Exists(cacheFile)); + } + + [Fact] + public async Task LoadAsync_ShouldNotThrow_WhenChecksumMatches() + { + // Arrange + var localCache = CreateUniqueTempDir(); + var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) }; + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); + var storage = new DemHttpStorage(localCache, httpClient); + var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", samplePath); + if (File.Exists(cacheFile)) + { + File.Delete(cacheFile); + } + + // Download once without checksum to get the real file + await storage.Load(samplePath); + + // Compute the real checksum + using var cacheFileStream = File.OpenRead(cacheFile); + var realHash = Convert.ToHexString(SHA256.HashData(cacheFileStream)).ToLowerInvariant(); + + // Act: second load with correct checksum should succeed + var dataCell = await storage.LoadAsync(samplePath, realHash); + + // Assert + Assert.NotNull(dataCell); + } + [Fact] public async Task ReadIndex_ShouldDownloadAndDeserializeIndex() { // Arrange - var localCache = Path.Combine(Path.GetTempPath(), "dem_test_cache"); + var localCache = CreateUniqueTempDir(); var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) }; httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); var storage = new DemHttpStorage(localCache, httpClient); @@ -58,5 +131,236 @@ public async Task ReadIndex_ShouldDownloadAndDeserializeIndex() Assert.Equal(new(0,6),cell.Metadata.Start); Assert.Equal(new(1,7),cell.Metadata.End); } + + [Fact] + public async Task ReadIndex_ShouldUseCachedIndex_WhenFresh() + { + // Arrange + var localCache = CreateUniqueTempDir(); + var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) }; + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); + var storage = new DemHttpStorage(localCache, httpClient); + var indexCacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", "index.json"); + + // Ensure a fresh cached index exists + await storage.ReadIndex(); + var firstWriteTime = File.GetLastWriteTimeUtc(indexCacheFile); + + // Act: second call should use the cache + await storage.ReadIndex(); + var secondWriteTime = File.GetLastWriteTimeUtc(indexCacheFile); + + // Assert: the cache file was not re-written + Assert.Equal(firstWriteTime, secondWriteTime); + } + + [Fact] + public async Task ReadIndex_ShouldRefreshCache_WhenExpired() + { + // Arrange + var localCache = CreateUniqueTempDir(); + var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) }; + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); + var storage = new DemHttpStorage(localCache, httpClient); + var indexCacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", "index.json"); + + // Create a stale cache file + Directory.CreateDirectory(Path.GetDirectoryName(indexCacheFile)!); + File.WriteAllText(indexCacheFile, "{\"cells\":[]}"); + File.SetLastWriteTimeUtc(indexCacheFile, DateTime.UtcNow - DemHttpStorage.IndexCacheDuration - TimeSpan.FromSeconds(1)); + + // Act: should re-download because the cache is expired + var index = await storage.ReadIndex(); + var writeTime = File.GetLastWriteTimeUtc(indexCacheFile); + + // Assert: file was refreshed + Assert.True(writeTime > DateTime.UtcNow - TimeSpan.FromMinutes(1)); + Assert.NotEmpty(index.Cells); + } + + [Fact] + public async Task GetSha256Async_ShouldDownloadFileAndReturnHash() + { + // Arrange + var localCache = CreateUniqueTempDir(); + var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) }; + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); + var storage = new DemHttpStorage(localCache, httpClient); + var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", samplePath); + if (File.Exists(cacheFile)) + { + File.Delete(cacheFile); + } + + // Act + var hash = await storage.GetSha256Async(samplePath); + + // Assert + Assert.NotNull(hash); + Assert.Equal(64, hash.Length); // SHA-256 hex is 64 chars + using var cacheFileStream = File.OpenRead(cacheFile); + var expectedHash = Convert.ToHexString(SHA256.HashData(cacheFileStream)).ToLowerInvariant(); + Assert.Equal(expectedHash, hash); + } + + [Fact] + public async Task GetSha256Async_ShouldReturnNull_WhenFileNotFound() + { + // Arrange + var localCache = CreateUniqueTempDir(); + var handler = new NotFoundHttpMessageHandler(); + var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) }; + var storage = new DemHttpStorage(localCache, httpClient); + + // Act + var hash = await storage.GetSha256Async("nonexistent_file.hgt.zst"); + + // Assert + Assert.Null(hash); + } + + [Fact] + public async Task DownloadFile_ShouldSucceed_AfterTransientFailures() + { + // Arrange: fail twice, then succeed on the third attempt + var localCache = CreateUniqueTempDir(); + var content = new byte[] { 1, 2, 3, 4 }; + var handler = new CountingHttpMessageHandler(failCount: 2, successContent: content); + var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) }; + var storage = new DemHttpStorage(localCache, httpClient); + var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", "retry_test.bin"); + if (File.Exists(cacheFile)) File.Delete(cacheFile); + + // Act + var hash = await storage.GetSha256Async("retry_test.bin"); + + // Assert: succeeded after retries, file exists with correct content + Assert.NotNull(hash); + Assert.True(File.Exists(cacheFile)); + Assert.Equal(3, handler.CallCount); + } + + [Fact] + public async Task DownloadFile_ShouldThrow_WhenAllAttemptsExhausted() + { + // Arrange: always fail with a transient error + var localCache = CreateUniqueTempDir(); + var handler = new CountingHttpMessageHandler(failCount: 99, successContent: Array.Empty()); + var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) }; + var storage = new DemHttpStorage(localCache, httpClient); + + // Act & Assert: should throw after MaxDownloadAttempts (3) attempts + await Assert.ThrowsAsync(() => storage.GetSha256Async("always_failing.bin")); + Assert.Equal(3, handler.CallCount); + } + + [Fact] + public async Task DownloadFile_ShouldNotRetry_On404() + { + // Arrange + var localCache = CreateUniqueTempDir(); + var handler = new NotFoundHttpMessageHandler(); + var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) }; + var storage = new DemHttpStorage(localCache, httpClient); + + // Act + var hash = await storage.GetSha256Async("missing.bin"); + + // Assert: returns null immediately without retry + Assert.Null(hash); + Assert.Equal(1, handler.CallCount); + } + + [Fact] + public async Task DownloadFile_ShouldNotRetry_WhenCancelled() + { + // Arrange: fail once, then cancel — should not retry + var localCache = CreateUniqueTempDir(); + using var cts = new System.Threading.CancellationTokenSource(); + var handler = new CancellingHttpMessageHandler(cts, failCount: 1, successContent: Array.Empty()); + var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) }; + var storage = new DemHttpStorage(localCache, httpClient); + + // Act & Assert + await Assert.ThrowsAnyAsync(() => + storage.GetSha256Async("cancel_test.bin", cts.Token)); + Assert.Equal(1, handler.CallCount); + } + + private sealed class NotFoundHttpMessageHandler : HttpMessageHandler + { + public int CallCount { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, System.Threading.CancellationToken cancellationToken) + { + CallCount++; + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.NotFound)); + } + } + + /// + /// Returns an HTTP error for the first calls, then returns success with the given content. + /// + private sealed class CountingHttpMessageHandler : HttpMessageHandler + { + private readonly int failCount; + private readonly byte[] successContent; + + public int CallCount { get; private set; } + + public CountingHttpMessageHandler(int failCount, byte[] successContent) + { + this.failCount = failCount; + this.successContent = successContent; + } + + protected override Task SendAsync(HttpRequestMessage request, System.Threading.CancellationToken cancellationToken) + { + CallCount++; + if (CallCount <= failCount) + { + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError)); + } + var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new System.Net.Http.ByteArrayContent(successContent) + }; + return Task.FromResult(response); + } + } + + /// + /// Cancels the token after failing calls, so the retry path sees a cancelled token. + /// + private sealed class CancellingHttpMessageHandler : HttpMessageHandler + { + private readonly System.Threading.CancellationTokenSource cts; + private readonly int failCount; + private readonly byte[] successContent; + + public int CallCount { get; private set; } + + public CancellingHttpMessageHandler(System.Threading.CancellationTokenSource cts, int failCount, byte[] successContent) + { + this.cts = cts; + this.failCount = failCount; + this.successContent = successContent; + } + + protected override Task SendAsync(HttpRequestMessage request, System.Threading.CancellationToken cancellationToken) + { + CallCount++; + if (CallCount <= failCount) + { + cts.Cancel(); + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError)); + } + var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new System.Net.Http.ByteArrayContent(successContent) + }; + return Task.FromResult(response); + } + } } } diff --git a/MapToolkit.Test/Databases/Sha256HelperTest.cs b/MapToolkit.Test/Databases/Sha256HelperTest.cs new file mode 100644 index 0000000..ad755d9 --- /dev/null +++ b/MapToolkit.Test/Databases/Sha256HelperTest.cs @@ -0,0 +1,87 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Threading.Tasks; +using Pmad.Cartography.Databases; + +namespace Pmad.Cartography.Test.Databases +{ + public class Sha256HelperTest + { + private static string CreateTempFileWithContent(byte[] content) + { + var path = Path.GetTempFileName(); + File.WriteAllBytes(path, content); + return path; + } + + private static string ComputeExpectedHex(byte[] content) + { + var hash = SHA256.HashData(content); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + [Fact] + public async Task ComputeHex_ReturnsCorrectHash() + { + var content = "hello world"u8.ToArray(); + var file = CreateTempFileWithContent(content); + try + { + var expected = ComputeExpectedHex(content); + Assert.Equal(expected, await Sha256Helper.ComputeHexAsync(file)); + } + finally + { + File.Delete(file); + } + } + + [Fact] + public async Task VerifyAsync_ReturnsTrue_WhenHashMatches() + { + var content = "hello world"u8.ToArray(); + var file = CreateTempFileWithContent(content); + try + { + var hex = ComputeExpectedHex(content); + Assert.True(await Sha256Helper.VerifyAsync(file, hex)); + } + finally + { + File.Delete(file); + } + } + + [Fact] + public async Task VerifyAsync_ReturnsTrue_WhenHashMatchesUpperCase() + { + var content = "hello world"u8.ToArray(); + var file = CreateTempFileWithContent(content); + try + { + var hex = ComputeExpectedHex(content).ToUpperInvariant(); + Assert.True(await Sha256Helper.VerifyAsync(file, hex)); + } + finally + { + File.Delete(file); + } + } + + [Fact] + public async Task VerifyAsync_ReturnsFalse_WhenHashDoesNotMatch() + { + var content = "hello world"u8.ToArray(); + var file = CreateTempFileWithContent(content); + try + { + Assert.False(await Sha256Helper.VerifyAsync(file, "0000000000000000000000000000000000000000000000000000000000000000")); + } + finally + { + File.Delete(file); + } + } + } +} diff --git a/MapToolkit.Test/Databases/WellKnownDatabasesTest.cs b/MapToolkit.Test/Databases/WellKnownDatabasesTest.cs index a1df485..eeeaac8 100644 --- a/MapToolkit.Test/Databases/WellKnownDatabasesTest.cs +++ b/MapToolkit.Test/Databases/WellKnownDatabasesTest.cs @@ -3,8 +3,14 @@ namespace Pmad.Cartography.Test.Databases { + [Collection("Sequential")] public class WellKnownDatabasesTest { + public WellKnownDatabasesTest() + { + DemHttpStorage.ClearDefaultCache(); + } + [Fact] public async Task AW3D30_ContainsLondon() { diff --git a/MapToolkit/Compression.cs b/MapToolkit/Compression.cs index e684650..ee697ad 100644 --- a/MapToolkit/Compression.cs +++ b/MapToolkit/Compression.cs @@ -1,10 +1,14 @@ -namespace Pmad.Cartography +using System; + +namespace Pmad.Cartography { public enum Compression { - None, - ZSTD, - GZib, - Brotli + None = 0, + ZSTD = 1, + GZip = 2, + [Obsolete("Use GZip instead.")] + GZib = 2, + Brotli = 3 } } \ No newline at end of file diff --git a/MapToolkit/CompressionHelper.cs b/MapToolkit/CompressionHelper.cs index 20f5199..b796294 100644 --- a/MapToolkit/CompressionHelper.cs +++ b/MapToolkit/CompressionHelper.cs @@ -22,7 +22,7 @@ public static string GetExtension(Compression compression) { case Compression.ZSTD: return ExtensionZStd; - case Compression.GZib: + case Compression.GZip: return ExtensionGZip; case Compression.Brotli: return ExtensionBrotli; @@ -191,7 +191,7 @@ public static void Write(string target, Compression compression, Action write(compressed); } break; - case Compression.GZib: + case Compression.GZip: using (var compressed = new GZipStream(stream, CompressionMode.Compress)) { write(compressed); diff --git a/MapToolkit/DataCells/DemDataCellMetadata.cs b/MapToolkit/DataCells/DemDataCellMetadata.cs index 725c9dc..f099655 100644 --- a/MapToolkit/DataCells/DemDataCellMetadata.cs +++ b/MapToolkit/DataCells/DemDataCellMetadata.cs @@ -49,8 +49,10 @@ internal static Coordinates EndFromResolution(Coordinates start, DemRasterType t public Coordinates End { get; } + [JsonPropertyName("PointsPerCellLat")] // Legacy name, kept for backward compatibility public int PointsLat { get; } + [JsonPropertyName("PointsPerCellLon")] // Legacy name, kept for backward compatibility public int PointsLon { get; } } } diff --git a/MapToolkit/Databases/DemDatabase.cs b/MapToolkit/Databases/DemDatabase.cs index 88fcac3..f857e3a 100644 --- a/MapToolkit/Databases/DemDatabase.cs +++ b/MapToolkit/Databases/DemDatabase.cs @@ -51,7 +51,7 @@ private async Task LoadIndexInternal() entry.UnLoad(cache); } entries.Clear(); - entries.AddRange((await storage.ReadIndex().ConfigureAwait(false)).Cells.Select(i => new DemDatabaseEntry(i.Path, i.Metadata))); + entries.AddRange((await storage.ReadIndex().ConfigureAwait(false)).Cells.Select(i => new DemDatabaseEntry(i.Path, i.Metadata, i.Sha256))); } private async Task EnsureIndexIsLoadedAsync() diff --git a/MapToolkit/Databases/DemDatabaseEntry.cs b/MapToolkit/Databases/DemDatabaseEntry.cs index 9edf742..3b72e42 100644 --- a/MapToolkit/Databases/DemDatabaseEntry.cs +++ b/MapToolkit/Databases/DemDatabaseEntry.cs @@ -7,16 +7,19 @@ namespace Pmad.Cartography.Databases { internal class DemDatabaseEntry { - internal DemDatabaseEntry(string path, IDemDataCellMetadata metadata) + internal DemDatabaseEntry(string path, IDemDataCellMetadata metadata, string? sha256 = null) { Path = path; Metadata = metadata; + Sha256 = sha256; } public string Path { get; } public IDemDataCellMetadata Metadata { get; } + public string? Sha256 { get; } + public bool Contains(Coordinates coordinates) { return Metadata.Start.Latitude <= coordinates.Latitude && @@ -38,7 +41,7 @@ public async Task Load(IDemStorage storage, IMemoryCache cache) if (!cache.TryGetValue(this, out IDemDataCell? result) || result == null) { using var entry = cache.CreateEntry(this); - result = await storage.Load(Path).ConfigureAwait(false); + result = await storage.LoadAsync(Path, Sha256).ConfigureAwait(false); entry.Value = result; entry.Size = result.SizeInBytes; } diff --git a/MapToolkit/Databases/DemDatabaseFileInfos.cs b/MapToolkit/Databases/DemDatabaseFileInfos.cs index 8b7d73f..ac26865 100644 --- a/MapToolkit/Databases/DemDatabaseFileInfos.cs +++ b/MapToolkit/Databases/DemDatabaseFileInfos.cs @@ -9,20 +9,28 @@ namespace Pmad.Cartography.Databases public class DemDatabaseFileInfos { [JsonConstructor] - public DemDatabaseFileInfos(string path, DemDataCellMetadata metadata) + public DemDatabaseFileInfos(string path, DemDataCellMetadata metadata, string? sha256 = null) { Path = path; Metadata = metadata; + Sha256 = sha256; } - public DemDatabaseFileInfos(string path, IDemDataCellMetadata metadata) + public DemDatabaseFileInfos(string path, IDemDataCellMetadata metadata, string? sha256 = null) { Path = path; Metadata = metadata as DemDataCellMetadata ?? new DemDataCellMetadata(metadata); + Sha256 = sha256; } public string Path { get; } public DemDataCellMetadata Metadata { get; } + + /// + /// Optional SHA-256 hex digest of the cell file, used to verify integrity after download. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Sha256 { get; } } } diff --git a/MapToolkit/Databases/DemFileSystemStorage.cs b/MapToolkit/Databases/DemFileSystemStorage.cs index 8c43388..908cf6f 100644 --- a/MapToolkit/Databases/DemFileSystemStorage.cs +++ b/MapToolkit/Databases/DemFileSystemStorage.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; using System.IO; -using System.Text; using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using Pmad.Cartography.DataCells; @@ -27,19 +27,28 @@ public async Task ReadIndex() return (await JsonSerializer.DeserializeAsync(input, DemDatabaseIndexContext.Default.DemDatabaseIndex).ConfigureAwait(false))!; } } - return BuildIndex(); + return await BuildIndexAsync().ConfigureAwait(false); } + [Obsolete] public DemDatabaseIndex BuildIndex() { + return BuildIndexAsync().GetAwaiter().GetResult(); + } + + public async Task BuildIndexAsync(IProgress? progress = null, CancellationToken cancellationToken = default) + { var entries = new List(); var files = Directory.GetFiles(basePath, "*.*", SearchOption.AllDirectories); + var done = 0; foreach (var file in files) { if (DemDataCell.IsDemDataCellFile(file)) { - entries.Add(new DemDatabaseFileInfos(GetRelative(file), DemDataCell.LoadMetadata(file))); + entries.Add(new DemDatabaseFileInfos(GetRelative(file), DemDataCell.LoadMetadata(file), await Sha256Helper.ComputeHexAsync(file, cancellationToken).ConfigureAwait(false))); } + done++; + progress?.Report((double)done / files.Length); } return new DemDatabaseIndex(entries); } @@ -51,7 +60,27 @@ private string GetRelative(string file) public Task Load(string path) { - return Task.FromResult(DemDataCell.Load(Path.Combine(basePath, path))); + return LoadAsync(path, null); + } + + public async Task LoadAsync(string path, string? expectedSha256, CancellationToken cancellationToken = default) + { + var fullPath = Path.Combine(basePath, path); + if (expectedSha256 != null && !await Sha256Helper.VerifyAsync(fullPath, expectedSha256, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidDataException($"SHA-256 checksum mismatch for '{path}'."); + } + return DemDataCell.Load(fullPath); + } + + public async Task GetSha256Async(string path, CancellationToken cancellationToken = default) + { + var fullPath = Path.Combine(basePath, path); + if (File.Exists(fullPath)) + { + return await Sha256Helper.ComputeHexAsync(fullPath, cancellationToken).ConfigureAwait(false); + } + return null; } } } diff --git a/MapToolkit/Databases/DemHttpStorage.cs b/MapToolkit/Databases/DemHttpStorage.cs index 2e6382d..465c1d9 100644 --- a/MapToolkit/Databases/DemHttpStorage.cs +++ b/MapToolkit/Databases/DemHttpStorage.cs @@ -2,6 +2,7 @@ using System.IO; using System.Net.Http; using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using Pmad.Cartography.DataCells; @@ -9,16 +10,23 @@ namespace Pmad.Cartography.Databases { public class DemHttpStorage : IDemStorage { + /// + /// How long a cached index.json is considered fresh before being re-downloaded. + /// + public static TimeSpan IndexCacheDuration { get; set; } = TimeSpan.FromHours(24); + + const int MaxDownloadAttempts = 3; + private readonly string localCache; private readonly HttpClient client; - public DemHttpStorage (string? localCache, HttpClient client) + public DemHttpStorage(string? localCache, HttpClient client) { this.localCache = localCache ?? DefaultCacheLocation; this.client = client; } - public DemHttpStorage(string? localCache, Uri baseAddress) + public DemHttpStorage(string? localCache, Uri baseAddress) : this(localCache, new HttpClient() { BaseAddress = baseAddress }) { @@ -31,33 +39,114 @@ public DemHttpStorage(Uri baseAddress) } public static string DefaultCacheLocation => Path.Combine(Path.GetTempPath(), "dem"); - - public async Task Load(string path) + + public static void ClearDefaultCache() + { + var cacheDir = DefaultCacheLocation; + if (Directory.Exists(cacheDir)) + { + Directory.Delete(cacheDir, recursive: true); + } + } + + private string GetCacheFile(string path) { var uri = new Uri(client.BaseAddress!, path); - var cacheFile = Path.Combine(localCache, uri.DnsSafeHost, uri.AbsolutePath.Substring(1).Replace('/', Path.DirectorySeparatorChar)); - if(!File.Exists(cacheFile)) + return Path.Combine(localCache, uri.DnsSafeHost, uri.AbsolutePath.Substring(1).Replace('/', Path.DirectorySeparatorChar)); + } + + public async Task LoadAsync(string path, string? expectedSha256 = null, CancellationToken cancellationToken = default) + { + var cacheFile = GetCacheFile(path); + if (File.Exists(cacheFile)) { - Directory.CreateDirectory(Path.GetDirectoryName(cacheFile)!); - // XXX: Limit cache size ? - // XXX: Cache invalidation ? - using (var input = await client.GetStreamAsync(path).ConfigureAwait(false)) + if (expectedSha256 != null && !await Sha256Helper.VerifyAsync(cacheFile, expectedSha256, cancellationToken).ConfigureAwait(false)) { - using (var cache = File.Create(cacheFile)) + File.Delete(cacheFile); + } + } + + if (!File.Exists(cacheFile)) + { + await DownloadFile(path, cacheFile, cancellationToken).ConfigureAwait(false); + if (expectedSha256 != null && !await Sha256Helper.VerifyAsync(cacheFile, expectedSha256, cancellationToken).ConfigureAwait(false)) + { + File.Delete(cacheFile); + throw new InvalidDataException($"SHA-256 checksum mismatch for '{path}'. The downloaded file may be corrupted."); + } + } + + return DemDataCell.Load(cacheFile); + } + + private async Task DownloadFile(string path, string cacheFile, CancellationToken cancellationToken = default) + { + var cacheDirectory = Path.GetDirectoryName(cacheFile)!; + + Directory.CreateDirectory(cacheDirectory); + + var tempFile = Path.Combine(cacheDirectory, Path.GetRandomFileName()); + + for (int attempt = 1; attempt <= MaxDownloadAttempts; attempt++) + { + try + { + using (var input = await client.GetStreamAsync(path, cancellationToken).ConfigureAwait(false)) + { + using var cache = File.Create(tempFile); + await input.CopyToAsync(cache, cancellationToken).ConfigureAwait(false); + } + File.Move(tempFile, cacheFile, true); + return; + } + catch (HttpRequestException httpException) when (httpException.StatusCode == System.Net.HttpStatusCode.NotFound) + { + throw; + } + catch (Exception) when (attempt < MaxDownloadAttempts && !cancellationToken.IsCancellationRequested) + { + await Task.Delay(Random.Shared.Next(500, 5000), cancellationToken).ConfigureAwait(false); + } + finally + { + if (File.Exists(tempFile)) { - await input.CopyToAsync(cache).ConfigureAwait(false); + File.Delete(tempFile); } } } - return DemDataCell.Load(cacheFile); } + public Task Load(string path) => LoadAsync(path, null); + public async Task ReadIndex() { - using (var input = await client.GetStreamAsync("index.json").ConfigureAwait(false)) + var cacheFile = GetCacheFile("index.json"); + + if (!File.Exists(cacheFile) || (DateTime.UtcNow - File.GetLastWriteTimeUtc(cacheFile)) > IndexCacheDuration) + { + await DownloadFile("index.json", cacheFile).ConfigureAwait(false); + } + + using (var stream = File.OpenRead(cacheFile)) + { + return (await JsonSerializer.DeserializeAsync(stream, DemDatabaseIndexContext.Default.DemDatabaseIndex).ConfigureAwait(false))!; + } + } + + public async Task GetSha256Async(string path, CancellationToken cancellationToken = default) + { + var cacheFile = GetCacheFile(path); + try + { + await DownloadFile(path, cacheFile, cancellationToken).ConfigureAwait(false); + } + catch(HttpRequestException httpException) when (httpException.StatusCode == System.Net.HttpStatusCode.NotFound) { - return (await JsonSerializer.DeserializeAsync(input, DemDatabaseIndexContext.Default.DemDatabaseIndex).ConfigureAwait(false))!; + // If the file doesn't exist on the server, we won't be able to get its SHA-256. + return null; } + return await Sha256Helper.ComputeHexAsync(cacheFile, cancellationToken).ConfigureAwait(false); } } } diff --git a/MapToolkit/Databases/IDemStorage.cs b/MapToolkit/Databases/IDemStorage.cs index b79adf1..93b161c 100644 --- a/MapToolkit/Databases/IDemStorage.cs +++ b/MapToolkit/Databases/IDemStorage.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Threading; using System.Threading.Tasks; using Pmad.Cartography.DataCells; @@ -9,5 +10,11 @@ public interface IDemStorage Task ReadIndex(); Task Load(string path); + + Task LoadAsync(string path, string? expectedSha256, CancellationToken cancellationToken = default) + => Load(path); + + Task GetSha256Async(string path, CancellationToken cancellationToken = default) + => Task.FromResult(null); } } \ No newline at end of file diff --git a/MapToolkit/Databases/Sha256Helper.cs b/MapToolkit/Databases/Sha256Helper.cs new file mode 100644 index 0000000..da26ab2 --- /dev/null +++ b/MapToolkit/Databases/Sha256Helper.cs @@ -0,0 +1,35 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Pmad.Cartography.Databases +{ + public static class Sha256Helper + { + public static async Task ComputeHexAsync(string filePath, CancellationToken cancellationToken = default) + { + using var sha256 = SHA256.Create(); + using var stream = File.OpenRead(filePath); + var hash = await sha256.ComputeHashAsync(stream, cancellationToken).ConfigureAwait(false); + return BytesToHex(hash); + } + + public static async Task VerifyAsync(string filePath, string expectedHex, CancellationToken cancellationToken = default) + { + return string.Equals(await ComputeHexAsync(filePath, cancellationToken).ConfigureAwait(false), expectedHex, StringComparison.OrdinalIgnoreCase); + } + + private static string BytesToHex(byte[] bytes) + { + var sb = new StringBuilder(bytes.Length * 2); + foreach (var b in bytes) + { + sb.Append(b.ToString("x2")); + } + return sb.ToString(); + } + } +} diff --git a/MapToolkit/Databases/WellKnownDatabases.cs b/MapToolkit/Databases/WellKnownDatabases.cs index efdc65d..61ec130 100644 --- a/MapToolkit/Databases/WellKnownDatabases.cs +++ b/MapToolkit/Databases/WellKnownDatabases.cs @@ -1,13 +1,11 @@ using System; -using System.Data; using System.Net.Http; -using static System.Runtime.InteropServices.JavaScript.JSType; namespace Pmad.Cartography.Databases { public static class WellKnownDatabases { - private const string DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"; + private const string DefaultUserAgent = "Mozilla/5.0 (Pmad-Cartography; Default)"; internal static HttpClient CreateClient(string baseAddress) {