diff --git a/DemUtility/DemUtility.csproj b/DemUtility/DemUtility.csproj
index f4e5935..538cd2c 100644
--- a/DemUtility/DemUtility.csproj
+++ b/DemUtility/DemUtility.csproj
@@ -2,13 +2,14 @@
Exe
- net8.0
+ net10.0
enable
dem
-
+
+
diff --git a/DemUtility/Program.cs b/DemUtility/Program.cs
index cb3d5c2..37867ee 100644
--- a/DemUtility/Program.cs
+++ b/DemUtility/Program.cs
@@ -1,80 +1,131 @@
using System;
using System.Collections.Generic;
+using System.CommandLine;
using System.IO;
using System.IO.Compression;
+using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
-using CommandLine;
using Pmad.Cartography;
using Pmad.Cartography.Databases;
using Pmad.Cartography.DataCells;
+using Pmad.ProgressTracking;
namespace DemUtility
{
- [Verb("repack", HelpText = "Create a copy of a DEM database with specified compresssion.")]
- internal class RepackOptions
+ internal class Program
{
- [Option('s', "source", Required = true, HelpText = "Source directory.")]
- public string? Source { get; set; }
+ static int Main(string[] args)
+ {
+ var rootCommand = new RootCommand("DEM database utility.");
- [Option('t', "target", Required = true, HelpText = "Target directory.")]
- public string? Target { get; set; }
+ // repack
+ var repackSourceOption = new Option("--source", "-s") { Description = "Source directory.", Required = true };
+ var repackTargetOption = new Option("--target", "-t") { Description = "Target directory.", Required = true };
+ var repackCompressionOption = new Option("--compression", "-c") { Description = "Compression to use: 'GZip', 'ZSTD', 'Brotli', or 'None' (ZSTD by default).", DefaultValueFactory = _ => Compression.ZSTD };
+ var repackMaxCpuOption = new Option("--max-cpu", "-m") { Description = "Number of CPU Cores that can be used for process.", DefaultValueFactory = _ => -1 };
+ var repackKeepOption = new Option("--keep", "-k") { Description = "Keep existing files." };
- [Option('c', "compression", Required = false, HelpText = "Compression to use: 'GZip', 'ZSTD', 'Brotli', or 'None' (ZSTD by default).")]
- public Compression TargetCompression { get; set; } = Compression.ZSTD;
+ var repackCommand = new Command("repack", "Create a copy of a DEM database with specified compression.");
+ repackCommand.Options.Add(repackSourceOption);
+ repackCommand.Options.Add(repackTargetOption);
+ repackCommand.Options.Add(repackCompressionOption);
+ repackCommand.Options.Add(repackMaxCpuOption);
+ repackCommand.Options.Add(repackKeepOption);
+ repackCommand.SetAction(parseResult =>
+ {
+ using var render = ConsoleProgessHelper.Create();
+ return Repack(
+ parseResult.GetValue(repackSourceOption)!,
+ parseResult.GetValue(repackTargetOption)!,
+ parseResult.GetValue(repackCompressionOption),
+ parseResult.GetValue(repackMaxCpuOption),
+ parseResult.GetValue(repackKeepOption),
+ render);
+ });
- [Option('m', "max-cpu", Required = false, HelpText = "Number of CPU Cores that can be used for process.")]
- public int MaxCPU { get; set; } = -1;
+ // index
+ var indexPathOption = new Option("--path", "-p") { Description = "Database directory.", Required = true };
- [Option('k', "keep", Required = false, HelpText = "Keep existing files.")]
- public bool Keep { get; set; }
- }
+ var indexCommand = new Command("index", "Build index.");
+ indexCommand.Options.Add(indexPathOption);
+ indexCommand.SetAction(async parseResult =>
+ {
+ using var render = ConsoleProgessHelper.Create();
+ return await Index(parseResult.GetValue(indexPathOption)!, render);
+ });
- [Verb("index", HelpText = "Build index.")]
- internal class IndexOptions
- {
- [Option('p', "path", Required = true, HelpText = "Database directory.")]
- public string? Source { get; set; }
- }
+ // update-index
+ var updateIndexPathOption = new Option("--path", "-p") { Description = "Database directory.", Required = true };
- internal class Program
- {
- static int Main(string[] args)
- {
- return CommandLine.Parser.Default.ParseArguments(args)
- .MapResult(
- (RepackOptions opts) => Repack(opts),
- (IndexOptions opts) => Index(opts),
- errs => 1);
+ var updateIndexCommand = new Command("update-index", "Add missing SHA-256 checksums to an existing index file.");
+ updateIndexCommand.Options.Add(updateIndexPathOption);
+ updateIndexCommand.SetAction(async parseResult =>
+ {
+ return await UpdateIndex(parseResult.GetValue(updateIndexPathOption)!);
+ });
+
+ // check
+ var checkPathOption = new Option("--path", "-p") { Description = "Local database directory." };
+ var checkUrlOption = new Option("--url", "-u") { Description = "HTTP base URL of the database." };
+ var checkVerifyOption = new Option("--verify", "-v") { Description = "Verify SHA-256 checksums of files (local or downloaded via HTTP)." };
+ var checkSampleOption = new Option("--sample", "-n") { Description = "Number of randomly sampled cells to verify (verifies all cells when omitted)." };
+
+ var checkCommand = new Command("check", "Check a DEM database (local or HTTP) and report its content.");
+ checkCommand.Options.Add(checkPathOption);
+ checkCommand.Options.Add(checkUrlOption);
+ checkCommand.Options.Add(checkVerifyOption);
+ checkCommand.Options.Add(checkSampleOption);
+ checkCommand.SetAction(async parseResult =>
+ {
+ var path = parseResult.GetValue(checkPathOption);
+ var url = parseResult.GetValue(checkUrlOption);
+ var verify = parseResult.GetValue(checkVerifyOption);
+ var sample = parseResult.GetValue(checkSampleOption);
+ if (path == null && url == null)
+ {
+ Console.Error.WriteLine("Either --path or --url must be specified.");
+ return 1;
+ }
+ if (path != null && url != null)
+ {
+ Console.Error.WriteLine("Only one of --path or --url can be specified.");
+ return 1;
+ }
+ if (sample.HasValue && sample.Value <= 0)
+ {
+ Console.Error.WriteLine("--sample must be a positive integer.");
+ return 1;
+ }
+ return await Check(path, url, verify, sample);
+ });
+
+ rootCommand.Subcommands.Add(repackCommand);
+ rootCommand.Subcommands.Add(indexCommand);
+ rootCommand.Subcommands.Add(updateIndexCommand);
+ rootCommand.Subcommands.Add(checkCommand);
+
+ return rootCommand.Parse(args).Invoke();
}
- private static int Index(IndexOptions opts)
+ private static async Task Index(string sourcePath, IProgressScope render)
{
- if (string.IsNullOrEmpty(opts.Source))
+ var source = new DemFileSystemStorage(sourcePath);
+ DemDatabaseIndex index;
+ using (var progress = render.CreatePercent("Build index"))
{
- throw new ArgumentNullException();
+ index = await source.BuildIndexAsync(progress).ConfigureAwait(false);
}
- var source = new DemFileSystemStorage(opts.Source);
- var index = source.BuildIndex();
- using(var file = File.Create(Path.Combine(opts.Source, "index.json")))
+ using (var file = File.Create(Path.Combine(sourcePath, "index.json")))
{
JsonSerializer.Serialize(file, index);
}
return 0;
}
- private static int Repack(RepackOptions opts)
+ private static int Repack(string sourcePath, string targetPath, Compression targetCompression, int maxCPU, bool keep, IProgressScope render)
{
- if (string.IsNullOrEmpty(opts.Source))
- {
- throw new ArgumentNullException();
- }
- if (string.IsNullOrEmpty(opts.Target))
- {
- throw new ArgumentNullException();
- }
-
- var files = Directory.GetFiles(opts.Source, "*.*", SearchOption.AllDirectories);
+ var files = Directory.GetFiles(sourcePath, "*.*", SearchOption.AllDirectories);
var demFiles = new List();
var zipFiles = new List();
@@ -92,25 +143,24 @@ private static int Repack(RepackOptions opts)
}
var parallel = new ParallelOptions();
- if (opts.MaxCPU > 0)
+ if (maxCPU > 0)
{
- parallel.MaxDegreeOfParallelism = opts.MaxCPU;
+ parallel.MaxDegreeOfParallelism = maxCPU;
}
- Directory.CreateDirectory(opts.Target);
+ Directory.CreateDirectory(targetPath);
if (demFiles.Count > 0)
{
- Console.WriteLine($"{demFiles.Count} DEM files to process.");
- using (var report = new ProgressReport("DEM", demFiles.Count))
+ using (var report = render.CreateInteger("DEM", demFiles.Count))
{
Parallel.ForEach(demFiles, parallel, file =>
{
- var filename = CompressionHelper.GetFileName(Path.GetFileName(file)) + CompressionHelper.GetExtension(opts.TargetCompression);
- var target = Path.Combine(opts.Target, filename);
- if (!opts.Keep || !File.Exists(target))
+ var filename = CompressionHelper.GetFileName(Path.GetFileName(file)) + CompressionHelper.GetExtension(targetCompression);
+ var target = Path.Combine(targetPath, filename);
+ if (!keep || !File.Exists(target))
{
- CompressionHelper.Write(target, opts.TargetCompression,
+ CompressionHelper.Write(target, targetCompression,
output => CompressionHelper.Read(file, input => input.CopyTo(output)));
}
report.ReportOneDone();
@@ -120,24 +170,23 @@ private static int Repack(RepackOptions opts)
if (zipFiles.Count > 0)
{
- Console.WriteLine($"{zipFiles.Count} ZIP files to scan.");
- using (var report = new ProgressReport("ZIP", zipFiles.Count))
+ using (var report = render.CreateInteger("ZIP", zipFiles.Count))
{
Parallel.ForEach(zipFiles, parallel, file =>
{
using (var archive = new ZipArchive(File.OpenRead(file), ZipArchiveMode.Read))
{
- foreach(var entry in archive.Entries)
+ foreach (var entry in archive.Entries)
{
if (entry.Name.EndsWith("_DSM.tif", StringComparison.OrdinalIgnoreCase))
{
- var filename = entry.Name + CompressionHelper.GetExtension(opts.TargetCompression);
- var target = Path.Combine(opts.Target, filename);
- if (!opts.Keep || !File.Exists(target))
+ var filename = entry.Name + CompressionHelper.GetExtension(targetCompression);
+ var target = Path.Combine(targetPath, filename);
+ if (!keep || !File.Exists(target))
{
using (var input = entry.Open())
{
- CompressionHelper.Write(target, opts.TargetCompression,
+ CompressionHelper.Write(target, targetCompression,
output => input.CopyTo(output));
}
}
@@ -150,5 +199,157 @@ private static int Repack(RepackOptions opts)
}
return 0;
}
+
+ private static async Task UpdateIndex(string sourcePath)
+ {
+ var indexFile = Path.Combine(sourcePath, "index.json");
+ if (!File.Exists(indexFile))
+ {
+ Console.Error.WriteLine($"No index.json found in '{sourcePath}'. Run 'index' first.");
+ return 1;
+ }
+
+ DemDatabaseIndex index;
+ using (var file = File.OpenRead(indexFile))
+ {
+ index = JsonSerializer.Deserialize(file)!;
+ }
+
+ if (index.Cells.Count(c => c.Sha256 == null) == 0)
+ {
+ Console.WriteLine("All entries already have a SHA-256 checksum. Nothing to do.");
+ return 0;
+ }
+
+ // Backup existing index
+ File.Copy(indexFile, indexFile + $"-{DateTime.Now:yyyyMMddHHmmss}.bak");
+
+ using var render = ConsoleProgessHelper.Create();
+ var updated = new List();
+ using (var report = render.CreateInteger("SHA-256", index.Cells.Count))
+ {
+ foreach (var cell in index.Cells)
+ {
+ if (cell.Sha256 == null)
+ {
+ var fullPath = Path.Combine(sourcePath, cell.Path.Replace('/', Path.DirectorySeparatorChar));
+ var hash = await Sha256Helper.ComputeHexAsync(fullPath).ConfigureAwait(false);
+ updated.Add(new DemDatabaseFileInfos(cell.Path, cell.Metadata, hash));
+ }
+ else
+ {
+ updated.Add(cell);
+ }
+ report.ReportOneDone();
+ }
+ }
+
+ var newIndex = new DemDatabaseIndex(updated);
+ using (var file = File.Create(indexFile))
+ {
+ JsonSerializer.Serialize(file, newIndex);
+ }
+ return 0;
+ }
+ private static async Task Check(string? localPath, string? url, bool verify, int? sample = null)
+ {
+ IDemStorage storage;
+ if (localPath != null)
+ {
+ if (!Directory.Exists(localPath))
+ {
+ Console.Error.WriteLine($"Directory '{localPath}' does not exist.");
+ return 1;
+ }
+ storage = new DemFileSystemStorage(localPath);
+ }
+ else
+ {
+ if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
+ {
+ Console.Error.WriteLine($"Invalid URL '{url}'.");
+ return 1;
+ }
+ storage = new DemHttpStorage(uri);
+ }
+
+ Console.WriteLine("Reading index...");
+ DemDatabaseIndex index;
+ try
+ {
+ index = await storage.ReadIndex();
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine($"Failed to read index: {ex.Message}");
+ return 1;
+ }
+
+ Console.WriteLine($"Cells : {index.Cells.Count}");
+
+ if (index.Cells.Count > 0)
+ {
+ var withSha256 = index.Cells.Count(c => c.Sha256 != null);
+ Console.WriteLine($"SHA-256 : {withSha256}/{index.Cells.Count}");
+
+ var minLat = index.Cells.Min(c => c.Metadata.Start.Latitude);
+ var maxLat = index.Cells.Max(c => c.Metadata.End.Latitude);
+ var minLon = index.Cells.Min(c => c.Metadata.Start.Longitude);
+ var maxLon = index.Cells.Max(c => c.Metadata.End.Longitude);
+ Console.WriteLine($"Coverage : Lat [{minLat:F4} ; {maxLat:F4}], Lon [{minLon:F4} ; {maxLon:F4}]");
+ }
+
+ if (verify)
+ {
+ Console.WriteLine("Verifying checksums...");
+ int errors = 0;
+ int skipped = 0;
+ var cellsWithHash = index.Cells.Where(c => c.Sha256 != null).ToList();
+ IEnumerable cellsToVerify;
+ if (sample.HasValue && sample.Value < cellsWithHash.Count)
+ {
+ var rng = new Random();
+ cellsToVerify = cellsWithHash.OrderBy(_ => rng.Next()).Take(sample.Value).ToList();
+ Console.WriteLine($" Sampling {sample.Value}/{cellsWithHash.Count} cells with a checksum.");
+ }
+ else
+ {
+ cellsToVerify = cellsWithHash;
+ }
+ skipped = index.Cells.Count - cellsWithHash.Count;
+ using (var render = ConsoleProgessHelper.Create())
+ {
+ foreach (var cell in cellsToVerify.WithProgress(render, "Checksum"))
+ {
+ var actualHash = await storage.GetSha256Async(cell.Path).ConfigureAwait(false);
+ if (actualHash == null)
+ {
+ render.WriteLine($" MISSING: {cell.Path}");
+ errors++;
+ }
+ else if (!string.Equals(actualHash, cell.Sha256, StringComparison.OrdinalIgnoreCase))
+ {
+ render.WriteLine($" INVALID: Actual:'{actualHash}' Expected:'{cell.Sha256}'");
+ errors++;
+ }
+ }
+ }
+ if (skipped > 0)
+ {
+ Console.WriteLine($" Skipped {skipped} cell(s) without a stored checksum.");
+ }
+ if (errors == 0)
+ {
+ Console.WriteLine($" All checksums OK.");
+ }
+ else
+ {
+ Console.Error.WriteLine($" {errors} error(s) found.");
+ return 1;
+ }
+ }
+
+ return 0;
+ }
}
}
\ No newline at end of file
diff --git a/DemUtility/ProgressReport.cs b/DemUtility/ProgressReport.cs
deleted file mode 100644
index 1bd34d1..0000000
--- a/DemUtility/ProgressReport.cs
+++ /dev/null
@@ -1,118 +0,0 @@
-using System;
-using System.Diagnostics;
-using System.Threading;
-
-namespace DemUtility
-{
- internal class ProgressReport : IDisposable
- {
- private readonly string taskName;
- private readonly int itemsToDo;
- private readonly Stopwatch sw;
- private readonly Stopwatch lastReport;
- private readonly object locker = new object();
- private int lastDone = 0;
-
- public ProgressReport(string taskName, int itemsToDo = 1)
- {
- this.taskName = taskName;
- this.itemsToDo = itemsToDo;
- this.sw = Stopwatch.StartNew();
- this.lastReport = Stopwatch.StartNew();
-
- Trace.WriteLine(string.Empty);
- Trace.WriteLine($"Begin task {taskName}");
- Console.Write(taskName);
- WritePercent(0);
- }
-
- public int Total
- {
- get { return itemsToDo; }
- }
-
- public void ReportOneDone()
- {
- Interlocked.Increment(ref lastDone);
- DrawDone(lastDone);
- }
-
- public void ReportItemsDone(int done)
- {
- lastDone = done;
- DrawDone(done);
- }
-
- private void DrawDone(int done)
- {
- if (lastReport.ElapsedMilliseconds > 500)
- {
- lock (locker)
- {
- if (lastReport.ElapsedMilliseconds > 500)
- {
- lastReport.Restart();
- WritePercent(done * 100.0 / itemsToDo);
-
- if (done > 0)
- {
- var milisecondsLeft = sw.ElapsedMilliseconds * (itemsToDo - done) / done;
- if (milisecondsLeft > 120000d)
- {
- Console.Write($"{Math.Round(milisecondsLeft / 60000d)} min left");
- }
- else
- {
- Console.Write($"{Math.Ceiling(milisecondsLeft / 1000d)} sec left");
- }
- }
- CleanEndOfLine();
- }
- }
- }
- }
-
- private void WritePercent(double percent)
- {
- if (!Console.IsOutputRedirected)
- {
- var cols = Math.Max(0, Math.Min(20, (int)(percent / 5)));
- Console.CursorLeft = 20;
- Console.ForegroundColor = ConsoleColor.Green;
- Console.Write(new string('#', cols));
- Console.ForegroundColor = ConsoleColor.Gray;
- Console.Write(new string('-', 20 - cols));
- Console.Write(' ');
- Console.Write($"{percent,6:0.00} % ");
- }
- }
-
- public void TaskDone()
- {
- if (Console.IsOutputRedirected)
- {
- Console.Write(' ');
- }
- sw.Stop();
- WritePercent(100d);
- Console.Write($"Done in {Math.Ceiling(sw.ElapsedMilliseconds / 1000d)} sec");
- CleanEndOfLine();
- Console.WriteLine();
- Trace.WriteLine($"Task {taskName} took {sw.ElapsedMilliseconds} msec");
- Trace.Flush();
- }
-
- private static void CleanEndOfLine()
- {
- if (!Console.IsOutputRedirected)
- {
- Console.Write(new string(' ', Console.BufferWidth - Console.CursorLeft - 1));
- }
- }
-
- public void Dispose()
- {
- TaskDone();
- }
- }
-}
diff --git a/DemUtility/Properties/launchSettings.json b/DemUtility/Properties/launchSettings.json
index 68aaa3a..16e899c 100644
--- a/DemUtility/Properties/launchSettings.json
+++ b/DemUtility/Properties/launchSettings.json
@@ -1,6 +1,6 @@
{
"profiles": {
- "Repack STRM": {
+ "Repack SRTM": {
"commandName": "Project",
"commandLineArgs": "repack -k -s E:\\Carto\\SRTMv3 -t c:\\temp\\SRTMv3 -m 10"
},
@@ -8,13 +8,57 @@
"commandName": "Project",
"commandLineArgs": "repack -k -s E:\\Carto\\AW3D30 -t c:\\temp\\AW3D30 -m 10"
},
- "Index STRM": {
+
+ "Index AW3D30": {
"commandName": "Project",
- "commandLineArgs": "index -p C:\\temp\\SRTMv3"
+ "commandLineArgs": "index -p D:\\dem\\AW3D30"
},
- "Index AW3D30": {
+ "Index SRTM15Plus": {
+ "commandName": "Project",
+ "commandLineArgs": "index -p D:\\dem\\SRTM15Plus"
+ },
+ "Index SRTM1": {
+ "commandName": "Project",
+ "commandLineArgs": "index -p D:\\dem\\SRTM1"
+ },
+
+ "Update Index AW3D30": {
+ "commandName": "Project",
+ "commandLineArgs": "update-index -p D:\\dem\\AW3D30"
+ },
+ "Update Index SRTM15Plus": {
+ "commandName": "Project",
+ "commandLineArgs": "update-index -p D:\\dem\\SRTM15Plus"
+ },
+ "Update Index SRTM1": {
+ "commandName": "Project",
+ "commandLineArgs": "update-index -p D:\\dem\\SRTM1"
+ },
+
+ "Check AW3D30": {
+ "commandName": "Project",
+ "commandLineArgs": "check -v -p D:\\dem\\AW3D30"
+ },
+ "Check SRTM15Plus": {
+ "commandName": "Project",
+ "commandLineArgs": "check -v -p D:\\dem\\SRTM15Plus"
+ },
+ "Check SRTM1": {
+ "commandName": "Project",
+ "commandLineArgs": "check -v -p D:\\dem\\SRTM1"
+ },
+
+ "Check CDN AW3D30": {
+ "commandName": "Project",
+ "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/AW3D30/ -n 100"
+ },
+ "Check CDN SRTM15Plus": {
+ "commandName": "Project",
+ "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/SRTM15Plus/ -n 100"
+ },
+ "Check CDN SRTM1": {
"commandName": "Project",
- "commandLineArgs": "index -p C:\\temp\\AW3D30"
+ "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/SRTM1/ -n 100"
}
}
}
\ No newline at end of file
diff --git a/MapToolkit.Test/CollectionDefinitions.cs b/MapToolkit.Test/CollectionDefinitions.cs
new file mode 100644
index 0000000..67dd774
--- /dev/null
+++ b/MapToolkit.Test/CollectionDefinitions.cs
@@ -0,0 +1,5 @@
+namespace Pmad.Cartography.Test
+{
+ [CollectionDefinition("Sequential", DisableParallelization = true)]
+ public class SequentialCollectionDefinition { }
+}
diff --git a/MapToolkit.Test/CompressionHelperTest.cs b/MapToolkit.Test/CompressionHelperTest.cs
index f7327c2..8a67835 100644
--- a/MapToolkit.Test/CompressionHelperTest.cs
+++ b/MapToolkit.Test/CompressionHelperTest.cs
@@ -22,9 +22,13 @@ public void GetExtension_FromFilename()
public void GetExtension_FromCompression()
{
Assert.Equal(".zst", CompressionHelper.GetExtension(Compression.ZSTD));
- Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZib));
+ Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZip));
Assert.Equal(".bt", CompressionHelper.GetExtension(Compression.Brotli));
Assert.Equal(string.Empty, CompressionHelper.GetExtension(Compression.None));
+
+#pragma warning disable CS0618 // Le type ou le membre est obsolète
+ Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZib));
+#pragma warning restore CS0618 // Le type ou le membre est obsolète
}
[Fact]
@@ -40,7 +44,7 @@ public void GetFileName_RemovesExtension()
[Fact]
public void ReadSeekable_ReadsCompressedFile_Compressed()
{
- var filename = WriteCompressedFile(Compression.GZib);
+ var filename = WriteCompressedFile(Compression.GZip);
var result = CompressionHelper.ReadSeekable(filename, stream =>
{
using (var reader = new StreamReader(stream))
@@ -68,9 +72,9 @@ public void ReadSeekable_ReadsCompressedFile_None()
}
[Fact]
- public void Read_ReadsCompressedFile_GZib()
+ public void Read_ReadsCompressedFile_GZip()
{
- var filename = WriteCompressedFile(Compression.GZib);
+ var filename = WriteCompressedFile(Compression.GZip);
var result = CompressionHelper.Read(filename, stream =>
{
using (var reader = new StreamReader(stream))
@@ -130,7 +134,7 @@ public void Read_ReadsCompressedFile_Brotli()
[Fact]
public void GetSize_ReturnsCorrectSize_GZip()
{
- var filename = WriteCompressedFile(Compression.GZib);
+ var filename = WriteCompressedFile(Compression.GZip);
var size = CompressionHelper.GetSize(filename);
Assert.Equal(12, size); // "test content" length
File.Delete(filename);
diff --git a/MapToolkit.Test/Databases/DemDatabaseEntryTest.cs b/MapToolkit.Test/Databases/DemDatabaseEntryTest.cs
index f0c0c3c..7d624a2 100644
--- a/MapToolkit.Test/Databases/DemDatabaseEntryTest.cs
+++ b/MapToolkit.Test/Databases/DemDatabaseEntryTest.cs
@@ -40,7 +40,7 @@ public async Task DemDatabaseEntry_Load()
var cache = new MemoryCache(new MemoryCacheOptions());
var dataCell = new Mock();
dataCell.Setup(d => d.SizeInBytes).Returns(100);
- storage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell.Object);
+ storage.Setup(s => s.LoadAsync(It.IsAny(), null, default)).ReturnsAsync(dataCell.Object);
var entry = new DemDatabaseEntry("test", metadata.Object);
var result = await entry.Load(storage.Object, cache);
diff --git a/MapToolkit.Test/Databases/DemDatabaseTest.cs b/MapToolkit.Test/Databases/DemDatabaseTest.cs
index 09da847..abce7cb 100644
--- a/MapToolkit.Test/Databases/DemDatabaseTest.cs
+++ b/MapToolkit.Test/Databases/DemDatabaseTest.cs
@@ -39,7 +39,7 @@ public async Task GetDataCellsAsync_ShouldReturnDataCells()
var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100));
var dataCell = new Mock().Object;
mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List { entry }));
- mockStorage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell);
+ mockStorage.Setup(s => s.LoadAsync(It.IsAny(), It.IsAny(), default)).ReturnsAsync(dataCell);
// Act
var result = await demDatabase.GetDataCellsAsync(start, end);
@@ -60,7 +60,7 @@ public async Task CreateView_ShouldReturnDemDataView()
var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100));
var dataCell = new DemDataCellPixelIsPoint(new Coordinates(0, 0), new Coordinates(1, 1), new float[100, 100]);
mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List { entry }));
- mockStorage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell);
+ mockStorage.Setup(s => s.LoadAsync(It.IsAny(), It.IsAny(), default)).ReturnsAsync(dataCell);
// Act
var result = await demDatabase.CreateView(start, end);
@@ -119,7 +119,7 @@ public void GetElevation_ShouldReturnElevation()
var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100));
var dataCell = new Mock().Object;
mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List { entry }));
- mockStorage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell);
+ mockStorage.Setup(s => s.LoadAsync(It.IsAny(), It.IsAny(), default)).ReturnsAsync(dataCell);
// Act
var result = demDatabase.GetElevation(coordinates, interpolation);
@@ -140,7 +140,7 @@ public async Task GetElevationAsync_ShouldReturnElevation()
var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100));
var dataCell = new DemDataCellPixelIsPoint(new Coordinates(0, 0), new Coordinates(1, 1), new float[100, 100]);
mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List { entry }));
- mockStorage.Setup(s => s.Load(It.IsAny())).ReturnsAsync(dataCell);
+ mockStorage.Setup(s => s.LoadAsync(It.IsAny(), It.IsAny(), default)).ReturnsAsync(dataCell);
// Act
var result = await demDatabase.GetElevationAsync(coordinates, interpolation);
diff --git a/MapToolkit.Test/Databases/DemFileSystemStorageTest.cs b/MapToolkit.Test/Databases/DemFileSystemStorageTest.cs
new file mode 100644
index 0000000..6cd3ea3
--- /dev/null
+++ b/MapToolkit.Test/Databases/DemFileSystemStorageTest.cs
@@ -0,0 +1,88 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+using System.Threading.Tasks;
+using Pmad.Cartography.Databases;
+using Xunit;
+
+namespace Pmad.Cartography.Test.Databases
+{
+ public class DemFileSystemStorageTest
+ {
+ private static string CreateTempFile(byte[] content)
+ {
+ var path = Path.GetTempFileName();
+ File.WriteAllBytes(path, content);
+ return path;
+ }
+
+ private static string ComputeExpectedHex(byte[] content)
+ {
+ return Convert.ToHexString(SHA256.HashData(content)).ToLowerInvariant();
+ }
+
+ [Fact]
+ public async Task GetSha256Async_ShouldReturnCorrectHash_WhenFileExists()
+ {
+ // Arrange
+ var content = "hello dem"u8.ToArray();
+ var file = CreateTempFile(content);
+ var basePath = Path.GetDirectoryName(file)!;
+ var fileName = Path.GetFileName(file);
+ var storage = new DemFileSystemStorage(basePath);
+
+ try
+ {
+ // Act
+ var hash = await storage.GetSha256Async(fileName);
+
+ // Assert
+ Assert.NotNull(hash);
+ Assert.Equal(ComputeExpectedHex(content), hash);
+ }
+ finally
+ {
+ File.Delete(file);
+ }
+ }
+
+ [Fact]
+ public async Task GetSha256Async_ShouldReturnNull_WhenFileDoesNotExist()
+ {
+ // Arrange
+ var basePath = Path.GetTempPath();
+ var storage = new DemFileSystemStorage(basePath);
+
+ // Act
+ var hash = await storage.GetSha256Async("nonexistent_file_that_does_not_exist.hgt.zst");
+
+ // Assert
+ Assert.Null(hash);
+ }
+
+ [Fact]
+ public async Task GetSha256Async_ShouldReturnLowercaseHex()
+ {
+ // Arrange
+ var content = "case check"u8.ToArray();
+ var file = CreateTempFile(content);
+ var basePath = Path.GetDirectoryName(file)!;
+ var fileName = Path.GetFileName(file);
+ var storage = new DemFileSystemStorage(basePath);
+
+ try
+ {
+ // Act
+ var hash = await storage.GetSha256Async(fileName);
+
+ // Assert
+ Assert.NotNull(hash);
+ Assert.Equal(hash!.ToLowerInvariant(), hash);
+ }
+ finally
+ {
+ File.Delete(file);
+ }
+ }
+ }
+}
diff --git a/MapToolkit.Test/Databases/DemHttpStorageTest.cs b/MapToolkit.Test/Databases/DemHttpStorageTest.cs
index 651a4ce..2046d92 100644
--- a/MapToolkit.Test/Databases/DemHttpStorageTest.cs
+++ b/MapToolkit.Test/Databases/DemHttpStorageTest.cs
@@ -1,25 +1,26 @@
using System;
using System.IO;
+using System.Linq;
using System.Net.Http;
+using System.Security.Cryptography;
using System.Threading.Tasks;
-using Xunit;
using Pmad.Cartography.Databases;
-using Pmad.Cartography.DataCells;
-using System.Linq;
namespace Pmad.Cartography.Test.Databases
{
public class DemHttpStorageTest
{
- private const string userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0";
+ private const string userAgent = "Mozilla/5.0 (Pmad-Cartography; UnitTests)";
private const string baseAddress = "https://cdn.dem.pmad.net/SRTM1/";
private const string samplePath = "N00E006.SRTMGL1.hgt.zst";
+ private static string CreateUniqueTempDir() => Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+
[Fact]
public async Task Load_ShouldDownloadAndCacheFile()
{
// Arrange
- var localCache = Path.Combine(Path.GetTempPath(), "dem_test_cache");
+ var localCache = CreateUniqueTempDir();
var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) };
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
var storage = new DemHttpStorage(localCache, httpClient);
@@ -37,11 +38,83 @@ public async Task Load_ShouldDownloadAndCacheFile()
Assert.True(File.Exists(cacheFile));
}
+ [Fact]
+ public async Task Load_ShouldUseCachedFile_OnSecondCall()
+ {
+ // Arrange
+ var localCache = CreateUniqueTempDir();
+ var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) };
+ httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
+ var storage = new DemHttpStorage(localCache, httpClient);
+ var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", samplePath);
+
+ // Ensure file is already cached
+ await storage.Load(samplePath);
+ var firstWriteTime = File.GetLastWriteTimeUtc(cacheFile);
+
+ // Act
+ await storage.Load(samplePath);
+ var secondWriteTime = File.GetLastWriteTimeUtc(cacheFile);
+
+ // Assert: cached file was not re-written
+ Assert.Equal(firstWriteTime, secondWriteTime);
+ }
+
+ [Fact]
+ public async Task Load_ShouldDeleteAndRedownload_WhenChecksumMismatch()
+ {
+ // Arrange
+ var localCache = CreateUniqueTempDir();
+ var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) };
+ httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
+ var storage = new DemHttpStorage(localCache, httpClient);
+ var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", samplePath);
+
+ // Pre-create a corrupted cache file
+ Directory.CreateDirectory(Path.GetDirectoryName(cacheFile)!);
+ File.WriteAllBytes(cacheFile, new byte[] { 0x00, 0x01 });
+
+ // Act: loading with a wrong checksum should re-download, fail validation, delete the file and throw
+ await Assert.ThrowsAsync(() =>
+ storage.LoadAsync(samplePath, "0000000000000000000000000000000000000000000000000000000000000000"));
+
+ // The file should have been deleted after the checksum mismatch
+ Assert.False(File.Exists(cacheFile));
+ }
+
+ [Fact]
+ public async Task LoadAsync_ShouldNotThrow_WhenChecksumMatches()
+ {
+ // Arrange
+ var localCache = CreateUniqueTempDir();
+ var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) };
+ httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
+ var storage = new DemHttpStorage(localCache, httpClient);
+ var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", samplePath);
+ if (File.Exists(cacheFile))
+ {
+ File.Delete(cacheFile);
+ }
+
+ // Download once without checksum to get the real file
+ await storage.Load(samplePath);
+
+ // Compute the real checksum
+ using var cacheFileStream = File.OpenRead(cacheFile);
+ var realHash = Convert.ToHexString(SHA256.HashData(cacheFileStream)).ToLowerInvariant();
+
+ // Act: second load with correct checksum should succeed
+ var dataCell = await storage.LoadAsync(samplePath, realHash);
+
+ // Assert
+ Assert.NotNull(dataCell);
+ }
+
[Fact]
public async Task ReadIndex_ShouldDownloadAndDeserializeIndex()
{
// Arrange
- var localCache = Path.Combine(Path.GetTempPath(), "dem_test_cache");
+ var localCache = CreateUniqueTempDir();
var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) };
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
var storage = new DemHttpStorage(localCache, httpClient);
@@ -58,5 +131,236 @@ public async Task ReadIndex_ShouldDownloadAndDeserializeIndex()
Assert.Equal(new(0,6),cell.Metadata.Start);
Assert.Equal(new(1,7),cell.Metadata.End);
}
+
+ [Fact]
+ public async Task ReadIndex_ShouldUseCachedIndex_WhenFresh()
+ {
+ // Arrange
+ var localCache = CreateUniqueTempDir();
+ var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) };
+ httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
+ var storage = new DemHttpStorage(localCache, httpClient);
+ var indexCacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", "index.json");
+
+ // Ensure a fresh cached index exists
+ await storage.ReadIndex();
+ var firstWriteTime = File.GetLastWriteTimeUtc(indexCacheFile);
+
+ // Act: second call should use the cache
+ await storage.ReadIndex();
+ var secondWriteTime = File.GetLastWriteTimeUtc(indexCacheFile);
+
+ // Assert: the cache file was not re-written
+ Assert.Equal(firstWriteTime, secondWriteTime);
+ }
+
+ [Fact]
+ public async Task ReadIndex_ShouldRefreshCache_WhenExpired()
+ {
+ // Arrange
+ var localCache = CreateUniqueTempDir();
+ var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) };
+ httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
+ var storage = new DemHttpStorage(localCache, httpClient);
+ var indexCacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", "index.json");
+
+ // Create a stale cache file
+ Directory.CreateDirectory(Path.GetDirectoryName(indexCacheFile)!);
+ File.WriteAllText(indexCacheFile, "{\"cells\":[]}");
+ File.SetLastWriteTimeUtc(indexCacheFile, DateTime.UtcNow - DemHttpStorage.IndexCacheDuration - TimeSpan.FromSeconds(1));
+
+ // Act: should re-download because the cache is expired
+ var index = await storage.ReadIndex();
+ var writeTime = File.GetLastWriteTimeUtc(indexCacheFile);
+
+ // Assert: file was refreshed
+ Assert.True(writeTime > DateTime.UtcNow - TimeSpan.FromMinutes(1));
+ Assert.NotEmpty(index.Cells);
+ }
+
+ [Fact]
+ public async Task GetSha256Async_ShouldDownloadFileAndReturnHash()
+ {
+ // Arrange
+ var localCache = CreateUniqueTempDir();
+ var httpClient = new HttpClient { BaseAddress = new Uri(baseAddress) };
+ httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
+ var storage = new DemHttpStorage(localCache, httpClient);
+ var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", samplePath);
+ if (File.Exists(cacheFile))
+ {
+ File.Delete(cacheFile);
+ }
+
+ // Act
+ var hash = await storage.GetSha256Async(samplePath);
+
+ // Assert
+ Assert.NotNull(hash);
+ Assert.Equal(64, hash.Length); // SHA-256 hex is 64 chars
+ using var cacheFileStream = File.OpenRead(cacheFile);
+ var expectedHash = Convert.ToHexString(SHA256.HashData(cacheFileStream)).ToLowerInvariant();
+ Assert.Equal(expectedHash, hash);
+ }
+
+ [Fact]
+ public async Task GetSha256Async_ShouldReturnNull_WhenFileNotFound()
+ {
+ // Arrange
+ var localCache = CreateUniqueTempDir();
+ var handler = new NotFoundHttpMessageHandler();
+ var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) };
+ var storage = new DemHttpStorage(localCache, httpClient);
+
+ // Act
+ var hash = await storage.GetSha256Async("nonexistent_file.hgt.zst");
+
+ // Assert
+ Assert.Null(hash);
+ }
+
+ [Fact]
+ public async Task DownloadFile_ShouldSucceed_AfterTransientFailures()
+ {
+ // Arrange: fail twice, then succeed on the third attempt
+ var localCache = CreateUniqueTempDir();
+ var content = new byte[] { 1, 2, 3, 4 };
+ var handler = new CountingHttpMessageHandler(failCount: 2, successContent: content);
+ var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) };
+ var storage = new DemHttpStorage(localCache, httpClient);
+ var cacheFile = Path.Combine(localCache, "cdn.dem.pmad.net", "SRTM1", "retry_test.bin");
+ if (File.Exists(cacheFile)) File.Delete(cacheFile);
+
+ // Act
+ var hash = await storage.GetSha256Async("retry_test.bin");
+
+ // Assert: succeeded after retries, file exists with correct content
+ Assert.NotNull(hash);
+ Assert.True(File.Exists(cacheFile));
+ Assert.Equal(3, handler.CallCount);
+ }
+
+ [Fact]
+ public async Task DownloadFile_ShouldThrow_WhenAllAttemptsExhausted()
+ {
+ // Arrange: always fail with a transient error
+ var localCache = CreateUniqueTempDir();
+ var handler = new CountingHttpMessageHandler(failCount: 99, successContent: Array.Empty());
+ var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) };
+ var storage = new DemHttpStorage(localCache, httpClient);
+
+ // Act & Assert: should throw after MaxDownloadAttempts (3) attempts
+ await Assert.ThrowsAsync(() => storage.GetSha256Async("always_failing.bin"));
+ Assert.Equal(3, handler.CallCount);
+ }
+
+ [Fact]
+ public async Task DownloadFile_ShouldNotRetry_On404()
+ {
+ // Arrange
+ var localCache = CreateUniqueTempDir();
+ var handler = new NotFoundHttpMessageHandler();
+ var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) };
+ var storage = new DemHttpStorage(localCache, httpClient);
+
+ // Act
+ var hash = await storage.GetSha256Async("missing.bin");
+
+ // Assert: returns null immediately without retry
+ Assert.Null(hash);
+ Assert.Equal(1, handler.CallCount);
+ }
+
+ [Fact]
+ public async Task DownloadFile_ShouldNotRetry_WhenCancelled()
+ {
+ // Arrange: fail once, then cancel — should not retry
+ var localCache = CreateUniqueTempDir();
+ using var cts = new System.Threading.CancellationTokenSource();
+ var handler = new CancellingHttpMessageHandler(cts, failCount: 1, successContent: Array.Empty());
+ var httpClient = new HttpClient(handler) { BaseAddress = new Uri(baseAddress) };
+ var storage = new DemHttpStorage(localCache, httpClient);
+
+ // Act & Assert
+ await Assert.ThrowsAnyAsync(() =>
+ storage.GetSha256Async("cancel_test.bin", cts.Token));
+ Assert.Equal(1, handler.CallCount);
+ }
+
+ private sealed class NotFoundHttpMessageHandler : HttpMessageHandler
+ {
+ public int CallCount { get; private set; }
+
+ protected override Task SendAsync(HttpRequestMessage request, System.Threading.CancellationToken cancellationToken)
+ {
+ CallCount++;
+ return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.NotFound));
+ }
+ }
+
+ ///
+ /// Returns an HTTP error for the first calls, then returns success with the given content.
+ ///
+ private sealed class CountingHttpMessageHandler : HttpMessageHandler
+ {
+ private readonly int failCount;
+ private readonly byte[] successContent;
+
+ public int CallCount { get; private set; }
+
+ public CountingHttpMessageHandler(int failCount, byte[] successContent)
+ {
+ this.failCount = failCount;
+ this.successContent = successContent;
+ }
+
+ protected override Task SendAsync(HttpRequestMessage request, System.Threading.CancellationToken cancellationToken)
+ {
+ CallCount++;
+ if (CallCount <= failCount)
+ {
+ return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError));
+ }
+ var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
+ {
+ Content = new System.Net.Http.ByteArrayContent(successContent)
+ };
+ return Task.FromResult(response);
+ }
+ }
+
+ ///
+ /// Cancels the token after failing calls, so the retry path sees a cancelled token.
+ ///
+ private sealed class CancellingHttpMessageHandler : HttpMessageHandler
+ {
+ private readonly System.Threading.CancellationTokenSource cts;
+ private readonly int failCount;
+ private readonly byte[] successContent;
+
+ public int CallCount { get; private set; }
+
+ public CancellingHttpMessageHandler(System.Threading.CancellationTokenSource cts, int failCount, byte[] successContent)
+ {
+ this.cts = cts;
+ this.failCount = failCount;
+ this.successContent = successContent;
+ }
+
+ protected override Task SendAsync(HttpRequestMessage request, System.Threading.CancellationToken cancellationToken)
+ {
+ CallCount++;
+ if (CallCount <= failCount)
+ {
+ cts.Cancel();
+ return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError));
+ }
+ var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
+ {
+ Content = new System.Net.Http.ByteArrayContent(successContent)
+ };
+ return Task.FromResult(response);
+ }
+ }
}
}
diff --git a/MapToolkit.Test/Databases/Sha256HelperTest.cs b/MapToolkit.Test/Databases/Sha256HelperTest.cs
new file mode 100644
index 0000000..ad755d9
--- /dev/null
+++ b/MapToolkit.Test/Databases/Sha256HelperTest.cs
@@ -0,0 +1,87 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+using System.Threading.Tasks;
+using Pmad.Cartography.Databases;
+
+namespace Pmad.Cartography.Test.Databases
+{
+ public class Sha256HelperTest
+ {
+ private static string CreateTempFileWithContent(byte[] content)
+ {
+ var path = Path.GetTempFileName();
+ File.WriteAllBytes(path, content);
+ return path;
+ }
+
+ private static string ComputeExpectedHex(byte[] content)
+ {
+ var hash = SHA256.HashData(content);
+ return Convert.ToHexString(hash).ToLowerInvariant();
+ }
+
+ [Fact]
+ public async Task ComputeHex_ReturnsCorrectHash()
+ {
+ var content = "hello world"u8.ToArray();
+ var file = CreateTempFileWithContent(content);
+ try
+ {
+ var expected = ComputeExpectedHex(content);
+ Assert.Equal(expected, await Sha256Helper.ComputeHexAsync(file));
+ }
+ finally
+ {
+ File.Delete(file);
+ }
+ }
+
+ [Fact]
+ public async Task VerifyAsync_ReturnsTrue_WhenHashMatches()
+ {
+ var content = "hello world"u8.ToArray();
+ var file = CreateTempFileWithContent(content);
+ try
+ {
+ var hex = ComputeExpectedHex(content);
+ Assert.True(await Sha256Helper.VerifyAsync(file, hex));
+ }
+ finally
+ {
+ File.Delete(file);
+ }
+ }
+
+ [Fact]
+ public async Task VerifyAsync_ReturnsTrue_WhenHashMatchesUpperCase()
+ {
+ var content = "hello world"u8.ToArray();
+ var file = CreateTempFileWithContent(content);
+ try
+ {
+ var hex = ComputeExpectedHex(content).ToUpperInvariant();
+ Assert.True(await Sha256Helper.VerifyAsync(file, hex));
+ }
+ finally
+ {
+ File.Delete(file);
+ }
+ }
+
+ [Fact]
+ public async Task VerifyAsync_ReturnsFalse_WhenHashDoesNotMatch()
+ {
+ var content = "hello world"u8.ToArray();
+ var file = CreateTempFileWithContent(content);
+ try
+ {
+ Assert.False(await Sha256Helper.VerifyAsync(file, "0000000000000000000000000000000000000000000000000000000000000000"));
+ }
+ finally
+ {
+ File.Delete(file);
+ }
+ }
+ }
+}
diff --git a/MapToolkit.Test/Databases/WellKnownDatabasesTest.cs b/MapToolkit.Test/Databases/WellKnownDatabasesTest.cs
index a1df485..eeeaac8 100644
--- a/MapToolkit.Test/Databases/WellKnownDatabasesTest.cs
+++ b/MapToolkit.Test/Databases/WellKnownDatabasesTest.cs
@@ -3,8 +3,14 @@
namespace Pmad.Cartography.Test.Databases
{
+ [Collection("Sequential")]
public class WellKnownDatabasesTest
{
+ public WellKnownDatabasesTest()
+ {
+ DemHttpStorage.ClearDefaultCache();
+ }
+
[Fact]
public async Task AW3D30_ContainsLondon()
{
diff --git a/MapToolkit/Compression.cs b/MapToolkit/Compression.cs
index e684650..ee697ad 100644
--- a/MapToolkit/Compression.cs
+++ b/MapToolkit/Compression.cs
@@ -1,10 +1,14 @@
-namespace Pmad.Cartography
+using System;
+
+namespace Pmad.Cartography
{
public enum Compression
{
- None,
- ZSTD,
- GZib,
- Brotli
+ None = 0,
+ ZSTD = 1,
+ GZip = 2,
+ [Obsolete("Use GZip instead.")]
+ GZib = 2,
+ Brotli = 3
}
}
\ No newline at end of file
diff --git a/MapToolkit/CompressionHelper.cs b/MapToolkit/CompressionHelper.cs
index 20f5199..b796294 100644
--- a/MapToolkit/CompressionHelper.cs
+++ b/MapToolkit/CompressionHelper.cs
@@ -22,7 +22,7 @@ public static string GetExtension(Compression compression)
{
case Compression.ZSTD:
return ExtensionZStd;
- case Compression.GZib:
+ case Compression.GZip:
return ExtensionGZip;
case Compression.Brotli:
return ExtensionBrotli;
@@ -191,7 +191,7 @@ public static void Write(string target, Compression compression, Action
write(compressed);
}
break;
- case Compression.GZib:
+ case Compression.GZip:
using (var compressed = new GZipStream(stream, CompressionMode.Compress))
{
write(compressed);
diff --git a/MapToolkit/DataCells/DemDataCellMetadata.cs b/MapToolkit/DataCells/DemDataCellMetadata.cs
index 725c9dc..f099655 100644
--- a/MapToolkit/DataCells/DemDataCellMetadata.cs
+++ b/MapToolkit/DataCells/DemDataCellMetadata.cs
@@ -49,8 +49,10 @@ internal static Coordinates EndFromResolution(Coordinates start, DemRasterType t
public Coordinates End { get; }
+ [JsonPropertyName("PointsPerCellLat")] // Legacy name, kept for backward compatibility
public int PointsLat { get; }
+ [JsonPropertyName("PointsPerCellLon")] // Legacy name, kept for backward compatibility
public int PointsLon { get; }
}
}
diff --git a/MapToolkit/Databases/DemDatabase.cs b/MapToolkit/Databases/DemDatabase.cs
index 88fcac3..f857e3a 100644
--- a/MapToolkit/Databases/DemDatabase.cs
+++ b/MapToolkit/Databases/DemDatabase.cs
@@ -51,7 +51,7 @@ private async Task LoadIndexInternal()
entry.UnLoad(cache);
}
entries.Clear();
- entries.AddRange((await storage.ReadIndex().ConfigureAwait(false)).Cells.Select(i => new DemDatabaseEntry(i.Path, i.Metadata)));
+ entries.AddRange((await storage.ReadIndex().ConfigureAwait(false)).Cells.Select(i => new DemDatabaseEntry(i.Path, i.Metadata, i.Sha256)));
}
private async Task EnsureIndexIsLoadedAsync()
diff --git a/MapToolkit/Databases/DemDatabaseEntry.cs b/MapToolkit/Databases/DemDatabaseEntry.cs
index 9edf742..3b72e42 100644
--- a/MapToolkit/Databases/DemDatabaseEntry.cs
+++ b/MapToolkit/Databases/DemDatabaseEntry.cs
@@ -7,16 +7,19 @@ namespace Pmad.Cartography.Databases
{
internal class DemDatabaseEntry
{
- internal DemDatabaseEntry(string path, IDemDataCellMetadata metadata)
+ internal DemDatabaseEntry(string path, IDemDataCellMetadata metadata, string? sha256 = null)
{
Path = path;
Metadata = metadata;
+ Sha256 = sha256;
}
public string Path { get; }
public IDemDataCellMetadata Metadata { get; }
+ public string? Sha256 { get; }
+
public bool Contains(Coordinates coordinates)
{
return Metadata.Start.Latitude <= coordinates.Latitude &&
@@ -38,7 +41,7 @@ public async Task Load(IDemStorage storage, IMemoryCache cache)
if (!cache.TryGetValue(this, out IDemDataCell? result) || result == null)
{
using var entry = cache.CreateEntry(this);
- result = await storage.Load(Path).ConfigureAwait(false);
+ result = await storage.LoadAsync(Path, Sha256).ConfigureAwait(false);
entry.Value = result;
entry.Size = result.SizeInBytes;
}
diff --git a/MapToolkit/Databases/DemDatabaseFileInfos.cs b/MapToolkit/Databases/DemDatabaseFileInfos.cs
index 8b7d73f..ac26865 100644
--- a/MapToolkit/Databases/DemDatabaseFileInfos.cs
+++ b/MapToolkit/Databases/DemDatabaseFileInfos.cs
@@ -9,20 +9,28 @@ namespace Pmad.Cartography.Databases
public class DemDatabaseFileInfos
{
[JsonConstructor]
- public DemDatabaseFileInfos(string path, DemDataCellMetadata metadata)
+ public DemDatabaseFileInfos(string path, DemDataCellMetadata metadata, string? sha256 = null)
{
Path = path;
Metadata = metadata;
+ Sha256 = sha256;
}
- public DemDatabaseFileInfos(string path, IDemDataCellMetadata metadata)
+ public DemDatabaseFileInfos(string path, IDemDataCellMetadata metadata, string? sha256 = null)
{
Path = path;
Metadata = metadata as DemDataCellMetadata ?? new DemDataCellMetadata(metadata);
+ Sha256 = sha256;
}
public string Path { get; }
public DemDataCellMetadata Metadata { get; }
+
+ ///
+ /// Optional SHA-256 hex digest of the cell file, used to verify integrity after download.
+ ///
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public string? Sha256 { get; }
}
}
diff --git a/MapToolkit/Databases/DemFileSystemStorage.cs b/MapToolkit/Databases/DemFileSystemStorage.cs
index 8c43388..908cf6f 100644
--- a/MapToolkit/Databases/DemFileSystemStorage.cs
+++ b/MapToolkit/Databases/DemFileSystemStorage.cs
@@ -1,8 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
-using System.Text;
using System.Text.Json;
+using System.Threading;
using System.Threading.Tasks;
using Pmad.Cartography.DataCells;
@@ -27,19 +27,28 @@ public async Task ReadIndex()
return (await JsonSerializer.DeserializeAsync(input, DemDatabaseIndexContext.Default.DemDatabaseIndex).ConfigureAwait(false))!;
}
}
- return BuildIndex();
+ return await BuildIndexAsync().ConfigureAwait(false);
}
+ [Obsolete]
public DemDatabaseIndex BuildIndex()
{
+ return BuildIndexAsync().GetAwaiter().GetResult();
+ }
+
+ public async Task BuildIndexAsync(IProgress? progress = null, CancellationToken cancellationToken = default)
+ {
var entries = new List();
var files = Directory.GetFiles(basePath, "*.*", SearchOption.AllDirectories);
+ var done = 0;
foreach (var file in files)
{
if (DemDataCell.IsDemDataCellFile(file))
{
- entries.Add(new DemDatabaseFileInfos(GetRelative(file), DemDataCell.LoadMetadata(file)));
+ entries.Add(new DemDatabaseFileInfos(GetRelative(file), DemDataCell.LoadMetadata(file), await Sha256Helper.ComputeHexAsync(file, cancellationToken).ConfigureAwait(false)));
}
+ done++;
+ progress?.Report((double)done / files.Length);
}
return new DemDatabaseIndex(entries);
}
@@ -51,7 +60,27 @@ private string GetRelative(string file)
public Task Load(string path)
{
- return Task.FromResult(DemDataCell.Load(Path.Combine(basePath, path)));
+ return LoadAsync(path, null);
+ }
+
+ public async Task LoadAsync(string path, string? expectedSha256, CancellationToken cancellationToken = default)
+ {
+ var fullPath = Path.Combine(basePath, path);
+ if (expectedSha256 != null && !await Sha256Helper.VerifyAsync(fullPath, expectedSha256, cancellationToken).ConfigureAwait(false))
+ {
+ throw new InvalidDataException($"SHA-256 checksum mismatch for '{path}'.");
+ }
+ return DemDataCell.Load(fullPath);
+ }
+
+ public async Task GetSha256Async(string path, CancellationToken cancellationToken = default)
+ {
+ var fullPath = Path.Combine(basePath, path);
+ if (File.Exists(fullPath))
+ {
+ return await Sha256Helper.ComputeHexAsync(fullPath, cancellationToken).ConfigureAwait(false);
+ }
+ return null;
}
}
}
diff --git a/MapToolkit/Databases/DemHttpStorage.cs b/MapToolkit/Databases/DemHttpStorage.cs
index 2e6382d..465c1d9 100644
--- a/MapToolkit/Databases/DemHttpStorage.cs
+++ b/MapToolkit/Databases/DemHttpStorage.cs
@@ -2,6 +2,7 @@
using System.IO;
using System.Net.Http;
using System.Text.Json;
+using System.Threading;
using System.Threading.Tasks;
using Pmad.Cartography.DataCells;
@@ -9,16 +10,23 @@ namespace Pmad.Cartography.Databases
{
public class DemHttpStorage : IDemStorage
{
+ ///
+ /// How long a cached index.json is considered fresh before being re-downloaded.
+ ///
+ public static TimeSpan IndexCacheDuration { get; set; } = TimeSpan.FromHours(24);
+
+ const int MaxDownloadAttempts = 3;
+
private readonly string localCache;
private readonly HttpClient client;
- public DemHttpStorage (string? localCache, HttpClient client)
+ public DemHttpStorage(string? localCache, HttpClient client)
{
this.localCache = localCache ?? DefaultCacheLocation;
this.client = client;
}
- public DemHttpStorage(string? localCache, Uri baseAddress)
+ public DemHttpStorage(string? localCache, Uri baseAddress)
: this(localCache, new HttpClient() { BaseAddress = baseAddress })
{
@@ -31,33 +39,114 @@ public DemHttpStorage(Uri baseAddress)
}
public static string DefaultCacheLocation => Path.Combine(Path.GetTempPath(), "dem");
-
- public async Task Load(string path)
+
+ public static void ClearDefaultCache()
+ {
+ var cacheDir = DefaultCacheLocation;
+ if (Directory.Exists(cacheDir))
+ {
+ Directory.Delete(cacheDir, recursive: true);
+ }
+ }
+
+ private string GetCacheFile(string path)
{
var uri = new Uri(client.BaseAddress!, path);
- var cacheFile = Path.Combine(localCache, uri.DnsSafeHost, uri.AbsolutePath.Substring(1).Replace('/', Path.DirectorySeparatorChar));
- if(!File.Exists(cacheFile))
+ return Path.Combine(localCache, uri.DnsSafeHost, uri.AbsolutePath.Substring(1).Replace('/', Path.DirectorySeparatorChar));
+ }
+
+ public async Task LoadAsync(string path, string? expectedSha256 = null, CancellationToken cancellationToken = default)
+ {
+ var cacheFile = GetCacheFile(path);
+ if (File.Exists(cacheFile))
{
- Directory.CreateDirectory(Path.GetDirectoryName(cacheFile)!);
- // XXX: Limit cache size ?
- // XXX: Cache invalidation ?
- using (var input = await client.GetStreamAsync(path).ConfigureAwait(false))
+ if (expectedSha256 != null && !await Sha256Helper.VerifyAsync(cacheFile, expectedSha256, cancellationToken).ConfigureAwait(false))
{
- using (var cache = File.Create(cacheFile))
+ File.Delete(cacheFile);
+ }
+ }
+
+ if (!File.Exists(cacheFile))
+ {
+ await DownloadFile(path, cacheFile, cancellationToken).ConfigureAwait(false);
+ if (expectedSha256 != null && !await Sha256Helper.VerifyAsync(cacheFile, expectedSha256, cancellationToken).ConfigureAwait(false))
+ {
+ File.Delete(cacheFile);
+ throw new InvalidDataException($"SHA-256 checksum mismatch for '{path}'. The downloaded file may be corrupted.");
+ }
+ }
+
+ return DemDataCell.Load(cacheFile);
+ }
+
+ private async Task DownloadFile(string path, string cacheFile, CancellationToken cancellationToken = default)
+ {
+ var cacheDirectory = Path.GetDirectoryName(cacheFile)!;
+
+ Directory.CreateDirectory(cacheDirectory);
+
+ var tempFile = Path.Combine(cacheDirectory, Path.GetRandomFileName());
+
+ for (int attempt = 1; attempt <= MaxDownloadAttempts; attempt++)
+ {
+ try
+ {
+ using (var input = await client.GetStreamAsync(path, cancellationToken).ConfigureAwait(false))
+ {
+ using var cache = File.Create(tempFile);
+ await input.CopyToAsync(cache, cancellationToken).ConfigureAwait(false);
+ }
+ File.Move(tempFile, cacheFile, true);
+ return;
+ }
+ catch (HttpRequestException httpException) when (httpException.StatusCode == System.Net.HttpStatusCode.NotFound)
+ {
+ throw;
+ }
+ catch (Exception) when (attempt < MaxDownloadAttempts && !cancellationToken.IsCancellationRequested)
+ {
+ await Task.Delay(Random.Shared.Next(500, 5000), cancellationToken).ConfigureAwait(false);
+ }
+ finally
+ {
+ if (File.Exists(tempFile))
{
- await input.CopyToAsync(cache).ConfigureAwait(false);
+ File.Delete(tempFile);
}
}
}
- return DemDataCell.Load(cacheFile);
}
+ public Task Load(string path) => LoadAsync(path, null);
+
public async Task ReadIndex()
{
- using (var input = await client.GetStreamAsync("index.json").ConfigureAwait(false))
+ var cacheFile = GetCacheFile("index.json");
+
+ if (!File.Exists(cacheFile) || (DateTime.UtcNow - File.GetLastWriteTimeUtc(cacheFile)) > IndexCacheDuration)
+ {
+ await DownloadFile("index.json", cacheFile).ConfigureAwait(false);
+ }
+
+ using (var stream = File.OpenRead(cacheFile))
+ {
+ return (await JsonSerializer.DeserializeAsync(stream, DemDatabaseIndexContext.Default.DemDatabaseIndex).ConfigureAwait(false))!;
+ }
+ }
+
+ public async Task GetSha256Async(string path, CancellationToken cancellationToken = default)
+ {
+ var cacheFile = GetCacheFile(path);
+ try
+ {
+ await DownloadFile(path, cacheFile, cancellationToken).ConfigureAwait(false);
+ }
+ catch(HttpRequestException httpException) when (httpException.StatusCode == System.Net.HttpStatusCode.NotFound)
{
- return (await JsonSerializer.DeserializeAsync(input, DemDatabaseIndexContext.Default.DemDatabaseIndex).ConfigureAwait(false))!;
+ // If the file doesn't exist on the server, we won't be able to get its SHA-256.
+ return null;
}
+ return await Sha256Helper.ComputeHexAsync(cacheFile, cancellationToken).ConfigureAwait(false);
}
}
}
diff --git a/MapToolkit/Databases/IDemStorage.cs b/MapToolkit/Databases/IDemStorage.cs
index b79adf1..93b161c 100644
--- a/MapToolkit/Databases/IDemStorage.cs
+++ b/MapToolkit/Databases/IDemStorage.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Threading;
using System.Threading.Tasks;
using Pmad.Cartography.DataCells;
@@ -9,5 +10,11 @@ public interface IDemStorage
Task ReadIndex();
Task Load(string path);
+
+ Task LoadAsync(string path, string? expectedSha256, CancellationToken cancellationToken = default)
+ => Load(path);
+
+ Task GetSha256Async(string path, CancellationToken cancellationToken = default)
+ => Task.FromResult(null);
}
}
\ No newline at end of file
diff --git a/MapToolkit/Databases/Sha256Helper.cs b/MapToolkit/Databases/Sha256Helper.cs
new file mode 100644
index 0000000..da26ab2
--- /dev/null
+++ b/MapToolkit/Databases/Sha256Helper.cs
@@ -0,0 +1,35 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Pmad.Cartography.Databases
+{
+ public static class Sha256Helper
+ {
+ public static async Task ComputeHexAsync(string filePath, CancellationToken cancellationToken = default)
+ {
+ using var sha256 = SHA256.Create();
+ using var stream = File.OpenRead(filePath);
+ var hash = await sha256.ComputeHashAsync(stream, cancellationToken).ConfigureAwait(false);
+ return BytesToHex(hash);
+ }
+
+ public static async Task VerifyAsync(string filePath, string expectedHex, CancellationToken cancellationToken = default)
+ {
+ return string.Equals(await ComputeHexAsync(filePath, cancellationToken).ConfigureAwait(false), expectedHex, StringComparison.OrdinalIgnoreCase);
+ }
+
+ private static string BytesToHex(byte[] bytes)
+ {
+ var sb = new StringBuilder(bytes.Length * 2);
+ foreach (var b in bytes)
+ {
+ sb.Append(b.ToString("x2"));
+ }
+ return sb.ToString();
+ }
+ }
+}
diff --git a/MapToolkit/Databases/WellKnownDatabases.cs b/MapToolkit/Databases/WellKnownDatabases.cs
index efdc65d..61ec130 100644
--- a/MapToolkit/Databases/WellKnownDatabases.cs
+++ b/MapToolkit/Databases/WellKnownDatabases.cs
@@ -1,13 +1,11 @@
using System;
-using System.Data;
using System.Net.Http;
-using static System.Runtime.InteropServices.JavaScript.JSType;
namespace Pmad.Cartography.Databases
{
public static class WellKnownDatabases
{
- private const string DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0";
+ private const string DefaultUserAgent = "Mozilla/5.0 (Pmad-Cartography; Default)";
internal static HttpClient CreateClient(string baseAddress)
{