Add checksum in index, and improve cache logic

DemUtility/DemUtility.csproj

@@ -2,13 +2,14 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net10.0</TargetFramework>
     <Nullable>enable</Nullable>
     <AssemblyName>dem</AssemblyName>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="System.CommandLine" Version="2.0.7" />
+    <PackageReference Include="Pmad.ProgressTracking" Version="0.1.28" />
   </ItemGroup>
 
   <ItemGroup>

DemUtility/Properties/launchSettings.json

@@ -1,20 +1,64 @@
 {
   "profiles": {
-    "Repack STRM": {
+    "Repack SRTM": {
       "commandName": "Project",
       "commandLineArgs": "repack -k -s E:\\Carto\\SRTMv3 -t c:\\temp\\SRTMv3 -m 10"
     },
     "Repack AW3D30": {
       "commandName": "Project",
       "commandLineArgs": "repack -k -s E:\\Carto\\AW3D30 -t c:\\temp\\AW3D30 -m 10"
     },
-    "Index STRM": {
+
+    "Index AW3D30": {
       "commandName": "Project",
-      "commandLineArgs": "index -p C:\\temp\\SRTMv3"
+      "commandLineArgs": "index -p D:\\dem\\AW3D30"
     },
-    "Index AW3D30": {
+    "Index SRTM15Plus": {
+      "commandName": "Project",
+      "commandLineArgs": "index -p D:\\dem\\SRTM15Plus"
+    },
+    "Index SRTM1": {
+      "commandName": "Project",
+      "commandLineArgs": "index -p D:\\dem\\SRTM1"
+    },
+
+    "Update Index AW3D30": {
+      "commandName": "Project",
+      "commandLineArgs": "update-index -p D:\\dem\\AW3D30"
+    },
+    "Update Index SRTM15Plus": {
+      "commandName": "Project",
+      "commandLineArgs": "update-index -p D:\\dem\\SRTM15Plus"
+    },
+    "Update Index SRTM1": {
+      "commandName": "Project",
+      "commandLineArgs": "update-index -p D:\\dem\\SRTM1"
+    },
+
+    "Check AW3D30": {
+      "commandName": "Project",
+      "commandLineArgs": "check -v -p D:\\dem\\AW3D30"
+    },
+    "Check SRTM15Plus": {
+      "commandName": "Project",
+      "commandLineArgs": "check -v -p D:\\dem\\SRTM15Plus"
+    },
+    "Check SRTM1": {
+      "commandName": "Project",
+      "commandLineArgs": "check -v -p D:\\dem\\SRTM1"
+    },
+
+    "Check CDN AW3D30": {
+      "commandName": "Project",
+      "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/AW3D30/ -n 100"
+    },
+    "Check CDN SRTM15Plus": {
+      "commandName": "Project",
+      "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/SRTM15Plus/ -n 100"
+    },
+    "Check CDN SRTM1": {
       "commandName": "Project",
-      "commandLineArgs": "index -p C:\\temp\\AW3D30"
+      "commandLineArgs": "check -v -u https://cdn.dem.pmad.net/SRTM1/ -n 100"
     }
   }
 }

MapToolkit.Test/CollectionDefinitions.cs

@@ -0,0 +1,5 @@
+namespace Pmad.Cartography.Test
+{
+    [CollectionDefinition("Sequential", DisableParallelization = true)]
+    public class SequentialCollectionDefinition { }
+}

MapToolkit.Test/CompressionHelperTest.cs

@@ -22,9 +22,13 @@ public void GetExtension_FromFilename()
         public void GetExtension_FromCompression()
         {
             Assert.Equal(".zst", CompressionHelper.GetExtension(Compression.ZSTD));
-            Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZib));
+            Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZip));
             Assert.Equal(".bt", CompressionHelper.GetExtension(Compression.Brotli));
             Assert.Equal(string.Empty, CompressionHelper.GetExtension(Compression.None));
+
+#pragma warning disable CS0618 // Le type ou le membre est obsolète
+            Assert.Equal(".gz", CompressionHelper.GetExtension(Compression.GZib));
+#pragma warning restore CS0618 // Le type ou le membre est obsolète
         }
 
         [Fact]
@@ -40,7 +44,7 @@ public void GetFileName_RemovesExtension()
         [Fact]
         public void ReadSeekable_ReadsCompressedFile_Compressed()
         {
-            var filename = WriteCompressedFile(Compression.GZib);
+            var filename = WriteCompressedFile(Compression.GZip);
             var result = CompressionHelper.ReadSeekable(filename, stream =>
             {
                 using (var reader = new StreamReader(stream))
@@ -68,9 +72,9 @@ public void ReadSeekable_ReadsCompressedFile_None()
         }
 
         [Fact]
-        public void Read_ReadsCompressedFile_GZib()
+        public void Read_ReadsCompressedFile_GZip()
         {
-            var filename = WriteCompressedFile(Compression.GZib);
+            var filename = WriteCompressedFile(Compression.GZip);
             var result = CompressionHelper.Read(filename, stream =>
             {
                 using (var reader = new StreamReader(stream))
@@ -130,7 +134,7 @@ public void Read_ReadsCompressedFile_Brotli()
         [Fact]
         public void GetSize_ReturnsCorrectSize_GZip()
         {
-            var filename = WriteCompressedFile(Compression.GZib);
+            var filename = WriteCompressedFile(Compression.GZip);
             var size = CompressionHelper.GetSize(filename);
             Assert.Equal(12, size); // "test content" length
             File.Delete(filename);

MapToolkit.Test/Databases/DemDatabaseEntryTest.cs

@@ -40,7 +40,7 @@ public async Task DemDatabaseEntry_Load()
             var cache = new MemoryCache(new MemoryCacheOptions());
             var dataCell = new Mock<IDemDataCell>();
             dataCell.Setup(d => d.SizeInBytes).Returns(100);
-            storage.Setup(s => s.Load(It.IsAny<string>())).ReturnsAsync(dataCell.Object);
+            storage.Setup(s => s.LoadAsync(It.IsAny<string>(), null, default)).ReturnsAsync(dataCell.Object);
 
             var entry = new DemDatabaseEntry("test", metadata.Object);
             var result = await entry.Load(storage.Object, cache);

MapToolkit.Test/Databases/DemDatabaseTest.cs

@@ -39,7 +39,7 @@ public async Task GetDataCellsAsync_ShouldReturnDataCells()
             var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100));
             var dataCell = new Mock<IDemDataCell>().Object;
             mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List<DemDatabaseFileInfos> { entry }));
-            mockStorage.Setup(s => s.Load(It.IsAny<string>())).ReturnsAsync(dataCell);
+            mockStorage.Setup(s => s.LoadAsync(It.IsAny<string>(), It.IsAny<string?>(), default)).ReturnsAsync(dataCell);
 
             // Act
             var result = await demDatabase.GetDataCellsAsync(start, end);
@@ -60,7 +60,7 @@ public async Task CreateView_ShouldReturnDemDataView()
             var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100));
             var dataCell = new DemDataCellPixelIsPoint<float>(new Coordinates(0, 0), new Coordinates(1, 1), new float[100, 100]);
             mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List<DemDatabaseFileInfos> { entry }));
-            mockStorage.Setup(s => s.Load(It.IsAny<string>())).ReturnsAsync(dataCell);
+            mockStorage.Setup(s => s.LoadAsync(It.IsAny<string>(), It.IsAny<string?>(), default)).ReturnsAsync(dataCell);
 
             // Act
             var result = await demDatabase.CreateView<float>(start, end);
@@ -119,7 +119,7 @@ public void GetElevation_ShouldReturnElevation()
             var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100));
             var dataCell = new Mock<IDemDataCell>().Object;
             mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List<DemDatabaseFileInfos> { entry }));
-            mockStorage.Setup(s => s.Load(It.IsAny<string>())).ReturnsAsync(dataCell);
+            mockStorage.Setup(s => s.LoadAsync(It.IsAny<string>(), It.IsAny<string?>(), default)).ReturnsAsync(dataCell);
 
             // Act
             var result = demDatabase.GetElevation(coordinates, interpolation);
@@ -140,7 +140,7 @@ public async Task GetElevationAsync_ShouldReturnElevation()
             var entry = new DemDatabaseFileInfos("path", new DemDataCellMetadata(DemRasterType.PixelIsPoint, new Coordinates(0, 0), new Coordinates(1, 1), 100, 100));
             var dataCell = new DemDataCellPixelIsPoint<float>(new Coordinates(0, 0), new Coordinates(1, 1), new float[100, 100]);
             mockStorage.Setup(s => s.ReadIndex()).ReturnsAsync(new DemDatabaseIndex(new List<DemDatabaseFileInfos> { entry }));
-            mockStorage.Setup(s => s.Load(It.IsAny<string>())).ReturnsAsync(dataCell);
+            mockStorage.Setup(s => s.LoadAsync(It.IsAny<string>(), It.IsAny<string?>(), default)).ReturnsAsync(dataCell);
 
             // Act
             var result = await demDatabase.GetElevationAsync(coordinates, interpolation);

MapToolkit.Test/Databases/DemFileSystemStorageTest.cs

@@ -0,0 +1,88 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+using System.Threading.Tasks;
+using Pmad.Cartography.Databases;
+using Xunit;
+
+namespace Pmad.Cartography.Test.Databases
+{
+    public class DemFileSystemStorageTest
+    {
+        private static string CreateTempFile(byte[] content)
+        {
+            var path = Path.GetTempFileName();
+            File.WriteAllBytes(path, content);
+            return path;
+        }
+
+        private static string ComputeExpectedHex(byte[] content)
+        {
+            return Convert.ToHexString(SHA256.HashData(content)).ToLowerInvariant();
+        }
+
+        [Fact]
+        public async Task GetSha256Async_ShouldReturnCorrectHash_WhenFileExists()
+        {
+            // Arrange
+            var content = "hello dem"u8.ToArray();
+            var file = CreateTempFile(content);
+            var basePath = Path.GetDirectoryName(file)!;
+            var fileName = Path.GetFileName(file);
+            var storage = new DemFileSystemStorage(basePath);
+
+            try
+            {
+                // Act
+                var hash = await storage.GetSha256Async(fileName);
+
+                // Assert
+                Assert.NotNull(hash);
+                Assert.Equal(ComputeExpectedHex(content), hash);
+            }
+            finally
+            {
+                File.Delete(file);
+            }
+        }
+
+        [Fact]
+        public async Task GetSha256Async_ShouldReturnNull_WhenFileDoesNotExist()
+        {
+            // Arrange
+            var basePath = Path.GetTempPath();
+            var storage = new DemFileSystemStorage(basePath);
+
+            // Act
+            var hash = await storage.GetSha256Async("nonexistent_file_that_does_not_exist.hgt.zst");
+
+            // Assert
+            Assert.Null(hash);
+        }
+
+        [Fact]
+        public async Task GetSha256Async_ShouldReturnLowercaseHex()
+        {
+            // Arrange
+            var content = "case check"u8.ToArray();
+            var file = CreateTempFile(content);
+            var basePath = Path.GetDirectoryName(file)!;
+            var fileName = Path.GetFileName(file);
+            var storage = new DemFileSystemStorage(basePath);
+
+            try
+            {
+                // Act
+                var hash = await storage.GetSha256Async(fileName);
+
+                // Assert
+                Assert.NotNull(hash);
+                Assert.Equal(hash!.ToLowerInvariant(), hash);
+            }
+            finally
+            {
+                File.Delete(file);
+            }
+        }
+    }
+}