From d1309f54cfcd5756786735c6a2c849419c187ed7 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:44:24 +0200 Subject: [PATCH 01/43] Update raster store refactor plan for DAG changes --- docs/raster-store/refactor-plan.md | 31 +++++++++++------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index f783e5d0..63b17e2c 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -57,9 +57,11 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. receive a codec template parameter and application call sites do not manage codec objects. - This refactor does not add synchronization to storage, indexes, or caches - and does not change their concurrency guarantees. Existing caller-side - synchronization and concurrency behaviour are preserved; any concurrency - bug fix is separate work. + and does not change their concurrency guarantees. Any concurrency bug fix is + separate work. +- Application callers retain their existing storage lifecycle, + synchronization, and failure-handling behaviour unless explicitly changed + by this plan. - Preserve `dag::ThreadSafeStorage` as the caller-side synchronization around DAG output storage. DAG storage remains cacheless while shared-lock reads are used. @@ -79,8 +81,7 @@ Persistent raster-fundamentalis formats, adapters, and tools are later work. booleans. - Preserve `StorageSettings::allow_overwrite`. It defaults to `false`; rejected overwrites return `AlreadyExists` through `std::expected`, and - enabling it preserves the existing DAG-builder overwrite and debug-export - behaviour. + enabling it replaces existing payloads. - The existing cache implementations are currently non-functional. Port their public API where it remains useful, keep application call sites building, and provide compile coverage only. Cache behaviour is not a compatibility @@ -261,9 +262,6 @@ caller include order. Preserve the current tuples exactly: - `NodeMetadata`: group assignment, groups; and - `Group`: children, error, bounds. -`Group::child_errors` is currently neither populated nor serialized. It remains -non-persistent in this refactor and the Phase 0 fixture locks down its omission. - ## Shared interfaces The names below are the intended shape, not signatures that must be copied @@ -620,9 +618,7 @@ Loading and saving likewise return storage-level expected errors which retain an invalid key, an underlying `CodecError`, and `AlreadyExists` for a rejected save. `CopyError` retains invalid-key, missing-source, overwrite, filesystem, and codec failures. The SF call chains changed in Phase 4 retain these errors -to their application boundary. DAG callers preserve their existing per-node -log-and-continue and command-line policies; making the DAG builder fail-fast is -not part of this refactor. +to their application boundary. Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes candidate endings by asking the supplied resolver, removes an accepted ending @@ -632,9 +628,6 @@ Only a missing index triggers this discovery path. An unreadable or malformed index, unknown layout, or unsupported codec selector returns `OpenError` and must not silently fall back to directory discovery. -Automatic dirty-index saving currently happens in the 3D storage destructor. -Preserve that behaviour for existing 3D entry points during the migration. - ### Copying one node and SF subtree reuse There are two separate responsibilities in the current implementation: @@ -780,8 +773,7 @@ No production behaviour changes. contains a valid serialized `dag::ClusterBatch`, and whose index contains `Leaf`, `Virtual`, and `Inner`. Use the current metadata-then-clustering format, include non-trivial group metadata, and prove that the same file can - be opened through the read-only `NodeMetadata` view. Lock down that - `Group::child_errors` is not serialized. + be opened through the read-only `NodeMetadata` view. 3. Test that all fixtures open, resolve the expected IDs and extensions, and traverse the expected sparse nodes. 4. Add path round-trip tests for boundary IDs and both layouts. @@ -894,13 +886,12 @@ path. storage consumers without exposing codec objects at application call sites. 11. Migrate the existing octree storage aliases and all other application callers, including adapting `dag::ThreadSafeStorage` to the new key and - expected-returning APIs while preserving its shared/exclusive locking. + expected-returning APIs. 12. Preserve the current 3D destructor-save behaviour until all callers have explicit index finalization. Test move construction, move assignment over dirty state, moved-from destruction, and the DAG move/release path. -13. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's - overwrite mode and repeat debug export. Replace process termination on a - rejected overwrite with `AlreadyExists` in the storage-level expected +13. Preserve `StorageSettings::allow_overwrite`. Replace process termination + on a rejected overwrite with `AlreadyExists` in the storage-level expected error. Test that a rejected save returns `AlreadyExists` and that enabling overwrite replaces the existing payload. 14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, writable From 75277f89e5def61f97a8f7ee810b7c26c1c8c53b Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:17:57 +0200 Subject: [PATCH 02/43] Capture raster store compatibility baseline --- docs/raster-store/refactor-status.md | 59 +++++ unittests/CMakeLists.txt | 2 + .../dag_builder/storage_compatibility.cpp | 58 +++++ .../raster-store-refactor/dag/0/0/0/0.bin | Bin 0 -> 276 bytes .../raster-store-refactor/dag/2/2/3/1.bin | Bin 0 -> 276 bytes .../raster-store-refactor/dag/terrain.index | Bin 0 -> 78 bytes .../sf-coordinates/terrain.index | Bin 0 -> 92 bytes .../sf-flat/terrain.index | Bin 0 -> 34 bytes unittests/terrainlib/store_compatibility.cpp | 212 ++++++++++++++++++ 9 files changed, 331 insertions(+) create mode 100644 docs/raster-store/refactor-status.md create mode 100644 unittests/dag_builder/storage_compatibility.cpp create mode 100644 unittests/data/raster-store-refactor/dag/0/0/0/0.bin create mode 100644 unittests/data/raster-store-refactor/dag/2/2/3/1.bin create mode 100644 unittests/data/raster-store-refactor/dag/terrain.index create mode 100644 unittests/data/raster-store-refactor/sf-coordinates/terrain.index create mode 100644 unittests/data/raster-store-refactor/sf-flat/terrain.index create mode 100644 unittests/terrainlib/store_compatibility.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md new file mode 100644 index 00000000..cf951dad --- /dev/null +++ b/docs/raster-store/refactor-status.md @@ -0,0 +1,59 @@ +# Raster store refactor status + +This file is the resumable implementation log for +[refactor-plan.md](refactor-plan.md). + +## Baseline + +- Branch: `raster-store` +- Baseline commit: `d1309f54cfcd5756786735c6a2c849419c187ed7` +- Baseline tag: `refactor_before` +- Started: 2026-08-14 +- Worktree was clean before implementation began. +- `dev_driver.py doctor` rejects this repository because its root + `CMakeLists.txt` is not the alpine-renderer project. Builds therefore use + this repository's own CMake configuration and build directories. + +## Current position + +- Phase: 0 — Capture current compatibility +- State: in progress +- Next action: run the full Phase 0 verification, inspect the diff, commit, + and tag the phase boundary. + +## Phase log + +### Phase 0 — Capture current compatibility + +- Added pre-refactor SF fixtures for `flat` and + `level_and_coordinate_directories` layouts. +- Added a pre-refactor DAG fixture using `.bin`, with `Leaf`, `Virtual`, and + `Inner` index states and non-trivial metadata. +- Added characterization coverage for fixture opening, metadata-prefix reads, + payload byte stability, layout/path round trips, hard links, + decode/re-encode, enabled overwrites, indexed/unindexed opens, and index + creation by directory scan. +- Recorded pre-refactor public aliases: + - SF builder, SF merger, and SF index browser use `octree::MeshStorage` and + `octree::IndexedStorage` (the latter aliases `IndexedMeshStorage`). + - DAG builder and DAG convert debug use `octree::DagStorage`, + `octree::IndexedDagStorage`, `octree::DagMetaStorage`, and + `octree::IndexedDagMetaStorage`. + - DAG output synchronization uses + `dag::ThreadSafeStorage`. +- State: complete. + +## Verification log + +- Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. +- Baseline `unittests_dagbuilder`: 400 assertions in 69 test cases passed. +- Focused Phase 0 terrainlib compatibility tests: 51 assertions in 6 test + cases passed before warning cleanup. +- Focused Phase 0 DAG compatibility test: 23 assertions passed. +- Phase 0 `unittests_terrainlib`: 23,732 assertions in 397 test cases passed. +- Phase 0 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. +- Phase 0 `git diff --check`: passed. + +## Commit and tag log + +- `refactor_before` — annotated baseline tag at `d1309f5`. diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 6e694422..7d89ca6f 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -30,6 +30,7 @@ add_executable(unittests_terrainlib terrainlib/offset_table.cpp terrainlib/segmented_buffer.cpp terrainlib/stamp_set.cpp + terrainlib/store_compatibility.cpp terrainlib/index_map.cpp terrainlib/index_traverse.cpp terrainlib/mesh_boundary.cpp @@ -120,6 +121,7 @@ if(TARGET dagbuilderlib) dag_builder/fixed_point.cpp dag_builder/texture_sizing.cpp dag_builder/texture_reuse.cpp + dag_builder/storage_compatibility.cpp ) target_link_libraries(unittests_dagbuilder PUBLIC dagbuilderlib Catch2::Catch2WithMain) atb_configure_test(unittests_dagbuilder) diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp new file mode 100644 index 00000000..e68750bf --- /dev/null +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -0,0 +1,58 @@ +#include + +#include + +#include "encoded.h" +#include "io/bytes.h" +#include "io/serialize.h" +#include "octree/disk/IndexFile.h" +#include "octree/storage/open.h" +#include "octree/traverse.h" +#include "storage.h" + +TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { + const std::filesystem::path path = + std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor" / "dag"; + const octree::Id root = octree::Id::root(); + const octree::Id parent = root.child(3).value(); + const octree::Id leaf = parent.child(6).value(); + + const auto index_file = io::read_from_path(path / "terrain.index"); + REQUIRE(index_file.has_value()); + CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); + CHECK(index_file->preferred_extension == ".bin"); + CHECK(index_file->map.is(octree::NodeStatus::Inner, root)); + CHECK(index_file->map.is(octree::NodeStatus::Virtual, parent)); + CHECK(index_file->map.is(octree::NodeStatus::Leaf, leaf)); + CHECK(std::filesystem::is_regular_file(path / "0/0/0/0.bin")); + CHECK(std::filesystem::is_regular_file(path / "2/2/3/1.bin")); + + std::vector visited; + octree::traverse(index_file->map, [&](const octree::Id id, const octree::NodeStatus) { + visited.push_back(id); + }); + CHECK(visited == std::vector{root, parent, leaf}); + + const auto batch_storage = octree::open_folder_indexed(path); + const auto batch = batch_storage.load(root); + REQUIRE(batch.has_value()); + CHECK(batch->metadata.group_assignment == std::vector{0}); + REQUIRE(batch->metadata.groups.size() == 1); + CHECK(batch->metadata.groups.front().error == 12.5); + CHECK(batch->metadata.groups.front().children == std::vector{{root.child(4).value(), 3}}); + REQUIRE(batch->clustering.clusters.size() == 1); + CHECK(batch->clustering.clusters.front().id == 17); + + const auto metadata_storage = octree::open_folder_indexed(path); + const auto metadata = metadata_storage.load(root); + REQUIRE(metadata.has_value()); + CHECK(metadata->group_assignment == batch->metadata.group_assignment); + REQUIRE(metadata->groups.size() == batch->metadata.groups.size()); + CHECK(metadata->groups.front().error == batch->metadata.groups.front().error); + + const auto encoded = io::write_to_bytes(batch.value()); + const auto fixture_bytes = io::read_bytes_from_path(path / "0/0/0/0.bin"); + REQUIRE(encoded.has_value()); + REQUIRE(fixture_bytes.has_value()); + CHECK(encoded.value() == fixture_bytes.value()); +} diff --git a/unittests/data/raster-store-refactor/dag/0/0/0/0.bin b/unittests/data/raster-store-refactor/dag/0/0/0/0.bin new file mode 100644 index 0000000000000000000000000000000000000000..7c8b1d28a199bea0ce758cdcc3ae51b44eed5bea GIT binary patch literal 276 zcmZQ%fB;4y1;#8;0cI%8z@X^>VSd;T6*>T+IiR$P1B7n^rR^MGdi8)33s>y|K?Vm2 z1`P=Y1rdna5B3n60jh_?0c<46;Dc dg{v8um|)7#X-SX*bU_$j5XN9&xBzn!0|33_7i<6k literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/dag/2/2/3/1.bin b/unittests/data/raster-store-refactor/dag/2/2/3/1.bin new file mode 100644 index 0000000000000000000000000000000000000000..7c8b1d28a199bea0ce758cdcc3ae51b44eed5bea GIT binary patch literal 276 zcmZQ%fB;4y1;#8;0cI%8z@X^>VSd;T6*>T+IiR$P1B7n^rR^MGdi8)33s>y|K?Vm2 z1`P=Y1rdna5B3n60jh_?0c<46;Dc dg{v8um|)7#X-SX*bU_$j5XN9&xBzn!0|33_7i<6k literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/dag/terrain.index b/unittests/data/raster-store-refactor/dag/terrain.index new file mode 100644 index 0000000000000000000000000000000000000000..e3f1d112fe64250be60fa4d32d0b944f214aaca5 GIT binary patch literal 78 zcmY#jU|`5eElbUbPs~e+PtMOTO3BPiEJ=+|$t+4uF3B&-Of6;ss?kfz%wq=9OmaX1 P3>ZON5MY8(K#~yvLw642 literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/sf-coordinates/terrain.index b/unittests/data/raster-store-refactor/sf-coordinates/terrain.index new file mode 100644 index 0000000000000000000000000000000000000000..546dd8bfb81ff88d2e18776d6159eb841eac8f5a GIT binary patch literal 92 zcmY#jU|`5eElbUbPs~e+PtMOTO3BPiEJ=+|$t+4uF3B&-OfBXBs?jS+EhSd6SNA(#jQ698sS5W4^X literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/sf-flat/terrain.index b/unittests/data/raster-store-refactor/sf-flat/terrain.index new file mode 100644 index 0000000000000000000000000000000000000000..c581a3f6f1d99d289ca268fad4d3421bf7cf191a GIT binary patch literal 34 ecmZQ!U|>kgNi5+2(t0JSMMa63d5l0d0ssJM*#y`C literal 0 HcmV?d00001 diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp new file mode 100644 index 00000000..84e28a85 --- /dev/null +++ b/unittests/terrainlib/store_compatibility.cpp @@ -0,0 +1,212 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "io/serialize.h" +#include "mesh/SimpleMesh.h" +#include "octree/Storage.h" +#include "octree/disk/IndexFile.h" +#include "octree/disk/layout/strategy/Flat.h" +#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" +#include "octree/storage/RawStorage.h" +#include "octree/storage/open.h" +#include "octree/traverse.h" + +namespace { + +std::filesystem::path fixture_path(const std::string_view name) { + return std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor" / name; +} + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-raster-store-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple triangle_mesh(const double offset) { + return mesh::Simple( + {{0, 1, 2}}, + {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); +} + +octree::IndexedStorage make_storage( + const std::filesystem::path &path, + std::unique_ptr strategy, + const std::string &extension) { + octree::disk::Layout layout(path, std::move(strategy), extension); + return octree::IndexedStorage(octree::RawStorage_(std::move(layout)), octree::IndexMap{}); +} + +} // namespace + +TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { + const octree::Id root = octree::Id::root(); + + SECTION("flat") { + const std::filesystem::path path = fixture_path("sf-flat"); + const auto index_file = io::read_from_path(path / "terrain.index"); + REQUIRE(index_file.has_value()); + CHECK(index_file->layout_strategy_id == "flat"); + CHECK(index_file->preferred_extension == ".terrain"); + CHECK(index_file->map.size() == 1); + CHECK(index_file->map.is(octree::NodeStatus::Leaf, root)); + CHECK(std::filesystem::is_regular_file(path / "0-0.terrain")); + + const octree::IndexedStorage storage = octree::open_folder_indexed(path); + const auto mesh = storage.load(root); + REQUIRE(mesh.has_value()); + CHECK(mesh->face_count() == 1); + } + + SECTION("level and coordinate directories") { + const std::filesystem::path path = fixture_path("sf-coordinates"); + const octree::Id child = root.child(2).value(); + const octree::Id deep = root.child(5).value().child(7).value(); + const octree::Id deep_parent = deep.parent().value(); + + const auto index_file = io::read_from_path(path / "terrain.index"); + REQUIRE(index_file.has_value()); + CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); + CHECK(index_file->preferred_extension == ".terrain"); + CHECK(index_file->map.is(octree::NodeStatus::Virtual, root)); + CHECK(index_file->map.is(octree::NodeStatus::Leaf, child)); + CHECK(index_file->map.is(octree::NodeStatus::Virtual, deep_parent)); + CHECK(index_file->map.is(octree::NodeStatus::Leaf, deep)); + CHECK(std::filesystem::is_regular_file(path / "1/0/1/0.terrain")); + CHECK(std::filesystem::is_regular_file(path / "2/3/1/3.terrain")); + + std::vector visited; + octree::traverse(index_file->map, [&](const octree::Id id, const octree::NodeStatus) { + visited.push_back(id); + }); + CHECK(visited == std::vector{octree::Id{root}, child, deep_parent, deep}); + } +} + +TEST_CASE("pre-refactor layouts round-trip boundary IDs") { + const std::vector ids{ + octree::Id::root(), + octree::Id(octree::Id::max_level(), octree::Id::Index{0}), + octree::Id(octree::Id::max_level(), octree::Id::max_index_on_level(octree::Id::max_level())), + }; + + octree::disk::layout::strategy::Flat flat; + octree::disk::layout::strategy::LevelAndCoordinateDirectories coordinates; + for (const octree::Id id : ids) { + const auto flat_path = flat.get_relative_node_path(id, ".terrain"); + CHECK(flat.get_id_from_relative_node_path(flat_path) == id); + + const auto coordinate_path = coordinates.get_relative_node_path(id, ".terrain"); + CHECK(coordinates.get_id_from_relative_node_path(coordinate_path) == id); + } +} + +TEST_CASE("pre-refactor storage hard-links matching payload formats") { + TemporaryDirectory source_directory("hard-link-source"); + TemporaryDirectory target_directory("hard-link-target"); + auto source = make_storage( + source_directory.path(), + std::make_unique(), + ".terrain"); + auto target = make_storage( + target_directory.path(), + std::make_unique(), + ".terrain"); + + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, triangle_mesh(0.0)).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(std::filesystem::equivalent(source.path_for(root), target.path_for(root))); + REQUIRE(source.save_index().has_value()); + REQUIRE(target.save_index().has_value()); +} + +TEST_CASE("pre-refactor storage re-encodes differing payload formats") { + TemporaryDirectory source_directory("reencode-source"); + TemporaryDirectory target_directory("reencode-target"); + auto source = make_storage( + source_directory.path(), + std::make_unique(), + ".terrain"); + auto target = make_storage( + target_directory.path(), + std::make_unique(), + ".glb"); + + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, triangle_mesh(4.0)).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + CHECK_FALSE(std::filesystem::equivalent(source.path_for(root), target.path_for(root))); + const auto loaded = target.load(root); + REQUIRE(loaded.has_value()); + CHECK(loaded->face_count() == 1); + REQUIRE(source.save_index().has_value()); + REQUIRE(target.save_index().has_value()); +} + +TEST_CASE("pre-refactor storage supports enabled overwrites") { + TemporaryDirectory directory("overwrite"); + auto storage = make_storage( + directory.path(), + std::make_unique(), + ".terrain"); + storage.settings().allow_overwrite = true; + + const octree::Id root = octree::Id::root(); + REQUIRE(storage.save(root, triangle_mesh(1.0)).has_value()); + REQUIRE(storage.save(root, triangle_mesh(9.0)).has_value()); + const auto loaded = storage.load(root); + REQUIRE(loaded.has_value()); + REQUIRE_FALSE(loaded->positions.empty()); + CHECK(loaded->positions.front().x == 9.0); + REQUIRE(storage.save_index().has_value()); +} + +TEST_CASE("pre-refactor folder opening scans payloads and creates an index") { + TemporaryDirectory directory("folder-open"); + octree::OpenOptions options; + options.default_layout_strategy = std::make_unique(); + options.preferred_extension_with_dot = ".terrain"; + + { + auto storage = octree::open_folder(directory.path(), false, std::move(options)); + CHECK_FALSE(storage.is_indexed()); + REQUIRE(storage.save(octree::Id::root(), triangle_mesh(2.0)).has_value()); + CHECK_FALSE(std::filesystem::exists(directory.path() / "terrain.index")); + } + + { + const auto storage = octree::open_folder_indexed(directory.path()); + CHECK(storage.is_indexed()); + CHECK(storage.has(octree::Id::root())); + CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); + } + + const auto reopened = octree::open_index<>(directory.path() / "terrain.index"); + REQUIRE(reopened.has_value()); + CHECK(reopened->has(octree::Id::root())); +} From d756a77b51aa7d6cd133546a96fed68b02c686e0 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:18:21 +0200 Subject: [PATCH 03/43] Add ignored SF compatibility payloads --- docs/raster-store/refactor-status.md | 6 ++++++ .../sf-coordinates/1/0/1/0.terrain | Bin 0 -> 100 bytes .../sf-coordinates/2/3/1/3.terrain | Bin 0 -> 100 bytes .../raster-store-refactor/sf-flat/0-0.terrain | Bin 0 -> 103 bytes 4 files changed, 6 insertions(+) create mode 100644 unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain create mode 100644 unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain create mode 100644 unittests/data/raster-store-refactor/sf-flat/0-0.terrain diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index cf951dad..5493f513 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -57,3 +57,9 @@ This file is the resumable implementation log for ## Commit and tag log - `refactor_before` — annotated baseline tag at `d1309f5`. +- `75277f8` — Phase 0 compatibility tests, indexes, DAG payloads, and status + log. +- Phase 0 fixture-payload follow-up — force-adds the intentionally committed + `.terrain` golden payloads ignored by the repository's general artifact + rule. +- `refactor_phase_0` — annotated Phase 0 completion tag. diff --git a/unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain b/unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain new file mode 100644 index 0000000000000000000000000000000000000000..8b7a579c003dd10778221743462a9aab1ae525ff GIT binary patch literal 100 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^=mA9+uG+Qg*Q!+vEI=*t Q3=DNZAs|q308=;#08EYz_5c6? literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain b/unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain new file mode 100644 index 0000000000000000000000000000000000000000..bebf97b788532b2663c16a2becfe57e15b2126d4 GIT binary patch literal 100 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^=mA9+uG+Qg*Q!+vOh9Gw Q3=DNZAs{eu08=;#08Q8q0ssI2 literal 0 HcmV?d00001 diff --git a/unittests/data/raster-store-refactor/sf-flat/0-0.terrain b/unittests/data/raster-store-refactor/sf-flat/0-0.terrain new file mode 100644 index 0000000000000000000000000000000000000000..8bc0949026e08a52b88a5f372d02ddf2a804eb07 GIT binary patch literal 103 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^7y?BXuKERps~F@J80u;n N7~~lk>VTp|006Aq4+sDN literal 0 HcmV?d00001 From 4c171c881ce67344fa6a2119e3ae965eaaba05df Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:27:47 +0200 Subject: [PATCH 04/43] Extract shared hierarchy topology --- docs/raster-store/refactor-status.md | 29 ++- src/terrainlib/CMakeLists.txt | 1 - src/terrainlib/octree/IndexMap.cpp | 174 -------------- src/terrainlib/octree/IndexMap.h | 87 +++++-- src/terrainlib/octree/NodeStatus.h | 78 +------ src/terrainlib/octree/NodeStatusOrMissing.h | 135 +---------- src/terrainlib/octree/StoreTraits.h | 28 +++ src/terrainlib/octree/traverse.h | 80 ++----- src/terrainlib/raster_store/StoreTraits.h | 45 ++++ src/terrainlib/store/Index.h | 239 ++++++++++++++++++++ src/terrainlib/store/InvalidKey.h | 12 + src/terrainlib/store/NodeStatus.h | 85 +++++++ src/terrainlib/store/NodeStatusOrMissing.h | 134 +++++++++++ src/terrainlib/store/Traits.h | 17 ++ src/terrainlib/store/traverse.h | 99 ++++++++ unittests/CMakeLists.txt | 2 + unittests/terrainlib/store_index.cpp | 81 +++++++ unittests/terrainlib/store_traverse.cpp | 68 ++++++ 18 files changed, 920 insertions(+), 474 deletions(-) delete mode 100644 src/terrainlib/octree/IndexMap.cpp create mode 100644 src/terrainlib/octree/StoreTraits.h create mode 100644 src/terrainlib/raster_store/StoreTraits.h create mode 100644 src/terrainlib/store/Index.h create mode 100644 src/terrainlib/store/InvalidKey.h create mode 100644 src/terrainlib/store/NodeStatus.h create mode 100644 src/terrainlib/store/NodeStatusOrMissing.h create mode 100644 src/terrainlib/store/Traits.h create mode 100644 src/terrainlib/store/traverse.h create mode 100644 unittests/terrainlib/store_index.cpp create mode 100644 unittests/terrainlib/store_traverse.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 5493f513..68cfdf06 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 0 — Capture current compatibility +- Phase: 1 — Extract topology into `store` - State: in progress -- Next action: run the full Phase 0 verification, inspect the diff, commit, - and tag the phase boundary. +- Next action: build the shared topology extraction and run focused topology + plus compatibility tests. ## Phase log @@ -43,6 +43,17 @@ This file is the resumable implementation log for `dag::ThreadSafeStorage`. - State: complete. +### Phase 1 — Extract topology into `store` + +- Added shared node-status types, hierarchy traits, invalid-key errors, + sparse index, and traversal under `terrainlib/store`. +- Added `octree::StoreTraits` and `raster_store::StoreTraits`. +- Converted `octree::IndexMap` and `octree::traverse` to compatibility + wrappers over the shared implementation. +- Added common 2D/3D topology and traversal coverage plus 2D boundary and + invalid-key tests. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -53,6 +64,18 @@ This file is the resumable implementation log for - Phase 0 `unittests_terrainlib`: 23,732 assertions in 397 test cases passed. - Phase 0 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. - Phase 0 `git diff --check`: passed. +- Phase 1 focused shared-store tests: 67 assertions in 7 test cases passed. +- Phase 1 pre-refactor terrain compatibility tests: 56 assertions in 6 test + cases passed. +- Phase 1 `unittests_terrainlib`: 23,799 assertions in 404 test cases passed. +- Phase 1 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. +- Phase 1 application builds passed for `sf-builder`, `dag-builder`, and + `dag-convert-debug`. The current Debug configuration has `sf-merger` and + `sf-index-browser` disabled. +- The all-target build remains blocked by a pre-existing, unrelated call to + removed `radix::tile::Scheme` in `unittests/sf_builder/texture.cpp`; the + affected refactor targets and suites build independently. +- Phase 1 `git diff --check`: passed. ## Commit and tag log diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index a51a7821..4046109b 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -37,7 +37,6 @@ add_library(terrainlib mesh/validate.cpp octree/storage/helpers.cpp - octree/IndexMap.cpp octree/Space.cpp octree/OddLevelShifted.cpp diff --git a/src/terrainlib/octree/IndexMap.cpp b/src/terrainlib/octree/IndexMap.cpp deleted file mode 100644 index 8bb7e53c..00000000 --- a/src/terrainlib/octree/IndexMap.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include - -#include "octree/IndexMap.h" -#include "log.h" - -namespace octree { - -std::optional IndexMap::get(Id id) const { - const NodeStatus* status = this->get_raw(id); - if (status != nullptr) { - return *status; - } else { - return std::nullopt; - } -} - -bool IndexMap::add(Id id) { - NodeStatus* status = this->get_raw(id); - if (status != nullptr) { - switch (*status) { - case NodeStatus::Inner: - case NodeStatus::Leaf: - // Nothing to do - return false; - case NodeStatus::Virtual: - this->set_raw(id, NodeStatus::Inner); - return true; - } - } - - const auto parent_opt = id.parent(); - if (!parent_opt.has_value()) { - DEBUG_ASSERT(id.is_root()); - if (this->empty()) { - this->set_raw(id, NodeStatus::Leaf); - return true; - } else { - UNREACHABLE(); - return true; - } - } - - const auto parent = parent_opt.value(); - const NodeStatus* parent_status = this->get_raw(parent); - if (parent_status == nullptr) { - this->add(parent); - this->set_raw(parent, NodeStatus::Virtual); - this->set_raw(id, NodeStatus::Leaf); - return true; - } - - switch (*parent_status) { - case NodeStatus::Leaf: - this->set_raw(parent, NodeStatus::Inner); - this->set_raw(id, NodeStatus::Leaf); - break; - case NodeStatus::Inner: - case NodeStatus::Virtual: - this->set_raw(id, NodeStatus::Leaf); - break; - } - - return true; -} - -bool IndexMap::remove(Id id) { - NodeStatus* status = this->get_raw(id); - if (status == nullptr) { - return false; - } - - switch (*status) { - case NodeStatus::Inner: - *status = NodeStatus::Virtual; - return true; - case NodeStatus::Virtual: - // Nothing to do - return false; - case NodeStatus::Leaf: - this->remove_raw(id); - this->update_parent_after_remove(id); - return true; - } - - UNREACHABLE(); -} - -bool IndexMap::is_present(Id id) const { - auto it = this->_index.find(id); - return it != this->_index.end(); -} -bool IndexMap::is_absent(Id id) const { - return !this->is_present(id); -} -bool IndexMap::is(NodeStatus status, Id id) const { - return this->get(id) == status; -} - -void IndexMap::clear() { - this->_index.clear(); -} -bool IndexMap::empty() const { - return this->_index.empty(); -} -size_t IndexMap::size() const { - return this->_index.size(); -} - -IndexMap::const_iterator IndexMap::begin() const { - return this->_index.begin(); -} -IndexMap::const_iterator IndexMap::end() const { - return this->_index.end(); -} -IndexMap::const_iterator IndexMap::cbegin() const { - return this->_index.cbegin(); -} -IndexMap::const_iterator IndexMap::cend() const { - return this->_index.cend(); -} - -NodeStatus* IndexMap::get_raw(Id id) { - if (auto it = this->_index.find(id); it != this->_index.end()) { - return &it->second; - } - return nullptr; -} -const NodeStatus* IndexMap::get_raw(Id id) const { - if (auto it = this->_index.find(id); it != this->_index.end()) { - return &it->second; - } - return nullptr; -} -void IndexMap::set_raw(Id id, NodeStatus status) { - this->_index[id] = status; -} -void IndexMap::remove_raw(Id id) { - this->_index.erase(id); -} - -void IndexMap::update_parent_after_remove(Id id) { - const auto parent_opt = id.parent(); - if (!parent_opt.has_value()) { - return; - } - - const auto parent = parent_opt.value(); - NodeStatus* parent_status = this->get_raw(parent); - DEBUG_ASSERT(parent_status != nullptr); - - // We know there are children since we just removed one - DEBUG_ASSERT(parent.has_children()); - const auto siblings = parent.children().value(); - for (const auto& sibling : siblings) { - if (this->is_present(sibling) && sibling != id) { - return; - } - } - - switch (*parent_status) { - case NodeStatus::Inner: - *parent_status = NodeStatus::Leaf; - break; - case NodeStatus::Virtual: - this->remove_raw(parent); - this->update_parent_after_remove(parent); - break; - case NodeStatus::Leaf: - UNREACHABLE(); - break; - } -} - -} diff --git a/src/terrainlib/octree/IndexMap.h b/src/terrainlib/octree/IndexMap.h index c5dbf535..c231e763 100644 --- a/src/terrainlib/octree/IndexMap.h +++ b/src/terrainlib/octree/IndexMap.h @@ -1,49 +1,88 @@ #pragma once -#include #include #include -#include "octree/Id.h" #include "octree/NodeStatus.h" +#include "octree/StoreTraits.h" +#include "store/Index.h" namespace octree { class IndexMap { public: - using Container = std::unordered_map; - using iterator = Container::iterator; - using const_iterator = Container::const_iterator; + using SharedIndex = store::Index; + using iterator = SharedIndex::iterator; + using const_iterator = SharedIndex::const_iterator; - std::optional get(Id id) const; - bool add(Id id); - bool remove(Id id); + std::optional get(const Id id) const { + return _index.get(id).value(); + } + bool add(const Id id) { + return _index.add(id).value(); + } + bool remove(const Id id) { + return _index.remove(id).value(); + } + bool is_present(const Id id) const { + return _index.is_present(id).value(); + } + bool is_absent(const Id id) const { + return _index.is_absent(id).value(); + } + bool is(const NodeStatus status, const Id id) const { + return _index.is(status, id).value(); + } - bool is_present(Id id) const; - bool is_absent(Id id) const; - bool is(NodeStatus status, Id id) const; + void clear() { + _index.clear(); + } + bool empty() const { + return _index.empty(); + } + size_t size() const { + return _index.size(); + } - void clear(); - bool empty() const; - size_t size() const; + const_iterator begin() const { + return _index.begin(); + } + const_iterator end() const { + return _index.end(); + } + const_iterator cbegin() const { + return _index.cbegin(); + } + const_iterator cend() const { + return _index.cend(); + } - const_iterator begin() const; - const_iterator end() const; - const_iterator cbegin() const; - const_iterator cend() const; + NodeStatus *get_raw(const Id id) { + return _index.get_raw(id).value(); + } + const NodeStatus *get_raw(const Id id) const { + return _index.get_raw(id).value(); + } + void set_raw(const Id id, const NodeStatus status) { + _index.set_raw(id, status).value(); + } + void remove_raw(const Id id) { + _index.remove_raw(id).value(); + } - NodeStatus* get_raw(Id id); - const NodeStatus* get_raw(Id id) const; - void set_raw(Id id, NodeStatus status); - void remove_raw(Id id); + SharedIndex &shared() { + return _index; + } + const SharedIndex &shared() const { + return _index; + } using serialize = zpp::bits::members<1>; friend zpp::bits::access; private: - Container _index; - void update_parent_after_remove(Id id); + SharedIndex _index; }; } // namespace octree diff --git a/src/terrainlib/octree/NodeStatus.h b/src/terrainlib/octree/NodeStatus.h index 9e00d97c..185f4fdd 100644 --- a/src/terrainlib/octree/NodeStatus.h +++ b/src/terrainlib/octree/NodeStatus.h @@ -1,81 +1,7 @@ #pragma once -#include - -#include "log.h" +#include "store/NodeStatus.h" namespace octree { - -class NodeStatus { -public: - using Underlying = uint8_t; - enum Value : Underlying { - Leaf = 0, - Inner = 1, - Virtual = 2 // not present on disk but has children - }; - - constexpr NodeStatus() = default; - constexpr NodeStatus(Value value) : _value(value) {} - - constexpr operator Value() const { - return this->_value; - } - explicit operator bool() const = delete; - constexpr bool operator==(NodeStatus other) const { - return this->_value == other._value; - } - constexpr bool operator!=(NodeStatus other) const { - return !(*this == other); - } - constexpr bool operator==(Value other_value) const { - return this->_value == other_value; - } - constexpr bool operator!=(Value other_value) const { - return this->_value != other_value; - } - - std::string to_string() const; - -private: - Value _value; - -public: - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; -}; - -} - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const octree::NodeStatus &status, FormatContext &ctx) const { - switch (status) { - case octree::NodeStatus::Leaf: - return fmt::format_to(ctx.out(), "Leaf"); - case octree::NodeStatus::Inner: - return fmt::format_to(ctx.out(), "Inner"); - case octree::NodeStatus::Virtual: - return fmt::format_to(ctx.out(), "Virtual"); - default: - UNREACHABLE(); - } - } -}; - -#include -#include -inline std::string octree::NodeStatus::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const octree::NodeStatus &status) { - fmt::print(os, "{}", status); - return os; +using NodeStatus = store::NodeStatus; } diff --git a/src/terrainlib/octree/NodeStatusOrMissing.h b/src/terrainlib/octree/NodeStatusOrMissing.h index 0868cb93..1c71a67d 100644 --- a/src/terrainlib/octree/NodeStatusOrMissing.h +++ b/src/terrainlib/octree/NodeStatusOrMissing.h @@ -1,138 +1,7 @@ #pragma once -#include - -#include - -#include "octree/NodeStatus.h" - -namespace octree { - -class NodeStatusOrMissing { -public: - using Underlying = NodeStatus::Underlying; - enum Value : Underlying { - Leaf = 0, - Inner = 1, - Virtual = 2, // not present on disk but has children - Missing = 3 - }; - - constexpr NodeStatusOrMissing() = default; - constexpr NodeStatusOrMissing(Value value) : _value(value) {} - NodeStatusOrMissing(NodeStatus opt) : _value(from_status(opt)) {} - NodeStatusOrMissing(std::optional opt) : _value(from_optional_status(opt)) {} - - constexpr operator Value() const { - return this->_value; - } - constexpr operator std::optional() const { - return this->to_optional_status(); - } - explicit operator bool() const = delete; - constexpr bool operator==(NodeStatusOrMissing other) const { - return this->_value == other._value; - } - constexpr bool operator!=(NodeStatusOrMissing other) const { - return !(*this == other); - } - constexpr bool operator==(Value other_value) const { - return this->_value == other_value; - } - constexpr bool operator!=(Value other_value) const { - return this->_value != other_value; - } - - std::string to_string() const; - -private: - constexpr static NodeStatusOrMissing from_status(NodeStatus status) noexcept { - return static_cast(static_cast(status)); - } - constexpr static NodeStatusOrMissing from_optional_status(std::optional opt) noexcept { - if (opt.has_value()) { - return from_status(opt.value()); - } else { - return NodeStatusOrMissing::Missing; - } - } - constexpr std::optional to_optional_status() const noexcept { - if (this->_value == NodeStatusOrMissing::Missing) { - return std::nullopt; - } else { - return static_cast(this->_value); - } - } - -public: - Value _value; -}; - -} - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const octree::NodeStatusOrMissing &status, FormatContext &ctx) const { - switch (status) { - case octree::NodeStatusOrMissing::Leaf: - return fmt::format_to(ctx.out(), "Leaf"); - case octree::NodeStatusOrMissing::Inner: - return fmt::format_to(ctx.out(), "Inner"); - case octree::NodeStatusOrMissing::Virtual: - return fmt::format_to(ctx.out(), "Virtual"); - case octree::NodeStatusOrMissing::Missing: - return fmt::format_to(ctx.out(), "Missing"); - default: - UNREACHABLE(); - } - } -}; - -#include -#include -inline std::string octree::NodeStatusOrMissing::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const octree::NodeStatusOrMissing &status) { - fmt::print(os, "{}", status); - return os; -} +#include "store/NodeStatusOrMissing.h" namespace octree { -namespace { -template -consteval bool _verify_enum_values(std::index_sequence) { - constexpr auto ns_values = magic_enum::enum_values(); - constexpr auto nsom_values = magic_enum::enum_values(); - - return ((magic_enum::enum_underlying(ns_values[Is]) == - magic_enum::enum_underlying(nsom_values[Is])) && - ...); -} - -consteval bool _verify_enum() { - static_assert( - std::is_same_v, - magic_enum::underlying_type_t>, - "Mismatched underlying type"); - - constexpr auto ns_values = magic_enum::enum_values(); - constexpr auto nsom_values = magic_enum::enum_values(); - - static_assert(nsom_values.size() == ns_values.size() + 1, "Mismatched enum counts"); - - static_assert(_verify_enum_values(std::make_index_sequence()), - "Enum value mismatch"); - - return true; -} -static_assert(_verify_enum()); -} +using NodeStatusOrMissing = store::NodeStatusOrMissing; } diff --git a/src/terrainlib/octree/StoreTraits.h b/src/terrainlib/octree/StoreTraits.h new file mode 100644 index 00000000..655656a4 --- /dev/null +++ b/src/terrainlib/octree/StoreTraits.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "octree/Id.h" + +namespace octree { + +struct StoreTraits { + using Key = Id; + using Hasher = std::hash; + + static constexpr Key root() { + return Key::root(); + } + static constexpr auto parent(const Key &key) { + return key.parent(); + } + static constexpr auto children(const Key &key) { + return key.children(); + } + static constexpr bool is_valid(const Key &key) { + return key.level() <= Key::max_level() + && key.index_on_level() <= Key::max_index_on_level(key.level()); + } +}; + +} // namespace octree diff --git a/src/terrainlib/octree/traverse.h b/src/terrainlib/octree/traverse.h index b912a0fa..6d0af8db 100644 --- a/src/terrainlib/octree/traverse.h +++ b/src/terrainlib/octree/traverse.h @@ -1,81 +1,35 @@ #pragma once #include -#include +#include -#include "octree/Id.h" -#include "octree/Storage.h" -#include "log.h" +#include + +#include "octree/IndexMap.h" +#include "store/traverse.h" namespace octree { -enum class TraversalOrder { - DepthFirst, - BreadthFirst -}; +using TraversalOrder = store::TraversalOrder; constexpr bool always_refine(const Id &) { return true; } -template < - typename VisitFn, - typename RefineFn = std::function> +template> void traverse( const IndexMap &index, - VisitFn &&visit_fn, - RefineFn &&refine_fn = always_refine, + VisitFn &&visit, + RefineFn &&refine = always_refine, const Id &root = Id::root(), - TraversalOrder order = TraversalOrder::DepthFirst) { - if (!index.is_present(root)) { - return; - } - - if (order == TraversalOrder::DepthFirst) { - std::function dfs; - dfs = [&](const Id& current) { - auto current_status_opt = index.get(current); - if (!current_status_opt) { - return; - } - const auto current_status = current_status_opt.value(); - - visit_fn(current, current_status); - - if (current.has_children() && refine_fn(current)) { - const auto children = current.children().value(); - for (const auto& child : children) { - dfs(child); - } - } - }; - dfs(root); - } else if (order == TraversalOrder::BreadthFirst) { - std::queue queue; - queue.push(root); - - while (!queue.empty()) { - Id current = queue.front(); - queue.pop(); - - auto current_status_opt = index.get(current); - if (!current_status_opt) { - continue; - } - const auto current_status = current_status_opt.value(); - - visit_fn(current, current_status); - - if (current.has_children() && refine_fn(current)) { - const auto children = current.children().value(); - for (const auto& child : children) { - queue.push(child); - } - } - } - } else { - UNREACHABLE(); - } + const TraversalOrder order = TraversalOrder::DepthFirst) { + const auto result = store::traverse( + index.shared(), + std::forward(visit), + std::forward(refine), + root, + order); + DEBUG_ASSERT(result.has_value()); } } // namespace octree diff --git a/src/terrainlib/raster_store/StoreTraits.h b/src/terrainlib/raster_store/StoreTraits.h new file mode 100644 index 00000000..a6ba4a8b --- /dev/null +++ b/src/terrainlib/raster_store/StoreTraits.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace raster_store { + +struct StoreTraits { + using Key = radix::tile::Id; + using Hasher = Key::Hasher; + + static constexpr unsigned max_zoom_level = std::numeric_limits::digits; + + static Key root() { + return {0, {0, 0}}; + } + static std::optional parent(const Key &key) { + if (key.zoom_level == 0) { + return std::nullopt; + } + return key.parent(); + } + static std::optional> children(const Key &key) { + if (key.zoom_level >= max_zoom_level) { + return std::nullopt; + } + return key.children(); + } + static bool is_valid(const Key &key) { + if (key.zoom_level > max_zoom_level) { + return false; + } + if (key.zoom_level == max_zoom_level) { + return true; + } + const uint32_t extent = uint32_t{1} << key.zoom_level; + return key.coords.x < extent && key.coords.y < extent; + } +}; + +} // namespace raster_store diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h new file mode 100644 index 00000000..22664de1 --- /dev/null +++ b/src/terrainlib/store/Index.h @@ -0,0 +1,239 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "log.h" +#include "store/InvalidKey.h" +#include "store/NodeStatus.h" +#include "store/Traits.h" + +namespace store { + +template +class Index { +public: + using Key = typename Traits::Key; + using Error = InvalidKey; + using Container = std::unordered_map; + using iterator = typename Container::iterator; + using const_iterator = typename Container::const_iterator; + + std::expected, Error> get(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + const NodeStatus *status = get_raw_unchecked(key); + if (status == nullptr) { + return std::nullopt; + } + return *status; + } + + std::expected add(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + + NodeStatus *status = get_raw_unchecked(key); + if (status != nullptr) { + switch (*status) { + case NodeStatus::Inner: + case NodeStatus::Leaf: + return false; + case NodeStatus::Virtual: + set_raw_unchecked(key, NodeStatus::Inner); + return true; + } + } + + const auto parent = Traits::parent(key); + if (!parent.has_value()) { + DEBUG_ASSERT(key == Traits::root()); + if (empty()) { + set_raw_unchecked(key, NodeStatus::Leaf); + return true; + } + UNREACHABLE(); + } + + if (!Traits::is_valid(parent.value())) { + return std::unexpected(Error{parent.value()}); + } + NodeStatus *parent_status = get_raw_unchecked(parent.value()); + if (parent_status == nullptr) { + const auto parent_result = add(parent.value()); + if (!parent_result.has_value()) { + return std::unexpected(parent_result.error()); + } + set_raw_unchecked(parent.value(), NodeStatus::Virtual); + set_raw_unchecked(key, NodeStatus::Leaf); + return true; + } + + switch (*parent_status) { + case NodeStatus::Leaf: + set_raw_unchecked(parent.value(), NodeStatus::Inner); + set_raw_unchecked(key, NodeStatus::Leaf); + break; + case NodeStatus::Inner: + case NodeStatus::Virtual: + set_raw_unchecked(key, NodeStatus::Leaf); + break; + } + return true; + } + + std::expected remove(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + + NodeStatus *status = get_raw_unchecked(key); + if (status == nullptr) { + return false; + } + switch (*status) { + case NodeStatus::Inner: + *status = NodeStatus::Virtual; + return true; + case NodeStatus::Virtual: + return false; + case NodeStatus::Leaf: + remove_raw_unchecked(key); + return update_parent_after_remove(key); + } + UNREACHABLE(); + } + + std::expected is_present(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return _index.contains(key); + } + std::expected is_absent(const Key &key) const { + const auto present = is_present(key); + if (!present.has_value()) { + return std::unexpected(present.error()); + } + return !present.value(); + } + std::expected is(const NodeStatus status, const Key &key) const { + const auto result = get(key); + if (!result.has_value()) { + return std::unexpected(result.error()); + } + return result.value() == status; + } + + std::expected get_raw(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return get_raw_unchecked(key); + } + std::expected get_raw(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + return get_raw_unchecked(key); + } + std::expected set_raw(const Key &key, const NodeStatus status) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + set_raw_unchecked(key, status); + return {}; + } + std::expected remove_raw(const Key &key) { + if (!Traits::is_valid(key)) { + return std::unexpected(Error{key}); + } + remove_raw_unchecked(key); + return {}; + } + + void clear() { + _index.clear(); + } + bool empty() const { + return _index.empty(); + } + size_t size() const { + return _index.size(); + } + + const_iterator begin() const { + return _index.begin(); + } + const_iterator end() const { + return _index.end(); + } + const_iterator cbegin() const { + return _index.cbegin(); + } + const_iterator cend() const { + return _index.cend(); + } + + using serialize = zpp::bits::members<1>; + friend zpp::bits::access; + +private: + NodeStatus *get_raw_unchecked(const Key &key) { + const auto iterator = _index.find(key); + return iterator == _index.end() ? nullptr : &iterator->second; + } + const NodeStatus *get_raw_unchecked(const Key &key) const { + const auto iterator = _index.find(key); + return iterator == _index.end() ? nullptr : &iterator->second; + } + void set_raw_unchecked(const Key &key, const NodeStatus status) { + _index[key] = status; + } + void remove_raw_unchecked(const Key &key) { + _index.erase(key); + } + + std::expected update_parent_after_remove(const Key &key) { + const auto parent = Traits::parent(key); + if (!parent.has_value()) { + return true; + } + if (!Traits::is_valid(parent.value())) { + return std::unexpected(Error{parent.value()}); + } + + NodeStatus *parent_status = get_raw_unchecked(parent.value()); + DEBUG_ASSERT(parent_status != nullptr); + const auto siblings = Traits::children(parent.value()); + DEBUG_ASSERT(siblings.has_value()); + for (const Key &sibling : siblings.value()) { + if (sibling != key && get_raw_unchecked(sibling) != nullptr) { + return true; + } + } + + switch (*parent_status) { + case NodeStatus::Inner: + *parent_status = NodeStatus::Leaf; + break; + case NodeStatus::Virtual: + remove_raw_unchecked(parent.value()); + return update_parent_after_remove(parent.value()); + case NodeStatus::Leaf: + UNREACHABLE(); + } + return true; + } + + Container _index; +}; + +} // namespace store diff --git a/src/terrainlib/store/InvalidKey.h b/src/terrainlib/store/InvalidKey.h new file mode 100644 index 00000000..d8caad27 --- /dev/null +++ b/src/terrainlib/store/InvalidKey.h @@ -0,0 +1,12 @@ +#pragma once + +namespace store { + +template +struct InvalidKey { + Key key; + + bool operator==(const InvalidKey &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/NodeStatus.h b/src/terrainlib/store/NodeStatus.h new file mode 100644 index 00000000..775cf246 --- /dev/null +++ b/src/terrainlib/store/NodeStatus.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include "log.h" + +namespace store { + +class NodeStatus { +public: + using Underlying = uint8_t; + enum Value : Underlying { + Leaf = 0, + Inner = 1, + Virtual = 2, + }; + + constexpr NodeStatus() = default; + constexpr NodeStatus(const Value value) : _value(value) {} + + constexpr operator Value() const { + return _value; + } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatus other) const { + return _value == other._value; + } + constexpr bool operator!=(const NodeStatus other) const { + return !(*this == other); + } + constexpr bool operator==(const Value other) const { + return _value == other; + } + constexpr bool operator!=(const Value other) const { + return _value != other; + } + + std::string to_string() const; + +private: + Value _value; + +public: + using serialize = zpp::bits::members<1>; + friend zpp::bits::access; +}; + +} // namespace store + +template<> +struct fmt::formatter { + template + constexpr auto parse(ParseContext &context) { + return context.begin(); + } + + template + auto format(const store::NodeStatus &status, FormatContext &context) const { + switch (status) { + case store::NodeStatus::Leaf: + return fmt::format_to(context.out(), "Leaf"); + case store::NodeStatus::Inner: + return fmt::format_to(context.out(), "Inner"); + case store::NodeStatus::Virtual: + return fmt::format_to(context.out(), "Virtual"); + default: + UNREACHABLE(); + } + } +}; + +inline std::string store::NodeStatus::to_string() const { + return fmt::format("{}", *this); +} + +inline std::ostream &operator<<(std::ostream &stream, const store::NodeStatus &status) { + fmt::print(stream, "{}", status); + return stream; +} diff --git a/src/terrainlib/store/NodeStatusOrMissing.h b/src/terrainlib/store/NodeStatusOrMissing.h new file mode 100644 index 00000000..4157b57d --- /dev/null +++ b/src/terrainlib/store/NodeStatusOrMissing.h @@ -0,0 +1,134 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "log.h" +#include "store/NodeStatus.h" + +namespace store { + +class NodeStatusOrMissing { +public: + using Underlying = NodeStatus::Underlying; + enum Value : Underlying { + Leaf = 0, + Inner = 1, + Virtual = 2, + Missing = 3, + }; + + constexpr NodeStatusOrMissing() = default; + constexpr NodeStatusOrMissing(const Value value) : _value(value) {} + NodeStatusOrMissing(const NodeStatus status) : _value(from_status(status)) {} + NodeStatusOrMissing(const std::optional status) : _value(from_optional_status(status)) {} + + constexpr operator Value() const { + return _value; + } + constexpr operator std::optional() const { + return to_optional_status(); + } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatusOrMissing other) const { + return _value == other._value; + } + constexpr bool operator!=(const NodeStatusOrMissing other) const { + return !(*this == other); + } + constexpr bool operator==(const Value other) const { + return _value == other; + } + constexpr bool operator!=(const Value other) const { + return _value != other; + } + + std::string to_string() const; + +private: + static constexpr NodeStatusOrMissing from_status(const NodeStatus status) noexcept { + return static_cast(static_cast(status)); + } + static constexpr NodeStatusOrMissing from_optional_status(const std::optional status) noexcept { + return status.has_value() + ? from_status(status.value()) + : NodeStatusOrMissing(NodeStatusOrMissing::Missing); + } + constexpr std::optional to_optional_status() const noexcept { + if (_value == Missing) { + return std::nullopt; + } + return static_cast(_value); + } + +public: + Value _value; +}; + +namespace detail { +template +consteval bool verify_node_status_values(std::index_sequence) { + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + return ((magic_enum::enum_underlying(status_values[Indices]) + == magic_enum::enum_underlying(optional_values[Indices])) + && ...); +} + +consteval bool verify_node_status_enums() { + static_assert( + std::is_same_v< + magic_enum::underlying_type_t, + magic_enum::underlying_type_t>); + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + static_assert(optional_values.size() == status_values.size() + 1); + static_assert(verify_node_status_values(std::make_index_sequence())); + return true; +} + +static_assert(verify_node_status_enums()); +} // namespace detail + +} // namespace store + +template<> +struct fmt::formatter { + template + constexpr auto parse(ParseContext &context) { + return context.begin(); + } + + template + auto format(const store::NodeStatusOrMissing &status, FormatContext &context) const { + switch (status) { + case store::NodeStatusOrMissing::Leaf: + return fmt::format_to(context.out(), "Leaf"); + case store::NodeStatusOrMissing::Inner: + return fmt::format_to(context.out(), "Inner"); + case store::NodeStatusOrMissing::Virtual: + return fmt::format_to(context.out(), "Virtual"); + case store::NodeStatusOrMissing::Missing: + return fmt::format_to(context.out(), "Missing"); + default: + UNREACHABLE(); + } + } +}; + +inline std::string store::NodeStatusOrMissing::to_string() const { + return fmt::format("{}", *this); +} + +inline std::ostream &operator<<(std::ostream &stream, const store::NodeStatusOrMissing &status) { + fmt::print(stream, "{}", status); + return stream; +} diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h new file mode 100644 index 00000000..80fe88a7 --- /dev/null +++ b/src/terrainlib/store/Traits.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace store { + +template +concept HierarchyTraits = requires(typename Traits::Key key) { + typename Traits::Key; + typename Traits::Hasher; + { Traits::root() } -> std::same_as; + Traits::parent(key); + Traits::children(key); + { Traits::is_valid(key) } -> std::same_as; +}; + +} // namespace store diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h new file mode 100644 index 00000000..8dbfe948 --- /dev/null +++ b/src/terrainlib/store/traverse.h @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include + +#include "store/Index.h" + +namespace store { + +enum class TraversalOrder { + DepthFirst, + BreadthFirst, +}; + +struct AlwaysRefine { + template + constexpr bool operator()(const Key &) const { + return true; + } +}; + +template +std::expected> traverse( + const Index &index, + VisitFn &&visit, + RefineFn &&refine = {}, + const typename Traits::Key &root = Traits::root(), + const TraversalOrder order = TraversalOrder::DepthFirst) { + using Key = typename Traits::Key; + using Error = InvalidKey; + + if (!Traits::is_valid(root)) { + return std::unexpected(Error{root}); + } + const auto root_status = index.get(root); + if (!root_status.has_value()) { + return std::unexpected(root_status.error()); + } + if (!root_status.value().has_value()) { + return {}; + } + + if (order == TraversalOrder::DepthFirst) { + std::function(const Key &)> depth_first; + depth_first = [&](const Key ¤t) -> std::expected { + const auto status = index.get(current); + if (!status.has_value()) { + return std::unexpected(status.error()); + } + if (!status.value().has_value()) { + return {}; + } + visit(current, status.value().value()); + + if (refine(current)) { + const auto children = Traits::children(current); + if (children.has_value()) { + for (const Key &child : children.value()) { + const auto result = depth_first(child); + if (!result.has_value()) { + return result; + } + } + } + } + return {}; + }; + return depth_first(root); + } + + std::queue queue; + queue.push(root); + while (!queue.empty()) { + const Key current = queue.front(); + queue.pop(); + + const auto status = index.get(current); + if (!status.has_value()) { + return std::unexpected(status.error()); + } + if (!status.value().has_value()) { + continue; + } + visit(current, status.value().value()); + + if (refine(current)) { + const auto children = Traits::children(current); + if (children.has_value()) { + for (const Key &child : children.value()) { + queue.push(child); + } + } + } + } + return {}; +} + +} // namespace store diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 7d89ca6f..3ebf239c 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -31,6 +31,8 @@ add_executable(unittests_terrainlib terrainlib/segmented_buffer.cpp terrainlib/stamp_set.cpp terrainlib/store_compatibility.cpp + terrainlib/store_index.cpp + terrainlib/store_traverse.cpp terrainlib/index_map.cpp terrainlib/index_traverse.cpp terrainlib/mesh_boundary.cpp diff --git a/unittests/terrainlib/store_index.cpp b/unittests/terrainlib/store_index.cpp new file mode 100644 index 00000000..454d7948 --- /dev/null +++ b/unittests/terrainlib/store_index.cpp @@ -0,0 +1,81 @@ +#include +#include + +#include +#include + +#include "octree/StoreTraits.h" +#include "raster_store/StoreTraits.h" +#include "store/Index.h" + +static_assert(store::HierarchyTraits); +static_assert(store::HierarchyTraits); + +TEMPLATE_TEST_CASE( + "shared hierarchy index transitions", + "[store][index]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + using Index = store::Index; + + Index index; + const auto root = Traits::root(); + const auto children = Traits::children(root).value(); + const auto first = children[0]; + const auto second = children[1]; + + REQUIRE(index.add(first).value()); + CHECK(index.is(store::NodeStatus::Virtual, root).value()); + CHECK(index.is(store::NodeStatus::Leaf, first).value()); + + REQUIRE(index.add(root).value()); + CHECK(index.is(store::NodeStatus::Inner, root).value()); + CHECK_FALSE(index.add(root).value()); + + REQUIRE(index.add(second).value()); + REQUIRE(index.remove(first).value()); + CHECK(index.is(store::NodeStatus::Inner, root).value()); + REQUIRE(index.remove(second).value()); + CHECK(index.is(store::NodeStatus::Leaf, root).value()); + REQUIRE(index.remove(root).value()); + CHECK(index.empty()); +} + +TEST_CASE("raster store traits validate tile boundaries", "[store][index][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + + CHECK(Traits::is_valid(Key{0, {0, 0}})); + CHECK_FALSE(Traits::is_valid(Key{0, {1, 0}})); + CHECK(Traits::is_valid(Key{31, {uint32_t{0x7fffffff}, uint32_t{0x7fffffff}}})); + CHECK_FALSE(Traits::is_valid(Key{31, {uint32_t{0x80000000}, 0}})); + CHECK(Traits::is_valid(Key{32, {0, 0}})); + CHECK(Traits::is_valid(Key{32, {UINT32_MAX, UINT32_MAX}})); + CHECK_FALSE(Traits::is_valid(Key{33, {0, 0}})); + + CHECK_FALSE(Traits::parent(Traits::root()).has_value()); + CHECK_FALSE(Traits::children(Key{32, {UINT32_MAX, UINT32_MAX}}).has_value()); + + const Key level_31{31, {uint32_t{0x7fffffff}, uint32_t{0x7fffffff}}}; + const auto children = Traits::children(level_31); + REQUIRE(children.has_value()); + CHECK(children->at(0) == Key{32, {UINT32_MAX - 1, UINT32_MAX - 1}}); + CHECK(children->at(1) == Key{32, {UINT32_MAX, UINT32_MAX - 1}}); + CHECK(children->at(2) == Key{32, {UINT32_MAX - 1, UINT32_MAX}}); + CHECK(children->at(3) == Key{32, {UINT32_MAX, UINT32_MAX}}); +} + +TEST_CASE("shared index rejects invalid keys", "[store][index][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + store::Index index; + const Key invalid{4, {16, 0}}; + + CHECK(index.get(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.add(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.remove(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is_present(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is_absent(invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.is(store::NodeStatus::Leaf, invalid) == std::unexpected(store::InvalidKey{invalid})); +} diff --git a/unittests/terrainlib/store_traverse.cpp b/unittests/terrainlib/store_traverse.cpp new file mode 100644 index 00000000..43992ed1 --- /dev/null +++ b/unittests/terrainlib/store_traverse.cpp @@ -0,0 +1,68 @@ +#include + +#include +#include + +#include "octree/StoreTraits.h" +#include "raster_store/StoreTraits.h" +#include "store/traverse.h" + +TEMPLATE_TEST_CASE( + "shared hierarchy traversal preserves trait child order", + "[store][traverse]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + using Key = typename Traits::Key; + + store::Index index; + const Key root = Traits::root(); + const auto children = Traits::children(root).value(); + const Key first = children[0]; + const Key second = children[1]; + const Key grandchild = Traits::children(first).value()[0]; + REQUIRE(index.add(root).has_value()); + REQUIRE(index.add(first).has_value()); + REQUIRE(index.add(second).has_value()); + REQUIRE(index.add(grandchild).has_value()); + + std::vector depth_first; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { depth_first.push_back(key); }) + .has_value()); + CHECK(depth_first == std::vector{root, first, grandchild, second}); + + std::vector breadth_first; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { breadth_first.push_back(key); }, + store::AlwaysRefine{}, + root, + store::TraversalOrder::BreadthFirst) + .has_value()); + CHECK(breadth_first == std::vector{root, first, second, grandchild}); + + std::vector explicit_root; + REQUIRE(store::traverse( + index, + [&](const Key &key, const store::NodeStatus) { explicit_root.push_back(key); }, + store::AlwaysRefine{}, + first) + .has_value()); + CHECK(explicit_root == std::vector{first, grandchild}); +} + +TEST_CASE("shared traversal rejects an invalid explicit root", "[store][traverse][raster]") { + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + const Key invalid{1, {2, 0}}; + store::Index index; + + const auto result = store::traverse( + index, + [](const Key &, const store::NodeStatus) {}, + store::AlwaysRefine{}, + invalid); + CHECK(result == std::unexpected(store::InvalidKey{invalid})); +} From fbc02d08543fdcfc3cca210e04acc80b9f961a60 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:37:15 +0200 Subject: [PATCH 05/43] Add runtime store layouts and codecs --- docs/raster-store/refactor-status.md | 31 ++- src/dag_builder/dag_id.h | 16 -- src/dag_builder/dag_node.h | 14 -- src/dag_builder/encoded.h | 173 +------------- src/dag_builder/metadata.h | 32 --- src/dag_builder/serialization.h | 212 ++++++++++++++++++ src/dag_builder/storage.h | 3 +- src/dag_builder/zpp_bits_glm.h | 20 -- src/terrainlib/mesh/codec/Gltf.h | 81 +++++++ src/terrainlib/mesh/codec/Terrain.h | 49 ++++ src/terrainlib/octree/store_layout/Flat.h | 39 ++++ .../LevelAndCoordinateDirectories.h | 49 ++++ src/terrainlib/octree/store_layout/Mappings.h | 38 ++++ src/terrainlib/store/Codec.h | 31 +++ src/terrainlib/store/CodecError.h | 38 ++++ src/terrainlib/store/Layout.h | 44 ++++ src/terrainlib/store/NodePath.h | 23 ++ src/terrainlib/store/PathMapping.h | 19 ++ src/terrainlib/store/codec/ZppBits.h | 57 +++++ unittests/CMakeLists.txt | 2 + .../dag_builder/storage_compatibility.cpp | 70 ++++++ unittests/terrainlib/store_codec.cpp | 202 +++++++++++++++++ unittests/terrainlib/store_layout.cpp | 70 ++++++ 23 files changed, 1061 insertions(+), 252 deletions(-) create mode 100644 src/dag_builder/serialization.h delete mode 100644 src/dag_builder/zpp_bits_glm.h create mode 100644 src/terrainlib/mesh/codec/Gltf.h create mode 100644 src/terrainlib/mesh/codec/Terrain.h create mode 100644 src/terrainlib/octree/store_layout/Flat.h create mode 100644 src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h create mode 100644 src/terrainlib/octree/store_layout/Mappings.h create mode 100644 src/terrainlib/store/Codec.h create mode 100644 src/terrainlib/store/CodecError.h create mode 100644 src/terrainlib/store/Layout.h create mode 100644 src/terrainlib/store/NodePath.h create mode 100644 src/terrainlib/store/PathMapping.h create mode 100644 src/terrainlib/store/codec/ZppBits.h create mode 100644 unittests/terrainlib/store_codec.cpp create mode 100644 unittests/terrainlib/store_layout.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 68cfdf06..6a2e1e83 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 1 — Extract topology into `store` +- Phase: 2 — Introduce path mappings and runtime codecs - State: in progress -- Next action: build the shared topology extraction and run focused topology - plus compatibility tests. +- Next action: run the complete Phase 2 verification, inspect the diff, + commit, and tag the phase boundary. ## Phase log @@ -54,6 +54,20 @@ This file is the resumable implementation log for invalid-key tests. - State: complete. +### Phase 2 — Introduce path mappings and runtime codecs + +- Added extensionless `NodePath`, `PathMapping`, and `Layout` values. +- Added the stateful runtime `Codec` interface and typed `CodecError`. +- Added runtime ZPP Bits, terrain, binary glTF, and JSON glTF codecs. +- Added ordinary `flat` and `level_and_coordinate_directories` mapping + function pairs plus explicit lookup. +- Consolidated DAG serialization in `dag_builder/serialization.h` and made + the legacy DAG storage adapter include it explicitly. +- Added mapping/codec composition, parser validation, single-/multi-file, + unsupported-operation, write-only, directory-creation, error-conversion, + and concurrent codec tests. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -76,6 +90,15 @@ This file is the resumable implementation log for removed `radix::tile::Scheme` in `unittests/sf_builder/texture.cpp`; the affected refactor targets and suites build independently. - Phase 1 `git diff --check`: passed. +- Phase 2 focused shared-store tests: 160 assertions in 17 test cases passed. +- Phase 2 DAG golden fixture: 30 assertions passed, including runtime codec + output byte equality. +- Phase 2 concurrent DAG runtime codec: 10 assertions passed. +- Phase 2 `unittests_terrainlib`: 23,892 assertions in 414 test cases passed. +- Phase 2 `unittests_dagbuilder`: 440 assertions in 71 test cases passed. +- Phase 2 application builds passed for `sf-builder`, `dag-builder`, and + `dag-convert-debug`. +- Phase 2 `git diff --check`: passed. ## Commit and tag log @@ -86,3 +109,5 @@ This file is the resumable implementation log for `.terrain` golden payloads ignored by the repository's general artifact rule. - `refactor_phase_0` — annotated Phase 0 completion tag. +- `4c171c8` / `refactor_phase_1` — shared topology extraction and Phase 1 + completion tag. diff --git a/src/dag_builder/dag_id.h b/src/dag_builder/dag_id.h index 232f66fd..3370d1bf 100644 --- a/src/dag_builder/dag_id.h +++ b/src/dag_builder/dag_id.h @@ -2,8 +2,6 @@ #include -#include - #include "octree/Id.h" #include "hash_utils.h" @@ -33,17 +31,3 @@ struct fmt::formatter { return fmt::format_to(ctx.out(), "{}:{}", id.source_batch, id.cluster_index); } }; - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -template -auto serialize(Archive &archive, dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -} diff --git a/src/dag_builder/dag_node.h b/src/dag_builder/dag_node.h index c276a27a..2b46e6ec 100644 --- a/src/dag_builder/dag_node.h +++ b/src/dag_builder/dag_node.h @@ -18,17 +18,3 @@ inline ClusterBatch make_leaf_batch(Clustering clustering) { } } - -namespace dag { - -template -auto serialize(Archive &archive, const dag::ClusterBatch &node) { - return archive(node.metadata, node.clustering); -} - -template -auto serialize(Archive &archive, dag::ClusterBatch &node) { - return archive(node.metadata, node.clustering); -} - -} // namespace dag diff --git a/src/dag_builder/encoded.h b/src/dag_builder/encoded.h index 2ccee0e9..e9b3510a 100644 --- a/src/dag_builder/encoded.h +++ b/src/dag_builder/encoded.h @@ -1,178 +1,21 @@ #pragma once -#include #include -#include -#include -#include #include -#include -#include -#include -#include "TextureSet.h" -#include "cluster.h" -#include "io/bytes.h" +#include "io/Error.h" #include "io/serialize.h" -#include "mesh/io/texture.h" -#include "meshopt.h" +#include "serialization.h" -template -auto serialize(Archive &archive, const TextureSet &textures) { - size_t size = textures.size(); - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - for (const auto &texture : textures) { - const mesh::io::ImageAndExt item{texture, ".jpeg"}; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - } - - return result; - } -} -template -auto serialize(Archive &archive, TextureSet &textures) { - size_t size; - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - std::vector images; - images.reserve(size); - for (size_t i = 0; i < size; i++) { - mesh::io::ImageAndExt item; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - - images.push_back(item.image); - } - textures = TextureSet(images); - - return result; - } -} - -template -auto serialize(Archive &archive, const Cluster &cluster) { - const size_t triangle_count = cluster.triangle_count(); - const size_t vertex_count = cluster.vertex_count(); - const size_t uv_count = cluster.uvs.size(); - - auto encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles); - auto encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices); - auto encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs); - - return archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); -} -template -auto serialize(Archive &archive, Cluster &cluster) { - size_t triangle_count; - size_t vertex_count; - size_t uv_count; - - std::vector encoded_triangles; - std::vector encoded_vertex_indices; - std::vector encoded_uvs; - - auto result = archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); - - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - meshopt::decode_index_buffer(cluster.local_triangles, triangle_count, encoded_triangles); - meshopt::decode_vertex_buffer(cluster.vertex_indices, vertex_count, encoded_vertex_indices); - meshopt::decode_vertex_buffer(cluster.uvs, uv_count, encoded_uvs); - - return result; - } -} - -template -auto serialize(Archive &archive, const Clustering &clustering) { - const size_t vertex_count = clustering.vertex_count(); - auto encoded_positions = meshopt::encode_vertex_buffer(clustering.positions); - - return archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); -} - -template -auto serialize(Archive &archive, Clustering &clustering) { - size_t vertex_count; - std::vector encoded_positions; - - auto result = archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); - - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - meshopt::decode_vertex_buffer(clustering.positions, vertex_count, encoded_positions); - - return result; - } -} - -inline std::expected -save_clustering(const Clustering &clustering, - const std::filesystem::path &path, - const bool make_dirs = true) { +inline std::expected save_clustering( + const Clustering &clustering, + const std::filesystem::path &path, + const bool make_dirs = true) { return ::io::write_to_path(clustering, path, make_dirs); } -inline std::expected -load_clustering(const std::filesystem::path &path) { +inline std::expected load_clustering( + const std::filesystem::path &path) { return ::io::read_from_path(path); } diff --git a/src/dag_builder/metadata.h b/src/dag_builder/metadata.h index 5527fe3a..6c12b2bf 100644 --- a/src/dag_builder/metadata.h +++ b/src/dag_builder/metadata.h @@ -8,7 +8,6 @@ #include "cluster.h" #include "dag_id.h" #include "utils.h" -#include "zpp_bits_glm.h" #include "octree/storage/defaults.h" #include "octree/storage/codec/ZppBitsCodec.h" #include "octree/storage/codec/ReadOnlyCodec.h" @@ -80,34 +79,3 @@ struct DefaultCodec { using type = ReadOnlyCodec>; }; } - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} -template -auto serialize(Archive &archive, radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} - -template -auto serialize(Archive &archive, const dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} -template -auto serialize(Archive &archive, dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} - -template -auto serialize(Archive &archive, const dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} -template -auto serialize(Archive &archive, dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} - -} // namespace zpp::bits diff --git a/src/dag_builder/serialization.h b/src/dag_builder/serialization.h new file mode 100644 index 00000000..b5c58481 --- /dev/null +++ b/src/dag_builder/serialization.h @@ -0,0 +1,212 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include "dag_node.h" +#include "glm_utils.h" +#include "io/bytes.h" +#include "mesh/io/texture.h" +#include "meshopt.h" + +namespace zpp::bits { + +template +auto serialize(Archive &archive, const glm::vec &vector) { + return archive(as_span(vector)); +} + +template +auto serialize(Archive &archive, glm::vec &vector) { + return archive(as_span(vector)); +} + +template +auto serialize(Archive &archive, const dag::Id &id) { + return archive(id.source_batch, id.cluster_index); +} + +template +auto serialize(Archive &archive, dag::Id &id) { + return archive(id.source_batch, id.cluster_index); +} + +template +auto serialize(Archive &archive, const radix::geometry::Aabb3d &bounds) { + return archive(bounds.min, bounds.max); +} + +template +auto serialize(Archive &archive, radix::geometry::Aabb3d &bounds) { + return archive(bounds.min, bounds.max); +} + +template +auto serialize(Archive &archive, const dag::Group &group) { + return archive(group.children, group.error, group.bounds); +} + +template +auto serialize(Archive &archive, dag::Group &group) { + return archive(group.children, group.error, group.bounds); +} + +template +auto serialize(Archive &archive, const dag::NodeMetadata &metadata) { + return archive(metadata.group_assignment, metadata.groups); +} + +template +auto serialize(Archive &archive, dag::NodeMetadata &metadata) { + return archive(metadata.group_assignment, metadata.groups); +} + +template +auto serialize(Archive &archive, const dag::ClusterBatch &batch) { + return archive(batch.metadata, batch.clustering); +} + +template +auto serialize(Archive &archive, dag::ClusterBatch &batch) { + return archive(batch.metadata, batch.clustering); +} + +} // namespace zpp::bits + +template +auto serialize(Archive &archive, const TextureSet &textures) { + size_t size = textures.size(); + auto result = archive(size); + using Result = std::remove_cvref_t; + if constexpr (!std::is_same_v + && !std::is_same_v) { + return result; + } else { + if (zpp::bits::failure(result)) { + return result; + } + for (const auto &texture : textures) { + const mesh::io::ImageAndExt item{texture, ".jpeg"}; + result = archive(item); + if (zpp::bits::failure(result)) { + return result; + } + } + return result; + } +} + +template +auto serialize(Archive &archive, TextureSet &textures) { + size_t size; + auto result = archive(size); + using Result = std::remove_cvref_t; + if constexpr (!std::is_same_v + && !std::is_same_v) { + return result; + } else { + if (zpp::bits::failure(result)) { + return result; + } + std::vector images; + images.reserve(size); + for (size_t index = 0; index < size; ++index) { + mesh::io::ImageAndExt item; + result = archive(item); + if (zpp::bits::failure(result)) { + return result; + } + images.push_back(item.image); + } + textures = TextureSet(images); + return result; + } +} + +template +auto serialize(Archive &archive, const Cluster &cluster) { + const size_t triangle_count = cluster.triangle_count(); + const size_t vertex_count = cluster.vertex_count(); + const size_t uv_count = cluster.uvs.size(); + auto encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles); + auto encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices); + auto encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs); + return archive( + triangle_count, + vertex_count, + uv_count, + encoded_triangles, + encoded_vertex_indices, + encoded_uvs, + cluster.id, + cluster.texture_id, + cluster.absolute_error); +} + +template +auto serialize(Archive &archive, Cluster &cluster) { + size_t triangle_count; + size_t vertex_count; + size_t uv_count; + std::vector encoded_triangles; + std::vector encoded_vertex_indices; + std::vector encoded_uvs; + auto result = archive( + triangle_count, + vertex_count, + uv_count, + encoded_triangles, + encoded_vertex_indices, + encoded_uvs, + cluster.id, + cluster.texture_id, + cluster.absolute_error); + using Result = std::remove_cvref_t; + if constexpr (!std::is_same_v + && !std::is_same_v) { + return result; + } else { + if (zpp::bits::failure(result)) { + return result; + } + meshopt::decode_index_buffer(cluster.local_triangles, triangle_count, encoded_triangles); + meshopt::decode_vertex_buffer(cluster.vertex_indices, vertex_count, encoded_vertex_indices); + meshopt::decode_vertex_buffer(cluster.uvs, uv_count, encoded_uvs); + return result; + } +} + +template +auto serialize(Archive &archive, const Clustering &clustering) { + const size_t vertex_count = clustering.vertex_count(); + auto encoded_positions = meshopt::encode_vertex_buffer(clustering.positions); + return archive(vertex_count, encoded_positions, clustering.clusters, clustering.textures); +} + +template +auto serialize(Archive &archive, Clustering &clustering) { + size_t vertex_count; + std::vector encoded_positions; + auto result = archive( + vertex_count, + encoded_positions, + clustering.clusters, + clustering.textures); + using Result = std::remove_cvref_t; + if constexpr (!std::is_same_v + && !std::is_same_v) { + return result; + } else { + if (zpp::bits::failure(result)) { + return result; + } + meshopt::decode_vertex_buffer(clustering.positions, vertex_count, encoded_positions); + return result; + } +} diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 8c520529..7fb24242 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -1,7 +1,6 @@ #pragma once -#include "dag_node.h" -#include "metadata.h" +#include "serialization.h" #include "octree/storage/Storage.h" #include "octree/storage/IndexedStorage.h" namespace octree { diff --git a/src/dag_builder/zpp_bits_glm.h b/src/dag_builder/zpp_bits_glm.h deleted file mode 100644 index 0dade30d..00000000 --- a/src/dag_builder/zpp_bits_glm.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include "glm_utils.h" - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const glm::vec &v) { - return archive(as_span(v)); -} - -template -auto serialize(Archive &archive, glm::vec &v) { - return archive(as_span(v)); -} - -} diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h new file mode 100644 index 00000000..9db26fe9 --- /dev/null +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/io/gltf.h" +#include "store/Codec.h" +#include "store/codec/ZppBits.h" + +namespace mesh::codec { + +enum class GltfContainer { + Binary, + Json, +}; + +class Gltf final : public store::Codec { +public: + explicit Gltf(const GltfContainer container) : _container(container) {} + + std::vector paths(const store::NodePath &node_path) const override { + return {store::codec::append_extension( + node_path, + _container == GltfContainer::Binary ? ".glb" : ".gltf")}; + } + + std::expected read( + const store::NodePath &node_path) const override { + const auto result = mesh::io::gltf::load_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + result.error() == mesh::io::LoadMeshErrorKind::FileNotFound + ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::InvalidData, + result.error().description(), + }); + } + return result.value(); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh) const override { + try { + const auto result = mesh::io::gltf::save_to_path(mesh, paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + result.error().description(), + }); + } + return {}; + } catch (const std::exception &error) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + error.what(), + }); + } catch (...) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + "unknown glTF writer exception", + }); + } + } + + GltfContainer container() const { + return _container; + } + +private: + GltfContainer _container; +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/Terrain.h b/src/terrainlib/mesh/codec/Terrain.h new file mode 100644 index 00000000..d1356e76 --- /dev/null +++ b/src/terrainlib/mesh/codec/Terrain.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/io/terrain.h" +#include "store/Codec.h" +#include "store/codec/ZppBits.h" + +namespace mesh::codec { + +class Terrain final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + return {store::codec::append_extension(node_path, ".terrain")}; + } + + std::expected read( + const store::NodePath &node_path) const override { + const auto result = mesh::io::terrain::load_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + result.error() == mesh::io::LoadMeshErrorKind::FileNotFound + ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::InvalidData, + result.error().description(), + }); + } + return result.value(); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh) const override { + const auto result = mesh::io::terrain::save_to_path(mesh, paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + result.error().description(), + }); + } + return {}; + } +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/octree/store_layout/Flat.h b/src/terrainlib/octree/store_layout/Flat.h new file mode 100644 index 00000000..c7f58a7f --- /dev/null +++ b/src/terrainlib/octree/store_layout/Flat.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "octree/Id.h" +#include "store/NodePath.h" +#include "string_utils.h" + +namespace octree::store_layout { + +inline store::NodePath flat_key_to_node_path(const Id &id) { + return store::NodePath(fmt::format("{}-{}", id.level(), id.index_on_level())); +} + +inline std::optional flat_node_path_to_key(const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.empty() || path.is_absolute() || path.has_parent_path() || path.has_extension()) { + return std::nullopt; + } + + const std::string text = path.string(); + const size_t separator = text.find('-'); + if (separator == std::string::npos || separator != text.rfind('-')) { + return std::nullopt; + } + const auto level = from_chars(std::string_view(text).substr(0, separator)); + const auto index = from_chars(std::string_view(text).substr(separator + 1)); + if (!level.has_value() || !index.has_value()) { + return std::nullopt; + } + return Id::try_make(level.value(), index.value()); +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h new file mode 100644 index 00000000..7036443a --- /dev/null +++ b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +#include + +#include "octree/Id.h" +#include "store/NodePath.h" +#include "string_utils.h" + +namespace octree::store_layout { + +inline store::NodePath level_and_coordinate_key_to_node_path(const Id &id) { + const Id::Coords coordinates = id.coords(); + return store::NodePath(fmt::format( + "{}/{}/{}/{}", + id.level(), + coordinates.x, + coordinates.y, + coordinates.z)); +} + +inline std::optional level_and_coordinate_node_path_to_key( + const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.empty() || path.is_absolute() || std::distance(path.begin(), path.end()) != 4) { + return std::nullopt; + } + + auto part = path.begin(); + const auto level = from_chars(part->string()); + ++part; + const auto x = from_chars(part->string()); + ++part; + const auto y = from_chars(part->string()); + ++part; + if (part->has_extension()) { + return std::nullopt; + } + const auto z = from_chars(part->string()); + if (!level.has_value() || !x.has_value() || !y.has_value() || !z.has_value()) { + return std::nullopt; + } + return Id::try_make(level.value(), {x.value(), y.value(), z.value()}); +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/Mappings.h b/src/terrainlib/octree/store_layout/Mappings.h new file mode 100644 index 00000000..1b46bed8 --- /dev/null +++ b/src/terrainlib/octree/store_layout/Mappings.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +#include "octree/store_layout/Flat.h" +#include "octree/store_layout/LevelAndCoordinateDirectories.h" +#include "store/PathMapping.h" + +namespace octree::store_layout { + +inline store::PathMapping flat() { + return {"flat", flat_key_to_node_path, flat_node_path_to_key}; +} + +inline store::PathMapping level_and_coordinate_directories() { + return { + "level_and_coordinate_directories", + level_and_coordinate_key_to_node_path, + level_and_coordinate_node_path_to_key, + }; +} + +inline std::array, 2> all() { + return {flat(), level_and_coordinate_directories()}; +} + +inline std::optional> from_id(const std::string_view id) { + for (const auto mapping : all()) { + if (mapping.id == id) { + return mapping; + } + } + return std::nullopt; +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h new file mode 100644 index 00000000..c9611589 --- /dev/null +++ b/src/terrainlib/store/Codec.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +#include + +#include "store/CodecError.h" +#include "store/NodePath.h" + +namespace store { + +template +class Codec { +public: + virtual ~Codec() = default; + + virtual std::vector paths(const NodePath &node_path) const = 0; + + virtual std::expected read(const NodePath &) const { + return std::unexpected( + CodecError::unsupported_operation(CodecOperation::Read, "read")); + } + + virtual std::expected write(const NodePath &, const NodeData &) const { + return std::unexpected( + CodecError::unsupported_operation(CodecOperation::Write, "write")); + } +}; + +} // namespace store diff --git a/src/terrainlib/store/CodecError.h b/src/terrainlib/store/CodecError.h new file mode 100644 index 00000000..ba4adeb8 --- /dev/null +++ b/src/terrainlib/store/CodecError.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +namespace store { + +enum class CodecOperation { + Read, + Write, + Resolve, +}; + +enum class CodecErrorCategory { + UnsupportedOperation, + UnsupportedCodec, + FileNotFound, + InvalidData, + Io, + Domain, +}; + +struct CodecError { + CodecOperation operation; + CodecErrorCategory category; + std::string message; + + static CodecError unsupported_operation( + const CodecOperation operation, + const std::string_view name) { + return {operation, CodecErrorCategory::UnsupportedOperation, std::string(name)}; + } + + bool operator==(const CodecError &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/Layout.h b/src/terrainlib/store/Layout.h new file mode 100644 index 00000000..b987c535 --- /dev/null +++ b/src/terrainlib/store/Layout.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +#include "store/NodePath.h" +#include "store/PathMapping.h" + +namespace store { + +template +class Layout { +public: + Layout(std::filesystem::path base_path, PathMapping mapping) + : _base_path(std::move(base_path)), _mapping(mapping) {} + + NodePath node_path(const Key &key) const { + return NodePath(_base_path / _mapping.key_to_node_path(key).path()); + } + + std::optional key_from_node_path(const NodePath &node_path) const { + std::error_code error; + const std::filesystem::path relative = + std::filesystem::relative(node_path.path(), _base_path, error); + if (error || relative.empty() || relative.is_absolute()) { + return std::nullopt; + } + return _mapping.node_path_to_key(NodePath(relative)); + } + + const std::filesystem::path &base_path() const { + return _base_path; + } + PathMapping mapping() const { + return _mapping; + } + +private: + std::filesystem::path _base_path; + PathMapping _mapping; +}; + +} // namespace store diff --git a/src/terrainlib/store/NodePath.h b/src/terrainlib/store/NodePath.h new file mode 100644 index 00000000..31673862 --- /dev/null +++ b/src/terrainlib/store/NodePath.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +namespace store { + +class NodePath { +public: + NodePath() = default; + explicit NodePath(std::filesystem::path path) : _path(std::move(path)) {} + + const std::filesystem::path &path() const { + return _path; + } + + bool operator==(const NodePath &) const = default; + +private: + std::filesystem::path _path; +}; + +} // namespace store diff --git a/src/terrainlib/store/PathMapping.h b/src/terrainlib/store/PathMapping.h new file mode 100644 index 00000000..0a2fe725 --- /dev/null +++ b/src/terrainlib/store/PathMapping.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include "store/NodePath.h" + +namespace store { + +template +struct PathMapping { + std::string_view id; + NodePath (*key_to_node_path)(const Key &); + std::optional (*node_path_to_key)(const NodePath &); + + bool operator==(const PathMapping &) const = default; +}; + +} // namespace store diff --git a/src/terrainlib/store/codec/ZppBits.h b/src/terrainlib/store/codec/ZppBits.h new file mode 100644 index 00000000..1a37153f --- /dev/null +++ b/src/terrainlib/store/codec/ZppBits.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include + +#include + +#include "io/serialize.h" +#include "store/Codec.h" + +namespace store::codec { + +inline std::filesystem::path append_extension( + const NodePath &node_path, + const std::string_view extension) { + std::filesystem::path result = node_path.path(); + result += extension; + return result; +} + +template +class ZppBits final : public Codec { +public: + std::vector paths(const NodePath &node_path) const override { + return {append_extension(node_path, ".bin")}; + } + + std::expected read(const NodePath &node_path) const override { + const auto result = io::read_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(CodecError{ + CodecOperation::Read, + result.error() == io::Error(io::Error::OpenFile) + ? CodecErrorCategory::FileNotFound + : CodecErrorCategory::Io, + fmt::format("{}", result.error()), + }); + } + return result.value(); + } + + std::expected write( + const NodePath &node_path, + const NodeData &data) const override { + const auto result = io::write_to_path(data, paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(CodecError{ + CodecOperation::Write, + CodecErrorCategory::Io, + fmt::format("{}", result.error()), + }); + } + return {}; + } +}; + +} // namespace store::codec diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 3ebf239c..72f787cf 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -31,7 +31,9 @@ add_executable(unittests_terrainlib terrainlib/segmented_buffer.cpp terrainlib/stamp_set.cpp terrainlib/store_compatibility.cpp + terrainlib/store_codec.cpp terrainlib/store_index.cpp + terrainlib/store_layout.cpp terrainlib/store_traverse.cpp terrainlib/index_map.cpp terrainlib/index_traverse.cpp diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index e68750bf..1b53b9fa 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -1,4 +1,7 @@ +#include +#include #include +#include #include @@ -9,6 +12,33 @@ #include "octree/storage/open.h" #include "octree/traverse.h" #include "storage.h" +#include "store/codec/ZppBits.h" + +namespace { + +class TemporaryDirectory { +public: + TemporaryDirectory() { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-dag-codec-" + std::to_string(timestamp) + "-" + + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +} // namespace TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { const std::filesystem::path path = @@ -55,4 +85,44 @@ TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { REQUIRE(encoded.has_value()); REQUIRE(fixture_bytes.has_value()); CHECK(encoded.value() == fixture_bytes.value()); + + store::codec::ZppBits runtime_codec; + const auto runtime_batch = runtime_codec.read(store::NodePath(path / "0/0/0/0")); + REQUIRE(runtime_batch.has_value()); + CHECK(runtime_batch->metadata.group_assignment == batch->metadata.group_assignment); + CHECK(runtime_codec.paths(store::NodePath(path / "0/0/0/0")) + == std::vector{path / "0/0/0/0.bin"}); + + TemporaryDirectory output; + const store::NodePath output_path(output.path() / "nested/node"); + REQUIRE(runtime_codec.write(output_path, batch.value()).has_value()); + const auto runtime_bytes = io::read_bytes_from_path(output.path() / "nested/node.bin"); + REQUIRE(runtime_bytes.has_value()); + CHECK(runtime_bytes.value() == fixture_bytes.value()); +} + +TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { + const std::filesystem::path fixture = + std::filesystem::path(ALP_TEST_DATA_DIR) + / "raster-store-refactor/dag/0/0/0/0.bin"; + const auto batch = io::read_from_path(fixture); + REQUIRE(batch.has_value()); + + TemporaryDirectory output; + store::codec::ZppBits codec; + std::vector> operations; + for (int index = 0; index < 8; ++index) { + operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { + const store::NodePath path(output.path() / std::to_string(index) / "node"); + if (!codec.write(path, batch.value()).has_value()) { + return false; + } + const auto loaded = codec.read(path); + return loaded.has_value() + && loaded->metadata.group_assignment == batch->metadata.group_assignment; + })); + } + for (auto &operation : operations) { + REQUIRE(operation.get()); + } } diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp new file mode 100644 index 00000000..5b4c2fca --- /dev/null +++ b/unittests/terrainlib/store_codec.cpp @@ -0,0 +1,202 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "io/bytes.h" +#include "mesh/codec/Gltf.h" +#include "mesh/codec/Terrain.h" +#include "store/Codec.h" +#include "store/codec/ZppBits.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-store-codec-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +class MultiFileCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path data = node_path.path(); + std::filesystem::path metadata = node_path.path(); + data += ".data"; + metadata += ".metadata"; + return {data, metadata}; + } +}; + +class WriteOnlyCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path path = node_path.path(); + path += ".out"; + return {path}; + } + std::expected write( + const store::NodePath &, + const int &) const override { + ++writes; + return {}; + } + + mutable std::atomic_uint32_t writes = 0; +}; + +mesh::Simple triangle_mesh(const double offset) { + return mesh::Simple( + {{0, 1, 2}}, + {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); +} + +template +void check_mesh_codec_reentrancy(const Codec &codec, const std::filesystem::path &base) { + std::vector> writes; + for (int index = 0; index < 4; ++index) { + writes.push_back(std::async(std::launch::async, [&codec, base, index] { + return codec.write( + store::NodePath(base / std::to_string(index) / "node"), + triangle_mesh(index)) + .has_value(); + })); + } + for (auto &write : writes) { + REQUIRE(write.get()); + } + + std::vector> reads; + for (int index = 0; index < 4; ++index) { + reads.push_back(std::async(std::launch::async, [&codec, base, index] { + const auto result = codec.read( + store::NodePath(base / std::to_string(index) / "node")); + return result.has_value() && result->face_count() == 1; + })); + } + for (auto &read : reads) { + REQUIRE(read.get()); + } +} + +} // namespace + +TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec]") { + MultiFileCodec codec; + const store::NodePath path("12/34/56/78"); + CHECK(codec.paths(path) + == std::vector{ + "12/34/56/78.data", + "12/34/56/78.metadata", + }); + + const auto read = codec.read(path); + REQUIRE_FALSE(read.has_value()); + CHECK(read.error().operation == store::CodecOperation::Read); + CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); + + const auto write = codec.write(path, 42); + REQUIRE_FALSE(write.has_value()); + CHECK(write.error().operation == store::CodecOperation::Write); + CHECK(write.error().category == store::CodecErrorCategory::UnsupportedOperation); +} + +TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") { + WriteOnlyCodec codec; + REQUIRE(codec.write(store::NodePath("node"), 42).has_value()); + CHECK(codec.writes == 1); + const auto read = codec.read(store::NodePath("node")); + REQUIRE_FALSE(read.has_value()); + CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); +} + +TEST_CASE("ZPP Bits runtime codec creates directories and converts errors", "[store][codec]") { + TemporaryDirectory directory("zpp"); + const store::NodePath path(directory.path() / "nested/deeper/node"); + store::codec::ZppBits codec; + + REQUIRE(codec.write(path, 12345).has_value()); + CHECK(codec.paths(path) == std::vector{directory.path() / "nested/deeper/node.bin"}); + REQUIRE(std::filesystem::is_regular_file(codec.paths(path).front())); + const auto value = codec.read(path); + REQUIRE(value.has_value()); + CHECK(value.value() == 12345); + + REQUIRE(io::write_bytes_to_path( + std::vector{0xff}, + directory.path() / "invalid.bin") + .has_value()); + const auto invalid = codec.read(store::NodePath(directory.path() / "invalid")); + REQUIRE_FALSE(invalid.has_value()); + CHECK(invalid.error().operation == store::CodecOperation::Read); + CHECK(invalid.error().category == store::CodecErrorCategory::Io); +} + +TEST_CASE("ZPP Bits runtime codec is reentrant", "[store][codec]") { + TemporaryDirectory directory("zpp-reentrant"); + store::codec::ZppBits codec; + std::vector> operations; + for (int index = 0; index < 16; ++index) { + operations.push_back(std::async(std::launch::async, [&codec, &directory, index] { + const store::NodePath path(directory.path() / std::to_string(index) / "node"); + if (!codec.write(path, index).has_value()) { + return false; + } + const auto value = codec.read(path); + return value.has_value() && value.value() == index; + })); + } + for (auto &operation : operations) { + REQUIRE(operation.get()); + } +} + +TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") { + TemporaryDirectory directory("mesh-reentrant"); + + SECTION("terrain") { + check_mesh_codec_reentrancy(mesh::codec::Terrain{}, directory.path() / "terrain"); + } + SECTION("binary glTF") { + check_mesh_codec_reentrancy( + mesh::codec::Gltf(mesh::codec::GltfContainer::Binary), + directory.path() / "glb"); + } + SECTION("JSON glTF") { + check_mesh_codec_reentrancy( + mesh::codec::Gltf(mesh::codec::GltfContainer::Json), + directory.path() / "gltf"); + } +} + +TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") { + TemporaryDirectory directory("gltf-error"); + mesh::codec::Gltf codec(mesh::codec::GltfContainer::Binary); + const store::NodePath node_path(directory.path() / "blocked"); + REQUIRE(std::filesystem::create_directory(codec.paths(node_path).front())); + + const auto result = codec.write(node_path, triangle_mesh(0.0)); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().operation == store::CodecOperation::Write); + CHECK(result.error().category == store::CodecErrorCategory::Domain); +} diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp new file mode 100644 index 00000000..ec4eaa4b --- /dev/null +++ b/unittests/terrainlib/store_layout.cpp @@ -0,0 +1,70 @@ +#include + +#include + +#include "mesh/codec/Gltf.h" +#include "mesh/codec/Terrain.h" +#include "octree/store_layout/Mappings.h" +#include "store/Layout.h" + +TEST_CASE("extensionless octree mappings preserve physical fixture paths", "[store][layout]") { + const std::filesystem::path fixtures = + std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor"; + mesh::codec::Terrain terrain; + + SECTION("flat") { + const store::Layout layout(fixtures / "sf-flat", octree::store_layout::flat()); + const store::NodePath node_path = layout.node_path(octree::Id::root()); + CHECK(node_path.path() == fixtures / "sf-flat/0-0"); + CHECK(terrain.paths(node_path) == std::vector{fixtures / "sf-flat/0-0.terrain"}); + CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); + } + + SECTION("level and coordinate directories") { + const store::Layout layout( + fixtures / "sf-coordinates", + octree::store_layout::level_and_coordinate_directories()); + const octree::Id child = octree::Id::root().child(2).value(); + const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); + CHECK(terrain.paths(layout.node_path(child)) + == std::vector{fixtures / "sf-coordinates/1/0/1/0.terrain"}); + CHECK(terrain.paths(layout.node_path(deep)) + == std::vector{fixtures / "sf-coordinates/2/3/1/3.terrain"}); + CHECK(layout.key_from_node_path(layout.node_path(child)) == child); + CHECK(layout.key_from_node_path(layout.node_path(deep)) == deep); + } +} + +TEST_CASE("octree mapping lookup is explicit", "[store][layout]") { + REQUIRE(octree::store_layout::from_id("flat").has_value()); + CHECK(octree::store_layout::from_id("flat")->id == "flat"); + REQUIRE(octree::store_layout::from_id("level_and_coordinate_directories").has_value()); + CHECK(octree::store_layout::from_id("level_and_coordinate_directories")->id + == "level_and_coordinate_directories"); + CHECK_FALSE(octree::store_layout::from_id("unknown").has_value()); + CHECK(octree::store_layout::all().size() == 2); +} + +TEST_CASE("extensionless octree mappings validate complete paths", "[store][layout]") { + const auto flat = octree::store_layout::flat(); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2.terrain")).has_value()); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("nested/1-2")).has_value()); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2-extra")).has_value()); + + const auto coordinates = octree::store_layout::level_and_coordinate_directories(); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0.terrain")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0/extra")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("22/0/0/0")).has_value()); +} + +TEST_CASE("runtime mesh codecs own their filename endings", "[store][layout][codec]") { + const store::NodePath path("12/34/56/78"); + mesh::codec::Terrain terrain; + mesh::codec::Gltf binary(mesh::codec::GltfContainer::Binary); + mesh::codec::Gltf json(mesh::codec::GltfContainer::Json); + + CHECK(terrain.paths(path) == std::vector{"12/34/56/78.terrain"}); + CHECK(binary.paths(path) == std::vector{"12/34/56/78.glb"}); + CHECK(json.paths(path) == std::vector{"12/34/56/78.gltf"}); +} From 61acb743b03843a4200f23d1221043c2cb7840cc Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:43:03 +0200 Subject: [PATCH 06/43] Add shared runtime-codec storage core --- docs/raster-store/refactor-status.md | 19 +- src/terrainlib/store/IndexFormat.h | 49 +++++ src/terrainlib/store/IndexedStorage.h | 36 +++ src/terrainlib/store/RawStorage.h | 181 +++++++++++++++ src/terrainlib/store/Storage.h | 292 +++++++++++++++++++++++++ src/terrainlib/store/StorageError.h | 52 +++++ src/terrainlib/store/StorageSettings.h | 9 + unittests/CMakeLists.txt | 1 + unittests/terrainlib/store_storage.cpp | 251 +++++++++++++++++++++ 9 files changed, 887 insertions(+), 3 deletions(-) create mode 100644 src/terrainlib/store/IndexFormat.h create mode 100644 src/terrainlib/store/IndexedStorage.h create mode 100644 src/terrainlib/store/RawStorage.h create mode 100644 src/terrainlib/store/Storage.h create mode 100644 src/terrainlib/store/StorageError.h create mode 100644 src/terrainlib/store/StorageSettings.h create mode 100644 unittests/terrainlib/store_storage.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 6a2e1e83..b5158ed2 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 2 — Introduce path mappings and runtime codecs +- Phase: 3 — Generalize storage and index lifecycle - State: in progress -- Next action: run the complete Phase 2 verification, inspect the diff, - commit, and tag the phase boundary. +- Next action: build and verify the shared storage core, then make the first + Phase 3 commit before migrating production adapters. ## Phase log @@ -68,6 +68,19 @@ This file is the resumable implementation log for and concurrent codec tests. - State: complete. +### Phase 3 — Generalize storage and index lifecycle + +- In progress. +- Added typed storage, filesystem, overwrite, and index-format errors. +- Added shared `RawStorage`, `Storage`, and `IndexedStorage` with exclusive + runtime-codec ownership, trait-keyed operations, multi-file `has`/`remove`, + overwrite handling, index persistence, and move finalization. +- Added shared storage instantiations for both 2D and 3D traits plus move, + overwrite, invalid-key, and multi-file tests. +- Shared storage core focused tests: 48 assertions in 6 test cases passed. +- Shared storage core build and `git diff --check`: passed. +- Next: migrate 3D format/opening adapters and application callers. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h new file mode 100644 index 00000000..7e0c3b8c --- /dev/null +++ b/src/terrainlib/store/IndexFormat.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "store/Index.h" +#include "store/PathMapping.h" + +namespace store { + +enum class IndexFormatErrorCategory { + Open, + Write, + Malformed, +}; + +struct IndexFormatError { + IndexFormatErrorCategory category; + std::filesystem::path path; + std::string message; + + bool operator==(const IndexFormatError &) const = default; +}; + +template +struct IndexMetadata { + Index index; + std::string layout_id; + std::string codec_selector; +}; + +template +struct IndexFormat { + std::string_view index_filename; + + std::expected, IndexFormatError> (*read)( + const std::filesystem::path &index_path); + std::expected (*write)( + const std::filesystem::path &index_path, + const IndexMetadata &metadata); + std::optional> (*mapping_from_id)(std::string_view id); + PathMapping (*default_mapping)(); +}; + +} // namespace store diff --git a/src/terrainlib/store/IndexedStorage.h b/src/terrainlib/store/IndexedStorage.h new file mode 100644 index 00000000..9a963a10 --- /dev/null +++ b/src/terrainlib/store/IndexedStorage.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "store/Storage.h" + +namespace store { + +template +class IndexedStorage : public Storage { +public: + using Base = Storage; + using typename Base::Persistence; + + explicit IndexedStorage(Storage storage) + : Base(std::move(storage)) { + this->ensure_indexed(); + } + + IndexedStorage( + RawStorage raw, + Index index, + Persistence persistence) + : Base(std::move(raw), std::move(index), std::move(persistence)) {} + + IndexedStorage(const IndexedStorage &) = delete; + IndexedStorage &operator=(const IndexedStorage &) = delete; + IndexedStorage(IndexedStorage &&) noexcept = default; + IndexedStorage &operator=(IndexedStorage &&) noexcept = default; + + const Index &index() const { + return this->index_ref(); + } +}; + +} // namespace store diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h new file mode 100644 index 00000000..487a3e97 --- /dev/null +++ b/src/terrainlib/store/RawStorage.h @@ -0,0 +1,181 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "store/Codec.h" +#include "store/Layout.h" +#include "store/StorageError.h" +#include "store/Traits.h" + +namespace store { + +template +class RawStorage { +public: + using Key = typename Traits::Key; + using value_type = NodeData; + + RawStorage(Layout layout, std::unique_ptr> codec) + : _layout(std::move(layout)), _codec(std::move(codec)) {} + + RawStorage(const RawStorage &) = delete; + RawStorage &operator=(const RawStorage &) = delete; + RawStorage(RawStorage &&) noexcept = default; + RawStorage &operator=(RawStorage &&) noexcept = default; + + std::expected> load(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(LoadError(InvalidKey{key})); + } + const auto result = _codec->read(_layout.node_path(key)); + if (!result.has_value()) { + return std::unexpected(LoadError(result.error())); + } + return result.value(); + } + + std::expected> save(const Key &key, const NodeData &data) const { + if (!Traits::is_valid(key)) { + return std::unexpected(SaveError(InvalidKey{key})); + } + const auto result = _codec->write(_layout.node_path(key), data); + if (!result.has_value()) { + return std::unexpected(SaveError(result.error())); + } + return {}; + } + + std::expected, InvalidKey> paths( + const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(InvalidKey{key}); + } + return _codec->paths(_layout.node_path(key)); + } + + std::expected> has(const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(FileOperationError(node_paths.error())); + } + if (node_paths->empty()) { + return false; + } + for (const auto &path : node_paths.value()) { + std::error_code error; + const bool exists = std::filesystem::exists(path, error); + if (error) { + return std::unexpected(FileOperationError(FilesystemError{ + path, + "exists", + error, + })); + } + if (!exists) { + return false; + } + } + return true; + } + + std::expected> remove(const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(FileOperationError(node_paths.error())); + } + bool removed = false; + for (const auto &path : node_paths.value()) { + std::error_code error; + removed = std::filesystem::remove(path, error) || removed; + if (error) { + return std::unexpected(FileOperationError(FilesystemError{ + path, + "remove", + error, + })); + } + } + return removed; + } + + std::expected> copy_from( + const Key &key, + const RawStorage &source) const { + if (!Traits::is_valid(key)) { + return std::unexpected(CopyError(InvalidKey{key})); + } + const auto source_exists = source.has(key); + if (!source_exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> CopyError { return error; }, + source_exists.error())); + } + if (!source_exists.value()) { + return std::unexpected(CopyError(MissingSource{key})); + } + + const NodePath probe("__codec_probe__/node"); + if (_codec->paths(probe) != source._codec->paths(probe)) { + const auto loaded = source._codec->read(source._layout.node_path(key)); + if (!loaded.has_value()) { + return std::unexpected(CopyError(loaded.error())); + } + const auto written = _codec->write(_layout.node_path(key), loaded.value()); + if (!written.has_value()) { + return std::unexpected(CopyError(written.error())); + } + return {}; + } + + const auto source_paths = source._codec->paths(source._layout.node_path(key)); + const auto target_paths = _codec->paths(_layout.node_path(key)); + for (size_t index = 0; index < source_paths.size(); ++index) { + std::error_code error; + std::filesystem::remove(target_paths[index], error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index], + "remove", + error, + })); + } + std::filesystem::create_directories(target_paths[index].parent_path(), error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index].parent_path(), + "create_directories", + error, + })); + } + std::filesystem::create_hard_link(source_paths[index], target_paths[index], error); + if (error) { + return std::unexpected(CopyError(FilesystemError{ + target_paths[index], + "create_hard_link", + error, + })); + } + } + return {}; + } + + const Layout &layout() const { + return _layout; + } + const Codec &codec() const { + return *_codec; + } + +private: + Layout _layout; + std::unique_ptr> _codec; +}; + +} // namespace store diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h new file mode 100644 index 00000000..fa99b6a4 --- /dev/null +++ b/src/terrainlib/store/Storage.h @@ -0,0 +1,292 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "log.h" +#include "store/IndexFormat.h" +#include "store/RawStorage.h" +#include "store/StorageSettings.h" + +namespace store { + +template +class Storage { +public: + using Key = typename Traits::Key; + using value_type = NodeData; + + struct Persistence { + IndexFormat format; + std::filesystem::path index_path; + std::string layout_id; + std::string codec_selector; + }; + + explicit Storage(RawStorage raw) : _raw(std::move(raw)) {} + + Storage( + RawStorage raw, + Index index, + Persistence persistence) + : _raw(std::move(raw)), + _index(std::move(index)), + _persistence(std::move(persistence)) {} + + virtual ~Storage() { + finalize_displaced_state(); + } + + Storage(const Storage &) = delete; + Storage &operator=(const Storage &) = delete; + + Storage(Storage &&other) noexcept + : _raw(std::move(other._raw)), + _index(std::move(other._index)), + _persistence(std::move(other._persistence)), + _settings(other._settings), + _dirty(std::exchange(other._dirty, false)) { + other._index.reset(); + other._persistence.reset(); + } + + Storage &operator=(Storage &&other) noexcept { + if (this == &other) { + return *this; + } + finalize_displaced_state(); + _raw = std::move(other._raw); + _index = std::move(other._index); + _persistence = std::move(other._persistence); + _settings = other._settings; + _dirty = std::exchange(other._dirty, false); + other._index.reset(); + other._persistence.reset(); + return *this; + } + + std::expected> load(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(LoadError(InvalidKey{key})); + } + if (_index.has_value()) { + const auto status = _index->get(key); + if (!status.has_value()) { + return std::unexpected(LoadError(status.error())); + } + if (!status->has_value() + || status->value() == NodeStatus::Virtual) { + return std::unexpected(LoadError(CodecError{ + CodecOperation::Read, + CodecErrorCategory::FileNotFound, + "node is not physically present in the index", + })); + } + } + return _raw.load(key); + } + + std::expected> save(const Key &key, const NodeData &data) { + if (!Traits::is_valid(key)) { + return std::unexpected(SaveError(InvalidKey{key})); + } + const auto exists = has(key); + if (!exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> SaveError { return error; }, + exists.error())); + } + if (exists.value() && !_settings.allow_overwrite) { + const auto node_paths = _raw.paths(key).value(); + return std::unexpected(SaveError(AlreadyExists{ + node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), + })); + } + + const auto result = _raw.save(key, data); + if (!result.has_value()) { + return result; + } + if (_index.has_value()) { + const auto added = _index->add(key); + if (!added.has_value()) { + return std::unexpected(SaveError(added.error())); + } + _dirty = _dirty || added.value(); + } + return {}; + } + + std::expected> copy_from( + const Key &key, + const Storage &source) { + if (!Traits::is_valid(key)) { + return std::unexpected(CopyError(InvalidKey{key})); + } + const auto exists = has(key); + if (!exists.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> CopyError { return error; }, + exists.error())); + } + if (exists.value() && !_settings.allow_overwrite) { + const auto node_paths = _raw.paths(key).value(); + return std::unexpected(CopyError(AlreadyExists{ + node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), + })); + } + if (exists.value() && _index.has_value()) { + const auto removed = _index->remove(key); + if (!removed.has_value()) { + return std::unexpected(CopyError(removed.error())); + } + _dirty = _dirty || removed.value(); + } + + const auto result = _raw.copy_from(key, source._raw); + if (!result.has_value()) { + return result; + } + if (_index.has_value()) { + const auto added = _index->add(key); + if (!added.has_value()) { + return std::unexpected(CopyError(added.error())); + } + _dirty = _dirty || added.value(); + } + return {}; + } + + std::expected> remove(const Key &key) { + const auto removed = _raw.remove(key); + if (!removed.has_value()) { + return removed; + } + if (_index.has_value()) { + const auto index_removed = _index->remove(key); + if (!index_removed.has_value()) { + return std::unexpected(FileOperationError(index_removed.error())); + } + _dirty = _dirty || index_removed.value(); + } + return removed.value(); + } + + std::expected> has(const Key &key) const { + if (!Traits::is_valid(key)) { + return std::unexpected(FileOperationError(InvalidKey{key})); + } + if (!_index.has_value()) { + return _raw.has(key); + } + const auto status = _index->get(key); + if (!status.has_value()) { + return std::unexpected(FileOperationError(status.error())); + } + return status->has_value() && status->value() != NodeStatus::Virtual; + } + + std::expected, InvalidKey> paths( + const Key &key) const { + return _raw.paths(key); + } + + const std::filesystem::path &base_path() const { + return _raw.layout().base_path(); + } + const Layout &layout() const { + return _raw.layout(); + } + const Codec &codec() const { + return _raw.codec(); + } + + bool is_indexed() const { + return _index.has_value(); + } + void ensure_indexed() { + if (!_index.has_value()) { + _index.emplace(); + _dirty = true; + } + } + + std::optional>> index() const { + if (!_index.has_value()) { + return std::nullopt; + } + return std::cref(_index.value()); + } + + std::expected save_index() const { + if (!_dirty) { + return {}; + } + if (!_index.has_value() || !_persistence.has_value()) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Write, + {}, + "indexed storage has no persistence configuration", + }); + } + const IndexMetadata metadata{ + _index.value(), + _persistence->layout_id, + _persistence->codec_selector, + }; + const auto result = _persistence->format.write( + _persistence->index_path, + metadata); + if (result.has_value()) { + _dirty = false; + } + return result; + } + + const StorageSettings &settings() const { + return _settings; + } + StorageSettings &settings() { + return _settings; + } + +protected: + Index &index_mut() { + return _index.value(); + } + const Index &index_ref() const { + return _index.value(); + } + bool is_index_dirty() const { + return _dirty; + } + void set_index_dirty(const bool dirty = true) const { + _dirty = dirty; + } + +private: + void finalize_displaced_state() noexcept { + if (!_dirty || !_index.has_value()) { + return; + } + const auto result = save_index(); + if (!result.has_value()) { + LOG_ERROR( + "Failed to automatically save index {}: {}", + result.error().path, + result.error().message); + } + } + + RawStorage _raw; + std::optional> _index; + std::optional _persistence; + StorageSettings _settings; + mutable bool _dirty = false; +}; + +} // namespace store diff --git a/src/terrainlib/store/StorageError.h b/src/terrainlib/store/StorageError.h new file mode 100644 index 00000000..ffd6d69b --- /dev/null +++ b/src/terrainlib/store/StorageError.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include + +#include "store/CodecError.h" +#include "store/InvalidKey.h" + +namespace store { + +struct AlreadyExists { + std::filesystem::path path; + bool operator==(const AlreadyExists &) const = default; +}; + +struct FilesystemError { + std::filesystem::path path; + std::string operation; + std::error_code error; + bool operator==(const FilesystemError &) const = default; +}; + +template +struct MissingSource { + Key key; + bool operator==(const MissingSource &) const = default; +}; + +template +using LoadError = std::variant, CodecError>; + +template +using SaveError = std::variant< + InvalidKey, + CodecError, + AlreadyExists, + FilesystemError>; + +template +using FileOperationError = std::variant, FilesystemError>; + +template +using CopyError = std::variant< + InvalidKey, + MissingSource, + AlreadyExists, + FilesystemError, + CodecError>; + +} // namespace store diff --git a/src/terrainlib/store/StorageSettings.h b/src/terrainlib/store/StorageSettings.h new file mode 100644 index 00000000..2a883ee1 --- /dev/null +++ b/src/terrainlib/store/StorageSettings.h @@ -0,0 +1,9 @@ +#pragma once + +namespace store { + +struct StorageSettings { + bool allow_overwrite = false; +}; + +} // namespace store diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 72f787cf..26db9b5d 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -34,6 +34,7 @@ add_executable(unittests_terrainlib terrainlib/store_codec.cpp terrainlib/store_index.cpp terrainlib/store_layout.cpp + terrainlib/store_storage.cpp terrainlib/store_traverse.cpp terrainlib/index_map.cpp terrainlib/index_traverse.cpp diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp new file mode 100644 index 00000000..04125969 --- /dev/null +++ b/unittests/terrainlib/store_storage.cpp @@ -0,0 +1,251 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "io/serialize.h" +#include "octree/StoreTraits.h" +#include "octree/store_layout/Mappings.h" +#include "raster_store/StoreTraits.h" +#include "store/IndexedStorage.h" +#include "store/codec/ZppBits.h" +#include "string_utils.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-store-storage-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +store::NodePath tile_to_path(const radix::tile::Id &key) { + return store::NodePath( + std::to_string(key.zoom_level) + "/" + std::to_string(key.coords.x) + + "/" + std::to_string(key.coords.y)); +} + +std::optional path_to_tile(const store::NodePath &node_path) { + const std::filesystem::path &path = node_path.path(); + if (path.is_absolute() || std::distance(path.begin(), path.end()) != 3) { + return std::nullopt; + } + auto part = path.begin(); + const auto zoom = from_chars(part->string()); + ++part; + const auto x = from_chars(part->string()); + ++part; + const auto y = from_chars(part->string()); + if (!zoom.has_value() || !x.has_value() || !y.has_value()) { + return std::nullopt; + } + const radix::tile::Id key{zoom.value(), {x.value(), y.value()}}; + return raster_store::StoreTraits::is_valid(key) + ? std::optional(key) + : std::nullopt; +} + +template +store::PathMapping test_mapping(); + +template<> +store::PathMapping test_mapping() { + return octree::store_layout::flat(); +} + +template<> +store::PathMapping test_mapping() { + return {"test_tiles", tile_to_path, path_to_tile}; +} + +template +store::Storage make_storage(const std::filesystem::path &path) { + return store::Storage(store::RawStorage( + store::Layout(path, test_mapping()), + std::make_unique>())); +} + +class MultiFileCodec final : public store::Codec { +public: + std::vector paths(const store::NodePath &node_path) const override { + std::filesystem::path first = node_path.path(); + std::filesystem::path second = node_path.path(); + first += ".data"; + second += ".metadata"; + return {first, second}; + } +}; + +std::atomic_uint32_t index_writes = 0; + +std::expected, store::IndexFormatError> +unused_index_read(const std::filesystem::path &path) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Open, + path, + "not used", + }); +} + +std::expected count_index_write( + const std::filesystem::path &, + const store::IndexMetadata &) { + ++index_writes; + return {}; +} + +std::optional> fake_mapping_from_id( + const std::string_view id) { + return id == "flat" + ? std::optional>(octree::store_layout::flat()) + : std::nullopt; +} + +store::PathMapping fake_default_mapping() { + return octree::store_layout::flat(); +} + +store::IndexFormat counting_index_format() { + return { + "terrain.index", + unused_index_read, + count_index_write, + fake_mapping_from_id, + fake_default_mapping, + }; +} + +store::IndexedStorage make_indexed_storage( + const std::filesystem::path &path) { + using Storage = store::Storage; + return store::IndexedStorage( + store::RawStorage( + store::Layout(path, octree::store_layout::flat()), + std::make_unique>()), + store::Index{}, + Storage::Persistence{ + counting_index_format(), + path / "terrain.index", + "flat", + ".bin", + }); +} + +} // namespace + +TEMPLATE_TEST_CASE( + "shared raw storage has no hidden octree dependency", + "[store][storage]", + octree::StoreTraits, + raster_store::StoreTraits) { + using Traits = TestType; + TemporaryDirectory directory("traits"); + auto storage = make_storage(directory.path()); + const auto root = Traits::root(); + + CHECK_FALSE(storage.has(root).value()); + REQUIRE(storage.save(root, 42).has_value()); + CHECK(storage.has(root).value()); + REQUIRE(storage.load(root).has_value()); + CHECK(storage.load(root).value() == 42); + REQUIRE(storage.remove(root).has_value()); + CHECK_FALSE(storage.has(root).value()); +} + +TEST_CASE("shared storage rejects invalid 2D keys", "[store][storage]") { + TemporaryDirectory directory("invalid-key"); + auto storage = make_storage(directory.path()); + const radix::tile::Id invalid{3, {8, 0}}; + + CHECK(std::holds_alternative>( + storage.load(invalid).error())); + CHECK(std::holds_alternative>( + storage.save(invalid, 1).error())); + CHECK(std::holds_alternative>( + storage.has(invalid).error())); + CHECK(std::holds_alternative>( + storage.remove(invalid).error())); +} + +TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") { + TemporaryDirectory directory("overwrite"); + auto storage = make_storage(directory.path()); + const octree::Id root = octree::Id::root(); + + REQUIRE(storage.save(root, 1).has_value()); + const auto rejected = storage.save(root, 2); + REQUIRE_FALSE(rejected.has_value()); + CHECK(std::holds_alternative(rejected.error())); + CHECK(storage.load(root).value() == 1); + + storage.settings().allow_overwrite = true; + REQUIRE(storage.save(root, 2).has_value()); + CHECK(storage.load(root).value() == 2); +} + +TEST_CASE("raw storage requires and removes every codec path", "[store][storage]") { + TemporaryDirectory directory("multi-file"); + store::RawStorage storage( + store::Layout(directory.path(), octree::store_layout::flat()), + std::make_unique()); + const octree::Id root = octree::Id::root(); + const auto paths = storage.paths(root).value(); + REQUIRE(paths.size() == 2); + + REQUIRE(io::write_to_path(1, paths[0]).has_value()); + CHECK_FALSE(storage.has(root).value()); + REQUIRE(io::write_to_path(2, paths[1]).has_value()); + CHECK(storage.has(root).value()); + REQUIRE(storage.remove(root).value()); + CHECK_FALSE(std::filesystem::exists(paths[0])); + CHECK_FALSE(std::filesystem::exists(paths[1])); +} + +TEST_CASE("storage moves transfer and finalize dirty index state", "[store][storage]") { + index_writes = 0; + TemporaryDirectory first_directory("move-first"); + TemporaryDirectory second_directory("move-second"); + + { + auto original = make_indexed_storage(first_directory.path()); + REQUIRE(original.save(octree::Id::root(), 1).has_value()); + auto moved = std::move(original); + CHECK(index_writes == 0); + REQUIRE(moved.save_index().has_value()); + CHECK(index_writes == 1); + } + CHECK(index_writes == 1); + + { + auto destination = make_indexed_storage(first_directory.path()); + destination.settings().allow_overwrite = true; + REQUIRE(destination.save(octree::Id::root(), 2).has_value()); + auto source = make_indexed_storage(second_directory.path()); + REQUIRE(source.save(octree::Id::root(), 3).has_value()); + destination = std::move(source); + CHECK(index_writes == 2); + } + CHECK(index_writes == 3); +} From 4ffcdeadbe34f16ed40f6ac7452ea331b6c9587e Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:10:25 +0200 Subject: [PATCH 07/43] Migrate applications to shared storage --- docs/raster-store/refactor-status.md | 31 +- src/dag_builder/build.cpp | 41 ++- src/dag_builder/codec/MetadataView.h | 38 +++ src/dag_builder/codec/from_extension.h | 40 +++ src/dag_builder/main.cpp | 28 +- src/dag_builder/metadata.h | 11 - src/dag_builder/storage.h | 65 +++- src/dag_builder/thread_safe_storage.h | 9 +- src/dag_convert_debug/main.cpp | 45 ++- src/sf_builder/terrainbuilder.cpp | 41 ++- src/sf_index_browser/main.cpp | 39 ++- src/sf_merger/NodeLoader.h | 11 +- src/sf_merger/NodeWriter.h | 6 +- src/sf_merger/cut.h | 33 +- src/sf_merger/main.cpp | 40 ++- src/sf_merger/merge.h | 6 +- src/terrainlib/CMakeLists.txt | 1 - src/terrainlib/mesh/codec/from_extension.h | 33 ++ src/terrainlib/mesh/storage.h | 49 +++ src/terrainlib/octree/IndexMap.h | 6 + src/terrainlib/octree/Storage.h | 11 +- src/terrainlib/octree/disk/Layout.h | 49 --- src/terrainlib/octree/disk/layout/Strategy.h | 19 -- .../octree/disk/layout/StrategyRegister.h | 84 ----- .../octree/disk/layout/strategy/Default.h | 15 - .../octree/disk/layout/strategy/Flat.h | 50 --- .../strategy/LevelAndCoordinateDirectories.h | 60 ---- src/terrainlib/octree/storage/CopyError.h | 59 ---- src/terrainlib/octree/storage/IndexFile.h | 68 ++++ .../octree/storage/IndexedStorage.h | 67 ---- src/terrainlib/octree/storage/MeshStorage.h | 11 +- src/terrainlib/octree/storage/RawStorage.h | 118 ------- src/terrainlib/octree/storage/Storage.h | 312 ------------------ src/terrainlib/octree/storage/cache/Dummy.h | 26 +- src/terrainlib/octree/storage/cache/ICache.h | 19 +- .../octree/storage/cache/LruCache.h | 70 +--- src/terrainlib/octree/storage/codec/Codec.h | 32 -- .../octree/storage/codec/MeshCodec.h | 35 -- .../octree/storage/codec/ReadOnlyCodec.h | 20 -- .../octree/storage/codec/ZppBitsCodec.h | 38 --- src/terrainlib/octree/storage/defaults.h | 25 -- src/terrainlib/octree/storage/helpers.cpp | 136 -------- src/terrainlib/octree/storage/helpers.h | 28 -- src/terrainlib/octree/storage/open.h | 44 ++- src/terrainlib/octree/storage/open.inl | 111 ------- src/terrainlib/octree/storage/open_runtime.h | 196 +++++++++++ src/terrainlib/octree/traverse.h | 17 + src/terrainlib/store/OpenError.h | 24 ++ src/terrainlib/store/Storage.h | 116 +++++++ src/terrainlib/store/cache/Dummy.h | 25 ++ src/terrainlib/store/cache/Interface.h | 21 ++ src/terrainlib/store/cache/Lru.h | 73 ++++ src/terrainlib/store/describe_error.h | 50 +++ src/terrainlib/store/open.h | 93 ++++++ .../dag_builder/storage_compatibility.cpp | 52 ++- unittests/terrainlib/store_compatibility.cpp | 123 +++++-- 56 files changed, 1347 insertions(+), 1523 deletions(-) create mode 100644 src/dag_builder/codec/MetadataView.h create mode 100644 src/dag_builder/codec/from_extension.h create mode 100644 src/terrainlib/mesh/codec/from_extension.h create mode 100644 src/terrainlib/mesh/storage.h delete mode 100644 src/terrainlib/octree/disk/Layout.h delete mode 100644 src/terrainlib/octree/disk/layout/Strategy.h delete mode 100644 src/terrainlib/octree/disk/layout/StrategyRegister.h delete mode 100644 src/terrainlib/octree/disk/layout/strategy/Default.h delete mode 100644 src/terrainlib/octree/disk/layout/strategy/Flat.h delete mode 100644 src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h delete mode 100644 src/terrainlib/octree/storage/CopyError.h create mode 100644 src/terrainlib/octree/storage/IndexFile.h delete mode 100644 src/terrainlib/octree/storage/IndexedStorage.h delete mode 100644 src/terrainlib/octree/storage/RawStorage.h delete mode 100644 src/terrainlib/octree/storage/Storage.h delete mode 100644 src/terrainlib/octree/storage/codec/Codec.h delete mode 100644 src/terrainlib/octree/storage/codec/MeshCodec.h delete mode 100644 src/terrainlib/octree/storage/codec/ReadOnlyCodec.h delete mode 100644 src/terrainlib/octree/storage/codec/ZppBitsCodec.h delete mode 100644 src/terrainlib/octree/storage/defaults.h delete mode 100644 src/terrainlib/octree/storage/helpers.cpp delete mode 100644 src/terrainlib/octree/storage/helpers.h delete mode 100644 src/terrainlib/octree/storage/open.inl create mode 100644 src/terrainlib/octree/storage/open_runtime.h create mode 100644 src/terrainlib/store/OpenError.h create mode 100644 src/terrainlib/store/cache/Dummy.h create mode 100644 src/terrainlib/store/cache/Interface.h create mode 100644 src/terrainlib/store/cache/Lru.h create mode 100644 src/terrainlib/store/describe_error.h create mode 100644 src/terrainlib/store/open.h diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index b5158ed2..1bfd3b4a 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -17,9 +17,9 @@ This file is the resumable implementation log for ## Current position - Phase: 3 — Generalize storage and index lifecycle -- State: in progress -- Next action: build and verify the shared storage core, then make the first - Phase 3 commit before migrating production adapters. +- State: complete; completion commit pending. +- Next action: commit and tag Phase 3, then begin Phase 4 copy hardening and + SF topology validation. ## Phase log @@ -70,7 +70,6 @@ This file is the resumable implementation log for ### Phase 3 — Generalize storage and index lifecycle -- In progress. - Added typed storage, filesystem, overwrite, and index-format errors. - Added shared `RawStorage`, `Storage`, and `IndexedStorage` with exclusive runtime-codec ownership, trait-keyed operations, multi-file `has`/`remove`, @@ -79,7 +78,18 @@ This file is the resumable implementation log for overwrite, invalid-key, and multi-file tests. - Shared storage core focused tests: 48 assertions in 6 test cases passed. - Shared storage core build and `git diff --check`: passed. -- Next: migrate 3D format/opening adapters and application callers. +- Added the legacy 3D `terrain.index` format adapter, explicit mesh and DAG + codec resolvers, folder discovery, and expected-returning opening APIs. +- Preserved independently readable, read-only DAG metadata views and migrated + DAG output synchronization to the shared storage types. +- Migrated SF builder, SF merger, SF index browser, DAG builder, and DAG + convert debug without exposing codec ownership at application call sites. +- Ported cache API aliases for compile compatibility. +- Deleted the legacy layout class hierarchy and registry, static codec stack, + and duplicate octree storage implementation. +- Added resolver, unsupported metadata write, unknown layout/codec, malformed + index no-fallback, and DAG synchronized move/release coverage. +- State: complete. ## Verification log @@ -112,6 +122,14 @@ This file is the resumable implementation log for - Phase 2 application builds passed for `sf-builder`, `dag-builder`, and `dag-convert-debug`. - Phase 2 `git diff --check`: passed. +- Phase 3 focused terrain store/open tests: 233 assertions in 25 test cases + passed. +- Phase 3 focused DAG store tests: 17 assertions in 2 test cases passed. +- Phase 3 `unittests_terrainlib`: 23,973 assertions in 422 test cases passed. +- Phase 3 `unittests_dagbuilder`: 459 assertions in 73 test cases passed. +- Phase 3 all-application configuration built `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. +- Phase 3 `git diff --check`: passed. ## Commit and tag log @@ -124,3 +142,6 @@ This file is the resumable implementation log for - `refactor_phase_0` — annotated Phase 0 completion tag. - `4c171c8` / `refactor_phase_1` — shared topology extraction and Phase 1 completion tag. +- `fbc02d0` / `refactor_phase_2` — runtime layouts/codecs and Phase 2 + completion tag. +- `61acb74` — Phase 3 shared runtime-codec storage core. diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 567fe005..203dca83 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -36,6 +36,7 @@ #include "range_utils.h" #include "storage.h" #include "thread_safe_storage.h" +#include "store/describe_error.h" #include "utils.h" #include "vertex_lock.h" #include "parallel.h" @@ -52,8 +53,13 @@ std::optional load_and_clusterize_mesh( const auto result = storage.load(id); if (!result) { - if (result.error() != mesh::io::LoadMeshErrorKind::FileNotFound) { - LOG_ERROR("Failed to read node {}: {}", id, result.error()); + const auto *codec_error = std::get_if(&result.error()); + if (codec_error == nullptr + || codec_error->category != store::CodecErrorCategory::FileNotFound) { + LOG_ERROR( + "Failed to read node {}: {}", + id, + store::describe_error(result.error())); } return std::nullopt; } @@ -448,7 +454,7 @@ LevelWorkplan build_level_workplan( // Pre-filter input nodes to only those that intersect the target root bounds. std::vector> gather_relevant_input_leaves( - const octree::IndexMap &index, + const store::Index &index, const octree::Space &space, const radix::geometry::Aabb3d &root_bounds) { @@ -500,7 +506,7 @@ std::unordered_set build_level( std::unordered_set already_built; if (ctx.options.continuation_mode != ContinuationMode::Overwrite) { for (const octree::Id &target : targets) { - if (ctx.output_storage.has(target)) { + if (DEBUG_ASSERT_VAL(ctx.output_storage.has(target)).value()) { already_built.insert(target); } } @@ -513,10 +519,18 @@ std::unordered_set build_level( // Initialize debug storage if requested (contains .glb meshes) std::optional debug_storage; if (ctx.options.write_debug_meshes) { - debug_storage = octree::open_folder( + octree::OpenOptions options; + options.preferred_extension = ".glb"; + auto debug_result = octree::open_folder( ctx.output_storage.base_path().string() + "-debug", false, - octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + std::move(options)); + if (!debug_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open debug mesh storage: {}", + store::describe_error(debug_result.error())); + } + debug_storage = std::move(debug_result.value()); } tbb::concurrent_vector saved_ids; @@ -540,12 +554,18 @@ std::unordered_set build_level( if (debug_storage) { const auto debug_save_result = debug_storage->save(target, clustering_to_mesh(result->clustering)); if (!debug_save_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save debug mesh for node {}: {}", target, debug_save_result.error()); + LOG_ERROR_AND_EXIT( + "Failed to save debug mesh for node {}: {}", + target, + store::describe_error(debug_save_result.error())); } } saved_ids.push_back(target); } else { - LOG_ERROR("Failed to save node {}: {}", target, save_result.error()); + LOG_ERROR( + "Failed to save node {}: {}", + target, + store::describe_error(save_result.error())); } } progress.task_finished(); @@ -610,7 +630,10 @@ void build_levels( // Persist per level so a finished level can be read back before the whole run completes if (const auto result = ctx.output_storage.save_or_create_index(); !result.has_value()) { - LOG_WARN("Could not save index after level {}: {}", level, result.error()); + LOG_WARN( + "Could not save index after level {}: {}", + level, + store::describe_error(result.error())); } } diff --git a/src/dag_builder/codec/MetadataView.h b/src/dag_builder/codec/MetadataView.h new file mode 100644 index 00000000..9df6b856 --- /dev/null +++ b/src/dag_builder/codec/MetadataView.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +#include + +#include "io/serialize.h" +#include "serialization.h" +#include "store/Codec.h" +#include "store/codec/ZppBits.h" + +namespace dag::codec { + +class MetadataView final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override { + return {store::codec::append_extension(node_path, ".bin")}; + } + + std::expected read( + const store::NodePath &node_path) const override { + const auto result = io::read_from_path(paths(node_path).front()); + if (!result.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + result.error() == io::Error(io::Error::OpenFile) + ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::Io, + fmt::format("{}", result.error()), + }); + } + return result.value(); + } +}; + +} // namespace dag::codec diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h new file mode 100644 index 00000000..3233f5af --- /dev/null +++ b/src/dag_builder/codec/from_extension.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include + +#include "codec/MetadataView.h" +#include "dag_node.h" +#include "serialization.h" +#include "store/codec/ZppBits.h" + +namespace dag::codec { + +inline std::expected>, store::CodecError> +from_extension(const std::string_view extension) { + if (extension == ".bin") { + return std::make_unique>(); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported DAG codec selector: " + std::string(extension), + }); +} + +inline std::expected>, store::CodecError> +metadata_from_extension(const std::string_view extension) { + if (extension == ".bin") { + return std::make_unique(); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported DAG metadata codec selector: " + std::string(extension), + }); +} + +} // namespace dag::codec diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index acce9d48..495bbf09 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -5,10 +5,9 @@ #include "cli.h" #include "build.h" #include "log.h" -#include "octree/storage/Storage.h" -#include "octree/storage/MeshStorage.h" #include "octree/storage/open.h" #include "storage.h" +#include "store/describe_error.h" #include "ContinuationMode.h" int main(int argc, char **argv) { @@ -17,8 +16,24 @@ int main(int argc, char **argv) { Log::init(args.log_level); try { - const octree::IndexedMeshStorage input_storage = octree::open_folder_indexed(args.input_path); - octree::IndexedDagStorage output_storage = octree::open_folder_indexed(args.output_path); + auto input_result = octree::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR( + "Failed to open input dataset {}: {}", + args.input_path, + store::describe_error(input_result.error())); + return EXIT_FAILURE; + } + auto output_result = dag::storage::open_folder_indexed(args.output_path); + if (!output_result.has_value()) { + LOG_ERROR( + "Failed to open output dataset {}: {}", + args.output_path, + store::describe_error(output_result.error())); + return EXIT_FAILURE; + } + const octree::IndexedMeshStorage input_storage = std::move(input_result.value()); + octree::IndexedDagStorage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = args.continuation_mode == ContinuationMode::Overwrite; dag::BuildOptions options{ @@ -46,7 +61,10 @@ int main(int argc, char **argv) { dag::build_levels(input_storage, output_storage, options, args.level_range); const auto index_result = output_storage.save_index(); if (!index_result.has_value()) { - LOG_ERROR("Failed to save output index in {}: {}", args.output_path, index_result.error()); + LOG_ERROR( + "Failed to save output index in {}: {}", + args.output_path, + store::describe_error(index_result.error())); return EXIT_FAILURE; } diff --git a/src/dag_builder/metadata.h b/src/dag_builder/metadata.h index 6c12b2bf..e5c5d35e 100644 --- a/src/dag_builder/metadata.h +++ b/src/dag_builder/metadata.h @@ -8,9 +8,6 @@ #include "cluster.h" #include "dag_id.h" #include "utils.h" -#include "octree/storage/defaults.h" -#include "octree/storage/codec/ZppBitsCodec.h" -#include "octree/storage/codec/ReadOnlyCodec.h" namespace dag { @@ -71,11 +68,3 @@ inline const radix::geometry::Aabb3d &get_group_bounds(const NodeMetadata &metad } } // namespace dag - -// Set default codec for metadata. -namespace octree { -template <> -struct DefaultCodec { - using type = ReadOnlyCodec>; -}; -} diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 7fb24242..92bccec9 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -1,14 +1,67 @@ #pragma once +#include +#include + +#include "codec/from_extension.h" +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open_runtime.h" #include "serialization.h" -#include "octree/storage/Storage.h" -#include "octree/storage/IndexedStorage.h" +#include "store/IndexedStorage.h" + namespace octree { -using DagStorage = Storage_; -using IndexedDagStorage = IndexedStorage_; +using DagStorage = store::Storage; +using IndexedDagStorage = store::IndexedStorage; +using DagMetaStorage = store::Storage; +using IndexedDagMetaStorage = store::IndexedStorage; + +} // namespace octree -using DagMetaStorage = Storage_; -using IndexedDagMetaStorage = IndexedStorage_; +namespace dag::storage { + +using OpenOptions = octree::storage::OpenOptions; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return store::open_index( + path, + octree::storage::index_format(), + dag::codec::from_extension); +} +inline std::expected> open_folder( + const std::filesystem::path &path, + const bool create_index = false, + OpenOptions options = {}) { + return octree::storage::open_folder( + path, + create_index, + ".bin", + dag::codec::from_extension, + std::move(options)); } + +inline std::expected> +open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { + return octree::storage::open_folder_indexed( + path, + ".bin", + dag::codec::from_extension, + std::move(options)); +} + +inline std::expected> +open_metadata_indexed(const std::filesystem::path &path) { + const std::filesystem::path index_path = + path.filename() == octree::disk::v1::index_file_name() + ? path + : path / octree::disk::v1::index_file_name(); + return store::open_index( + index_path, + octree::storage::index_format(), + dag::codec::metadata_from_extension); +} + +} // namespace dag::storage diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h index 9b5e6120..da54f8a0 100644 --- a/src/dag_builder/thread_safe_storage.h +++ b/src/dag_builder/thread_safe_storage.h @@ -6,8 +6,6 @@ #include -#include "octree/Id.h" - namespace dag { // A mutex-based thread-safe wrapper around a Storage_ instance. @@ -18,6 +16,7 @@ class ThreadSafeStorage { public: using value_type = typename Storage::value_type; + using key_type = typename Storage::key_type; using load_error = typename Storage::load_error; using save_error = typename Storage::save_error; @@ -33,12 +32,12 @@ class ThreadSafeStorage { return std::move(this->_storage); } - std::expected load(const octree::Id &id) const { + std::expected load(const key_type &id) const { std::shared_lock lock(this->_mutex); return this->_storage.load(id); } - bool has(const octree::Id &id) const { + auto has(const key_type &id) const { std::shared_lock lock(this->_mutex); return this->_storage.has(id); } @@ -47,7 +46,7 @@ class ThreadSafeStorage { return this->_storage.base_path(); } - std::expected save(const octree::Id &id, const value_type &value) const { + std::expected save(const key_type &id, const value_type &value) const { std::unique_lock lock(this->_mutex); return this->_storage.save(id, value); } diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index 88f776fa..d72be3a2 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -4,19 +4,21 @@ #include "cli.h" #include "dag_node.h" #include "encoded.h" +#include "io/serialize.h" #include "log.h" #include "mesh/io.h" #include "octree/storage/MeshStorage.h" -#include "octree/storage/codec/ZppBitsCodec.h" #include "octree/storage/open.h" #include "ProgressIndicator.h" #include "storage.h" #include "utils.h" +#include "serialization.h" +#include "store/describe_error.h" namespace { void export_node(const cli::Args &args) { - const auto load_result = octree::ZppBitsCodec::load_from_path(args.input_path); + const auto load_result = io::read_from_path(args.input_path); if (!load_result.has_value()) { LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error()); return; @@ -31,12 +33,30 @@ void export_node(const cli::Args &args) { } void export_storage(const cli::Args &args) { - const octree::IndexedDagStorage input_storage = octree::open_folder_indexed(args.input_path); + auto input_result = dag::storage::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR( + "Failed to open input storage {}: {}", + args.input_path, + store::describe_error(input_result.error())); + return; + } + const octree::IndexedDagStorage input_storage = std::move(input_result.value()); - octree::MeshStorage output_storage = octree::open_folder( + octree::OpenOptions options; + options.preferred_extension = ".glb"; + auto output_result = octree::open_folder( args.output_path, false, - octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + std::move(options)); + if (!output_result.has_value()) { + LOG_ERROR( + "Failed to open output storage {}: {}", + args.output_path, + store::describe_error(output_result.error())); + return; + } + octree::MeshStorage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = true; size_t exported_count = 0; @@ -52,7 +72,10 @@ void export_storage(const cli::Args &args) { const auto load_result = input_storage.load(id); if (!load_result.has_value()) { - LOG_ERROR("Failed to load node {}: {}", id, load_result.error()); + LOG_ERROR( + "Failed to load node {}: {}", + id, + store::describe_error(load_result.error())); progress.task_finished(); continue; } @@ -61,7 +84,10 @@ void export_storage(const cli::Args &args) { const auto save_result = output_storage.save(id, mesh); if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh for node {}: {}", id, save_result.error().description()); + LOG_ERROR( + "Failed to save mesh for node {}: {}", + id, + store::describe_error(save_result.error())); progress.task_finished(); continue; } @@ -74,7 +100,10 @@ void export_storage(const cli::Args &args) { const auto index_result = output_storage.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR("Failed to save index for {}: {}", args.output_path, index_result.error()); + LOG_ERROR( + "Failed to save index for {}: {}", + args.output_path, + store::describe_error(index_result.error())); } LOG_INFO("Exported {} debug meshes to {}", exported_count, args.output_path); diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 083ab6ba..a5780999 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -42,8 +42,9 @@ #include "octree/Id.h" #include "octree/Space.h" #include "octree/Storage.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" +#include "octree/storage/open.h" #include "octree/utils.h" +#include "store/describe_error.h" namespace terrainbuilder { @@ -189,13 +190,19 @@ void build_all_patches( LOG_ERROR_AND_EXIT("Output base path {} exists but is not a directory", output_base_path); } - octree::Storage storage = octree::open_folder( + octree::OpenOptions open_options; + open_options.preferred_extension = output_format; + auto storage_result = octree::open_folder( output_base_path, false, - octree::OpenOptions { - .preferred_extension_with_dot = output_format - } - ); + std::move(open_options)); + if (!storage_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + output_base_path, + store::describe_error(storage_result.error())); + } + octree::Storage storage = std::move(storage_result.value()); storage.settings().allow_overwrite = overwrite_existing; const auto dataset_srs = dataset.srs(); @@ -281,7 +288,17 @@ void build_all_patches( tbb::task_group_context context; tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; - if (!overwrite_existing && storage.has(node)) { + const auto already_exists = storage.has(node); + if (!already_exists.has_value()) { + LOG_ERROR( + "Failed to inspect node {}: {}", + node, + store::describe_error(already_exists.error())); + progress.task_finished(); + context.cancel_group_execution(); + return; + } + if (!overwrite_existing && already_exists.value()) { progress.task_finished(); // TODO: correctly handle virtual nodes return; } @@ -305,7 +322,10 @@ void build_all_patches( mesh::validate(mesh); const auto save_result = storage.save(node, mesh); if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh for node {}: {}", node, save_result.error()); + LOG_ERROR( + "Failed to save mesh for node {}: {}", + node, + store::describe_error(save_result.error())); progress.task_finished(); context.cancel_group_execution(); return; @@ -329,7 +349,10 @@ void build_all_patches( const auto index_result = storage.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", storage.base_path(), index_result.error()); + LOG_ERROR_AND_EXIT( + "Failed to save output index in {}: {}", + storage.base_path(), + store::describe_error(index_result.error())); } } } diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index 74c8f304..0c6e6ddd 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -15,8 +16,10 @@ #include "octree/NodeStatus.h" #include "octree/NodeStatusOrMissing.h" #include "octree/Storage.h" +#include "octree/storage/open.h" #include "octree/traverse.h" #include "cli.h" +#include "store/describe_error.h" namespace { struct IndexNode { @@ -42,7 +45,8 @@ class TreeView { using const_iterator = std::list::const_iterator; explicit TreeView(const octree::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { - const octree::NodeStatusOrMissing status = this->_storage.index().get(root); + const octree::NodeStatusOrMissing status( + DEBUG_ASSERT_VAL(this->_storage.index().get(root)).value()); this->_view.emplace_back(IndexNode{root, status, false}); } @@ -85,7 +89,7 @@ class TreeView { const auto &mesh = result.value(); entry = MeshNode{parent.id, mesh.vertex_count(), mesh.face_count()}; } else { - entry = ErrorNode{parent.id, result.error().description()}; + entry = ErrorNode{parent.id, store::describe_error(result.error())}; } it++; it = this->_view.emplace(it, entry); @@ -97,11 +101,12 @@ class TreeView { } const auto children = parent.id.children().value(); for (const auto &child_id : children) { - auto status_opt = this->_storage.index().get(child_id); - if (!status_opt.has_value()) { + auto status_result = this->_storage.index().get(child_id); + DEBUG_ASSERT(status_result.has_value()); + if (!status_result->has_value()) { continue; } - const octree::NodeStatus status = status_opt.value(); + const octree::NodeStatus status = status_result->value(); IndexNode child_node{child_id, status, false}; it++; it = this->_view.emplace(it, child_node); @@ -148,8 +153,10 @@ class TreeView { const octree::IndexedStorage& _storage; }; -const octree::Id find_deepest_root(const octree::IndexMap &index, const octree::Id &root = octree::Id::root()) { - switch (octree::NodeStatusOrMissing(index.get(root))) { +const octree::Id find_deepest_root( + const store::Index &index, + const octree::Id &root = octree::Id::root()) { + switch (octree::NodeStatusOrMissing(DEBUG_ASSERT_VAL(index.get(root)).value())) { case octree::NodeStatusOrMissing::Missing: return octree::Id::root(); case octree::NodeStatusOrMissing::Leaf: @@ -163,7 +170,8 @@ const octree::Id find_deepest_root(const octree::IndexMap &index, const octree:: const auto children = current.children().value(); std::optional next; for (const octree::Id &child_id : children) { - if (!index.get(child_id).has_value()) { + const auto status = DEBUG_ASSERT_VAL(index.get(child_id)).value(); + if (!status.has_value()) { continue; } if (next.has_value()) { @@ -184,11 +192,16 @@ const octree::Id find_deepest_root(const octree::IndexMap &index, const octree:: } // namespace octree::IndexedStorage open_path_indexed(const std::filesystem::path& path) { - if (std::filesystem::is_directory(path)) { - return octree::open_folder_indexed(path); - } else { - return octree::open_index(path).value(); + auto result = std::filesystem::is_directory(path) + ? octree::open_folder_indexed(path) + : octree::open_index(path); + if (!result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open dataset {}: {}", + path, + store::describe_error(result.error())); } + return std::move(result.value()); } std::string render_line_content(const DisplayEntry &entry) { @@ -220,7 +233,7 @@ ftxui::Element render_line(const DisplayEntry &entry, size_t base_level, bool is int run(const cli::Args &args) { const octree::IndexedStorage storage = open_path_indexed(args.dataset_path); - const octree::IndexMap &index = storage.index(); + const store::Index &index = storage.index(); const octree::Id root_id = args.full_view ? octree::Id::root() : find_deepest_root(index); TreeView tree_view(storage, root_id); diff --git a/src/sf_merger/NodeLoader.h b/src/sf_merger/NodeLoader.h index c6ac4027..c7c44b90 100644 --- a/src/sf_merger/NodeLoader.h +++ b/src/sf_merger/NodeLoader.h @@ -11,21 +11,18 @@ #include "octree/Id.h" #include "octree/NodeStatusOrMissing.h" #include "octree/Space.h" -#include "octree/storage/IndexedStorage.h" +#include "octree/Storage.h" #include "octree/storage/cache/Dummy.h" #include "octree/storage/cache/ICache.h" class NodeLoader { public: NodeLoader(const octree::IndexedStorage &storage, octree::cache::ICache &cache, octree::Space space) - : _storage(storage), _cache(cache), _space(space) { - if (storage.cache().has_value() && dynamic_cast *>(&cache) != nullptr) { - LOG_WARN("Backing storage for NodeLoader instance is cached, but another cache was provided leading to double caching."); - } - } + : _storage(storage), _cache(cache), _space(space) {} octree::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { - return this->_storage.index().get(id); + return octree::NodeStatusOrMissing( + DEBUG_ASSERT_VAL(this->_storage.index().get(id)).value()); } // Try to retrieve or reconstruct node mesh by ID diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index f160a8ad..6c5bf290 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -6,7 +6,7 @@ #include "mesh/SimpleMesh.h" #include "octree/Id.h" #include "octree/NodeStatus.h" -#include "octree/storage/Storage.h" +#include "octree/Storage.h" #include "octree/traverse.h" // TODO: make thread safe @@ -15,13 +15,13 @@ class NodeWriter { NodeWriter(octree::Storage &storage) : _storage(storage) {} bool has_node(const octree::Id &id) { - return this->_storage.has(id); + return DEBUG_ASSERT_VAL(this->_storage.has(id)).value(); } void write_node(const octree::Id &id, const SimpleMesh &mesh) { mesh::validate(mesh); DEBUG_ASSERT_VAL(this->_storage.save(id, mesh)); - auto p = this->_storage.path_for(id); + auto p = DEBUG_ASSERT_VAL(this->_storage.path_for(id)).value(); // change extension to .png p.replace_extension(".png"); cv::imwrite(p, mesh.texture.value_or(cv::Mat())); diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 6f61d554..1e785618 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -7,12 +7,13 @@ #include "mask.h" #include "octree/Id.h" #include "octree/NodeStatus.h" -#include "octree/storage/IndexedStorage.h" +#include "octree/Storage.h" #include "octree/storage/open.h" #include "octree/Space.h" #include "octree/traverse.h" #include "utils.h" #include "containers/Cow.h" +#include "store/describe_error.h" struct Context { const octree::IndexedStorage& input; @@ -31,7 +32,8 @@ inline void cut_leaf_node( const octree::Id& id, const MeshMask& mask ) { - DEBUG_ASSERT(ctx.input.index().is(octree::NodeStatus::Leaf, id)); + DEBUG_ASSERT(DEBUG_ASSERT_VAL( + ctx.input.index().is(octree::NodeStatus::Leaf, id)).value()); const SimpleMesh mesh = DEBUG_ASSERT_VAL(ctx.input.load(id)).value(); LOG_TRACE("Cutting mesh at {} using mask with {} vertices and {} triangles", @@ -57,7 +59,8 @@ inline void cut_virtual_node( const octree::Id& id, const MeshMask& mask ) { - DEBUG_ASSERT(ctx.input.index().is(octree::NodeStatus::Virtual, id)); + DEBUG_ASSERT(DEBUG_ASSERT_VAL( + ctx.input.index().is(octree::NodeStatus::Virtual, id)).value()); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); @@ -88,12 +91,13 @@ inline void cut_node( mesh::io::save_to_path(mask.mesh, new_path); */ - const auto status_opt = ctx.input.index().get(id); - if (!status_opt.has_value()) { + const auto status_result = ctx.input.index().get(id); + DEBUG_ASSERT(status_result.has_value()); + if (!status_result->has_value()) { return; } - const octree::NodeStatus status = status_opt.value(); + const octree::NodeStatus status = status_result->value(); switch (status) { case octree::NodeStatus::Virtual: cut_virtual_node(ctx, id, mask); @@ -116,7 +120,10 @@ inline void cut_dataset( cut_node(ctx, octree::Id::root(), mask); const auto index_result = output.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output.base_path(), index_result.error()); + LOG_ERROR_AND_EXIT( + "Failed to save output index in {}: {}", + output.base_path(), + store::describe_error(index_result.error())); } } @@ -128,7 +135,17 @@ inline void cut_dataset( LOG_TRACE("Creating output dataset at {}", output_path); std::filesystem::create_directories(output_path); - octree::IndexedStorage output_dataset = octree::open_folder_indexed(output_path, octree::OpenOptions(octree::disk::layout::strategy::make_default(), ".glb")); + octree::OpenOptions options; + options.default_mapping = octree::store_layout::level_and_coordinate_directories(); + options.preferred_extension = ".glb"; + auto output_result = octree::open_folder_indexed(output_path, std::move(options)); + if (!output_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + output_path, + store::describe_error(output_result.error())); + } + octree::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { LOG_ERROR_AND_EXIT("Output dataset has to be empty."); } diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 6ff4896a..76894680 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -11,6 +11,7 @@ #include "octree/Storage.h" #include "optional_utils.h" #include "earth.h" +#include "store/describe_error.h" std::optional load_mask_from_path(const std::filesystem::path& path) { if (std::filesystem::exists(path)) { @@ -34,14 +35,38 @@ std::optional load_mask_from_path(const std::filesystem::path& path) { void run(const cli::MergeArgs& args) { LOG_TRACE("Loading base dataset from {}", args.base_path); - octree::IndexedStorage base_dataset = octree::open_folder_indexed(args.base_path); + auto base_result = octree::open_folder_indexed(args.base_path); + if (!base_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open base dataset {}: {}", + args.base_path, + store::describe_error(base_result.error())); + } + octree::IndexedStorage base_dataset = std::move(base_result.value()); LOG_TRACE("Loading new dataset from {}", args.new_path); - octree::IndexedStorage new_dataset = octree::open_folder_indexed(args.new_path); + auto new_result = octree::open_folder_indexed(args.new_path); + if (!new_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open new dataset {}: {}", + args.new_path, + store::describe_error(new_result.error())); + } + octree::IndexedStorage new_dataset = std::move(new_result.value()); LOG_TRACE("Creating output dataset at {}", args.output_path); std::filesystem::create_directories(args.output_path); - octree::Storage output_dataset = octree::open_folder(args.output_path, false, octree::OpenOptions{.preferred_extension_with_dot = std::string(base_dataset.layout().extension_with_dot())}); + octree::OpenOptions options; + options.default_mapping = base_dataset.layout().mapping(); + options.preferred_extension = std::string(base_dataset.codec_selector().value_or(".terrain")); + auto output_result = octree::open_folder(args.output_path, false, std::move(options)); + if (!output_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + args.output_path, + store::describe_error(output_result.error())); + } + octree::Storage output_dataset = std::move(output_result.value()); std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); @@ -50,7 +75,14 @@ void run(const cli::MergeArgs& args) { void run(const cli::CutArgs& args) { LOG_TRACE("Loading input dataset from {}", args.input_path); - const octree::IndexedStorage input_dataset = octree::open_folder_indexed(args.input_path); + auto input_result = octree::open_folder_indexed(args.input_path); + if (!input_result.has_value()) { + LOG_ERROR_AND_EXIT( + "Failed to open input dataset {}: {}", + args.input_path, + store::describe_error(input_result.error())); + } + const octree::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); cut_dataset(input_dataset, mask, args.output_path, args.keep_inside); } diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index 180dbb59..b84fe38d 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -14,6 +14,7 @@ #include "octree/Id.h" #include "octree/NodeStatusOrMissing.h" #include "octree/storage/cache/Dummy.h" +#include "store/describe_error.h" inline std::string get_dataset_name(const octree::Storage &storage) { return storage.layout().base_path().filename().string(); @@ -179,6 +180,9 @@ inline void merge_datasets( const auto index_result = output_dataset.save_or_create_index(); if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output_dataset.base_path(), index_result.error()); + LOG_ERROR_AND_EXIT( + "Failed to save output index in {}: {}", + output_dataset.base_path(), + store::describe_error(index_result.error())); } } diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 4046109b..5ed36a37 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -36,7 +36,6 @@ add_library(terrainlib mesh/texture_trim.cpp mesh/validate.cpp - octree/storage/helpers.cpp octree/Space.cpp octree/OddLevelShifted.cpp diff --git a/src/terrainlib/mesh/codec/from_extension.h b/src/terrainlib/mesh/codec/from_extension.h new file mode 100644 index 00000000..abc18813 --- /dev/null +++ b/src/terrainlib/mesh/codec/from_extension.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +#include + +#include "mesh/codec/Gltf.h" +#include "mesh/codec/Terrain.h" +#include "store/Codec.h" + +namespace mesh::codec { + +inline std::expected>, store::CodecError> +from_extension(const std::string_view extension) { + if (extension == ".terrain") { + return std::make_unique(); + } + if (extension == ".glb") { + return std::make_unique(GltfContainer::Binary); + } + if (extension == ".gltf") { + return std::make_unique(GltfContainer::Json); + } + return std::unexpected(store::CodecError{ + store::CodecOperation::Resolve, + store::CodecErrorCategory::UnsupportedCodec, + "unsupported mesh codec selector: " + std::string(extension), + }); +} + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/storage.h b/src/terrainlib/mesh/storage.h new file mode 100644 index 00000000..102f6afc --- /dev/null +++ b/src/terrainlib/mesh/storage.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/codec/from_extension.h" +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open_runtime.h" +#include "store/IndexedStorage.h" + +namespace mesh::storage { + +using Storage = store::Storage; +using IndexedStorage = store::IndexedStorage; +using OpenOptions = octree::storage::OpenOptions; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return store::open_index( + path, + octree::storage::index_format(), + mesh::codec::from_extension); +} + +inline std::expected> open_folder( + const std::filesystem::path &path, + const bool create_index = false, + OpenOptions options = {}) { + return octree::storage::open_folder( + path, + create_index, + ".terrain", + mesh::codec::from_extension, + std::move(options)); +} + +inline std::expected> open_folder_indexed( + const std::filesystem::path &path, + OpenOptions options = {}) { + return octree::storage::open_folder_indexed( + path, + ".terrain", + mesh::codec::from_extension, + std::move(options)); +} + +} // namespace mesh::storage diff --git a/src/terrainlib/octree/IndexMap.h b/src/terrainlib/octree/IndexMap.h index c231e763..c45d7da1 100644 --- a/src/terrainlib/octree/IndexMap.h +++ b/src/terrainlib/octree/IndexMap.h @@ -16,6 +16,9 @@ class IndexMap { using iterator = SharedIndex::iterator; using const_iterator = SharedIndex::const_iterator; + IndexMap() = default; + explicit IndexMap(SharedIndex index) : _index(std::move(index)) {} + std::optional get(const Id id) const { return _index.get(id).value(); } @@ -77,6 +80,9 @@ class IndexMap { const SharedIndex &shared() const { return _index; } + SharedIndex take_shared() && { + return std::move(_index); + } using serialize = zpp::bits::members<1>; friend zpp::bits::access; diff --git a/src/terrainlib/octree/Storage.h b/src/terrainlib/octree/Storage.h index e6776555..adb5181e 100644 --- a/src/terrainlib/octree/Storage.h +++ b/src/terrainlib/octree/Storage.h @@ -1,14 +1,13 @@ #pragma once -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/MeshStorage.h" -#include "octree/storage/RawStorage.h" -#include "octree/storage/Storage.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" namespace octree { +using MeshStorage = mesh::storage::Storage; +using IndexedMeshStorage = mesh::storage::IndexedStorage; using Storage = MeshStorage; using IndexedStorage = IndexedMeshStorage; +using StorageSettings = store::StorageSettings; -} +} // namespace octree diff --git a/src/terrainlib/octree/disk/Layout.h b/src/terrainlib/octree/disk/Layout.h deleted file mode 100644 index ba56f40b..00000000 --- a/src/terrainlib/octree/disk/Layout.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" - -namespace octree::disk { - -class Layout { -public: - Layout(std::filesystem::path base_path, std::unique_ptr strategy, std::string extension_with_dot) - : _base_path(std::move(base_path)), _strategy(std::move(strategy)), _extension_with_dot(std::move(extension_with_dot)) { - DEBUG_ASSERT(!_extension_with_dot.empty() && _extension_with_dot[0] == '.'); - } - - std::filesystem::path get_node_path(Id id) const { - return this->_base_path / this->_strategy->get_relative_node_path(id, this->_extension_with_dot); - } - std::optional get_id_from_node_path(const std::filesystem::path& path) const { - const std::filesystem::path relative_path = std::filesystem::relative(path, _base_path); - return this->_strategy->get_id_from_relative_node_path(relative_path); - } - - const std::filesystem::path& base_path() const { - return this->_base_path; - } - const layout::Strategy &strategy() const { - return *this->_strategy; - } - std::string_view extension_with_dot() const { - return this->_extension_with_dot; - } - std::string_view extension() const { - return this->extension_with_dot().substr(1); - } - -private: - std::filesystem::path _base_path; - std::unique_ptr _strategy; - std::string _extension_with_dot; -}; - -} diff --git a/src/terrainlib/octree/disk/layout/Strategy.h b/src/terrainlib/octree/disk/layout/Strategy.h deleted file mode 100644 index a1d5fb70..00000000 --- a/src/terrainlib/octree/disk/layout/Strategy.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "octree/Id.h" - -namespace octree::disk::layout { - -class Strategy { -public: - virtual ~Strategy() = default; - - virtual std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const = 0; - virtual std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const = 0; -}; - -} diff --git a/src/terrainlib/octree/disk/layout/StrategyRegister.h b/src/terrainlib/octree/disk/layout/StrategyRegister.h deleted file mode 100644 index 026a44f7..00000000 --- a/src/terrainlib/octree/disk/layout/StrategyRegister.h +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include "octree/disk/layout/Strategy.h" -#include "log.h" - -namespace octree::disk::layout { - -class StrategyRegister { -public: - using Factory = std::function()>; - - static StrategyRegister &instance() { - static StrategyRegister inst; - return inst; - } - - template - void register_strategy(const std::string &id) { - this->_factories[id] = [] { return std::make_unique(); }; - this->_type_to_id[std::type_index(typeid(T))] = id; - } - - std::unique_ptr create(const std::string &id) const { - auto it = this->_factories.find(id); - if (it != this->_factories.end()) { - return (it->second)(); - } - return nullptr; - } - - const std::unordered_map &factories() const { - return this->_factories; - } - - template - std::string get_id() const { - return this->_get_id(typeid(T)); - } - - std::string get_id(const Strategy &s) const { - return this->_get_id(typeid(s)); - } - -private: - std::unordered_map _factories; - std::unordered_map _type_to_id; - - // Private ctor/dtor to enforce singleton - StrategyRegister() = default; - ~StrategyRegister() = default; - StrategyRegister(const StrategyRegister &) = delete; - StrategyRegister &operator=(const StrategyRegister &) = delete; - - std::string _get_id(std::type_index type_index) const { - auto it = this->_type_to_id.find(type_index); - if (it != this->_type_to_id.end()) { - return it->second; - } - LOG_ERROR_AND_EXIT("Encountered octree layout strategy not registered"); - } -}; - -} // namespace octree::disk::layout - -#include "octree/disk/layout/strategy/Flat.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" - -namespace { -template -struct Registrar { - Registrar(const std::string &id) { - octree::disk::layout::StrategyRegister::instance().register_strategy(id); - } -}; - -const Registrar reg_flat("flat"); -const Registrar reg_lacd("level_and_coordinate_directories"); -} diff --git a/src/terrainlib/octree/disk/layout/strategy/Default.h b/src/terrainlib/octree/disk/layout/strategy/Default.h deleted file mode 100644 index 45c7842d..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/Default.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" - -namespace octree::disk::layout::strategy { - -using Default = LevelAndCoordinateDirectories; - -inline std::unique_ptr make_default() { - return std::make_unique(); -} - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/disk/layout/strategy/Flat.h b/src/terrainlib/octree/disk/layout/strategy/Flat.h deleted file mode 100644 index 0a11a941..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/Flat.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" -#include "string_utils.h" - -namespace octree::disk::layout::strategy { - -class Flat : public Strategy { -public: - std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const override { - return fmt::format("{}-{}{}", id.level(), id.index_on_level(), extension_with_dot); - } - - std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const override { - DEBUG_ASSERT(relative_path.is_relative()); - - const std::string _filestem = relative_path.stem().string(); - const std::string_view filestem = _filestem; - - const size_t dash_pos = filestem.find('-'); - if (dash_pos == std::string::npos) { - return std::nullopt; - } - const std::string_view level_str = filestem.substr(0, dash_pos); - const std::string_view index_str = filestem.substr(dash_pos + 1); - - const auto level_opt = from_chars(level_str); - if (!level_opt) { - return std::nullopt; - } - - const auto index_opt = from_chars(index_str); - if (!index_opt) { - return std::nullopt; - } - - const Id::Level level = *level_opt; - const Id::Index index = *index_opt; - return Id::try_make(level, index); - } -}; - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h deleted file mode 100644 index 6a5ffa68..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" -#include "string_utils.h" - -namespace octree::disk::layout::strategy { - -class LevelAndCoordinateDirectories : public Strategy { -public: - std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const override { - const Id::Coords coords = id.coords(); - return fmt::format("{}/{}/{}/{}{}", id.level(), coords.x, coords.y, coords.z, extension_with_dot); - } - std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const override { - DEBUG_ASSERT(relative_path.is_relative()); - - auto it = relative_path.begin(); - if (std::distance(it, relative_path.end()) < 4) { - return std::nullopt; - } - - const auto level_opt = from_chars(it->native()); - if (!level_opt) { - return std::nullopt; - } - const Id::Level level = *level_opt; - ++it; - - const auto x_opt = from_chars(it->native()); - if (!x_opt) { - return std::nullopt; - } - const Id::Coord x = *x_opt; - ++it; - - const auto y_opt = from_chars(it->native()); - if (!y_opt) { - return std::nullopt; - } - const Id::Coord y = *y_opt; - ++it; - - const auto z_opt = from_chars(it->stem().native()); - if (!z_opt) { - return std::nullopt; - } - const Id::Coord z = *z_opt; - - return Id::try_make(level, {x, y, z}); - } -}; - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/storage/CopyError.h b/src/terrainlib/octree/storage/CopyError.h deleted file mode 100644 index 7bd6c278..00000000 --- a/src/terrainlib/octree/storage/CopyError.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include -#include - -namespace octree { - -enum class CopyErrorKind { - FileNotFound, - CreateLink, - CreateDirectories, - RemoveOld, - Read, - Write -}; - -class CopyError { -public: - constexpr CopyError(CopyErrorKind kind) - : kind(kind) {} - - operator CopyErrorKind() const { - return this->kind; - } - constexpr bool operator==(CopyError other) const { - return this->kind == other.kind; - } - constexpr bool operator!=(CopyError other) const { - return this->kind != other.kind; - } - - std::string description() const { - switch (kind) { - case CopyErrorKind::FileNotFound: - return "file not found"; - case CopyErrorKind::CreateLink: - return "cannot create hard link"; - case CopyErrorKind::CreateDirectories: - return "cannot create directories"; - case CopyErrorKind::RemoveOld: - return "cannot remove current file"; - case CopyErrorKind::Read: - return "cannot read file"; - case CopyErrorKind::Write: - return "cannot write file"; - default: - return "undefined error"; - } - } - - friend std::ostream &operator<<(std::ostream &os, const CopyError &err) { - return os << err.description(); - } - -private: - CopyErrorKind kind; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h new file mode 100644 index 00000000..7d70fbde --- /dev/null +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -0,0 +1,68 @@ +#pragma once + +#include + +#include + +#include "io/serialize.h" +#include "octree/IndexMap.h" +#include "octree/StoreTraits.h" +#include "octree/disk/IndexFile.h" +#include "octree/store_layout/Mappings.h" +#include "store/IndexFormat.h" + +namespace octree::storage { + +inline std::expected, store::IndexFormatError> +read_index_file(const std::filesystem::path &path) { + auto result = io::read_from_path(path); + if (!result.has_value()) { + return std::unexpected(store::IndexFormatError{ + result.error() == io::Error(io::Error::OpenFile) + ? store::IndexFormatErrorCategory::Open + : store::IndexFormatErrorCategory::Malformed, + path, + fmt::format("{}", result.error()), + }); + } + disk::v1::IndexFile file = std::move(result.value()); + return store::IndexMetadata{ + std::move(file.map).take_shared(), + std::move(file.layout_strategy_id), + std::move(file.preferred_extension), + }; +} + +inline std::expected write_index_file( + const std::filesystem::path &path, + const store::IndexMetadata &metadata) { + disk::v1::IndexFile file; + file.layout_strategy_id = metadata.layout_id; + file.preferred_extension = metadata.codec_selector; + file.map = IndexMap(metadata.index); + const auto result = io::write_to_path(file, path); + if (!result.has_value()) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Write, + path, + fmt::format("{}", result.error()), + }); + } + return {}; +} + +inline store::PathMapping default_mapping() { + return store_layout::level_and_coordinate_directories(); +} + +inline store::IndexFormat index_format() { + return { + disk::v1::index_file_name(), + read_index_file, + write_index_file, + store_layout::from_id, + default_mapping, + }; +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/storage/IndexedStorage.h b/src/terrainlib/octree/storage/IndexedStorage.h deleted file mode 100644 index 3ba94e8f..00000000 --- a/src/terrainlib/octree/storage/IndexedStorage.h +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/disk/Layout.h" -#include "octree/storage/Storage.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" -#include "octree/storage/helpers.h" - -namespace octree { - -template Codec = DefaultCodecFor> -class IndexedStorage_ : public Storage_ { -public: - explicit IndexedStorage_(Storage_ inner) noexcept - : Storage_(std::move(inner)) { - this->ensure_indexed(); - DEBUG_ASSERT(this->is_indexed()); - } - explicit IndexedStorage_(RawStorage_ inner, IndexMap map) noexcept - : Storage_(std::move(inner), std::move(map)) { - DEBUG_ASSERT(this->is_indexed()); - } - - IndexedStorage_ &operator=(const IndexedStorage_ &) = delete; - IndexedStorage_(const IndexedStorage_ &) = delete; - IndexedStorage_(IndexedStorage_ &&) = default; - IndexedStorage_ &operator=(IndexedStorage_ &&) = default; - - ~IndexedStorage_() override { - if (this->is_index_dirty()) { - LOG_WARN("Index was not saved upon IndexedStorage_ destruction, use save_index()."); - } - } - - const IndexMap& index() const noexcept { - DEBUG_ASSERT(this->is_indexed()); - return Storage_::index().value(); - } - void update_index() noexcept { - Storage_::update_index(); - } - std::expected save_index() const noexcept { - if (!this->is_index_dirty()) { - return {}; - } - - auto result = helpers::save_index_map(this->index(), this->layout()); - if (result.has_value()) { - this->set_index_dirty(false); - } - return result; - } - -private: - IndexMap& index_mut() noexcept { - DEBUG_ASSERT(this->is_indexed()); - return Storage_::index_mut().value(); - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/MeshStorage.h b/src/terrainlib/octree/storage/MeshStorage.h index 46b69813..e2d8b585 100644 --- a/src/terrainlib/octree/storage/MeshStorage.h +++ b/src/terrainlib/octree/storage/MeshStorage.h @@ -1,12 +1,3 @@ #pragma once -#include "mesh/SimpleMesh.h" -#include "octree/storage/Storage.h" -#include "octree/storage/IndexedStorage.h" - -namespace octree { - -using MeshStorage = Storage_; -using IndexedMeshStorage = IndexedStorage_; - -} +#include "octree/Storage.h" diff --git a/src/terrainlib/octree/storage/RawStorage.h b/src/terrainlib/octree/storage/RawStorage.h deleted file mode 100644 index 39482250..00000000 --- a/src/terrainlib/octree/storage/RawStorage.h +++ /dev/null @@ -1,118 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/Layout.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/CopyError.h" -#include "octree/storage/defaults.h" - -namespace octree { - -template Codec = DefaultCodecFor> -class RawStorage_ { -public: - using value_type = T; - using codec_type = Codec; - using load_error = typename Codec::load_error; - using save_error = typename Codec::save_error; - - explicit RawStorage_(disk::Layout layout) noexcept : _layout(std::move(layout)) {} - - ~RawStorage_() = default; - RawStorage_ &operator=(const RawStorage_ &) = delete; - RawStorage_(const RawStorage_ &) = delete; - RawStorage_(RawStorage_ &&) = default; - RawStorage_ &operator=(RawStorage_ &&) = default; - - std::expected load(const Id &id) const noexcept { - const auto path = this->path_for(id); - return Codec::load_from_path(path); - } - - std::expected save(const Id &id, const T &node) const noexcept { - const auto path = this->path_for(id); - return Codec::save_to_path(node, path); - } - - std::expected copy_to(const Id &id, RawStorage_ &target) const noexcept { - return target.copy_from(id, *this); - } - - std::expected copy_from(const Id &id, const RawStorage_ &source) noexcept { - if (!source.has(id)) { - return std::unexpected(CopyErrorKind::FileNotFound); - } - - const auto source_path = source.path_for(id); - const auto target_path = this->path_for(id); - if (source_path.extension() != target_path.extension()) { - // TODO: should this error instead? - const auto load_result = source.load(id); - if (!load_result.has_value()) { - return std::unexpected(CopyErrorKind::Read); - } - const value_type node = load_result.value(); - - const auto save_result = this->save(id, node); - if (!save_result.has_value()) { - return std::unexpected(CopyErrorKind::Write); - } - return {}; - } - - std::error_code ec; - if (std::filesystem::remove(target_path, ec)) { - if (ec) { - return std::unexpected(CopyErrorKind::RemoveOld); - } - } - - std::filesystem::create_directories(target_path.parent_path(), ec); - if (ec) { - return std::unexpected(CopyErrorKind::CreateDirectories); - } - - std::filesystem::create_hard_link(source_path, target_path, ec); - if (ec) { - return std::unexpected(CopyErrorKind::CreateLink); - } - - return {}; - } - - bool remove(const Id &id) const noexcept { - const auto path = this->path_for(id); - std::error_code ec; - const auto removed = std::filesystem::remove(path, ec); - return !ec && removed; - } - - bool has(const Id &id) const noexcept { - const auto path = this->path_for(id); - std::error_code ec; - const auto exists = std::filesystem::exists(path, ec); - return !ec && exists; - } - - std::filesystem::path path_for(const Id &id) const noexcept { - return this->_layout.get_node_path(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_layout.base_path(); - } - - const disk::Layout &layout() const noexcept { - return this->_layout; - } - -private: - disk::Layout _layout; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/Storage.h b/src/terrainlib/octree/storage/Storage.h deleted file mode 100644 index 782a9c03..00000000 --- a/src/terrainlib/octree/storage/Storage.h +++ /dev/null @@ -1,312 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -#include "mesh/io.h" -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/strategy/Default.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/storage/RawStorage.h" -#include "octree/storage/cache/ICache.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" -#include "octree/storage/helpers.h" - -namespace octree { - -namespace detail { -struct MaybeIndex { - std::optional map; - mutable bool dirty = false; - - explicit MaybeIndex() : map(std::nullopt) {} - explicit MaybeIndex(IndexMap map) : map(std::move(map)) {} - - MaybeIndex(const MaybeIndex &) = default; - MaybeIndex &operator=(const MaybeIndex &) = default; - - MaybeIndex(MaybeIndex &&other) noexcept - : map(std::move(other.map)), dirty(std::exchange(other.dirty, false)) { - other.map.reset(); - } - - MaybeIndex &operator=(MaybeIndex &&other) noexcept { - this->map = std::move(other.map); - this->dirty = std::exchange(other.dirty, false); - other.map.reset(); - return *this; - } - - bool add(const Id &id) noexcept { - if (!map.has_value()) { - return false; - } - - const bool changed = map->add(id); - dirty = dirty || changed; - return changed; - } - - bool remove(const Id &id) noexcept { - if (!map.has_value()) { - return false; - } - - const bool changed = map->remove(id); - dirty = dirty || changed; - return changed; - } - - bool contains(const Id &id, const bool def = false) const noexcept { - if (this->map.has_value()) { - const octree::NodeStatusOrMissing status = this->map->get(id); - return status == octree::NodeStatusOrMissing::Leaf || status == NodeStatusOrMissing::Inner; - } - return def; - } -}; - -template -struct MaybeCache : public cache::ICache { - std::optional>> cache; - - explicit MaybeCache() : cache(std::nullopt) {} - explicit MaybeCache(std::unique_ptr> cache) : cache(std::move(cache)) {} - - std::optional get(const Id& id) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->get(id); - } - return std::nullopt; - } - - bool put(const Id &id, const T &value) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->put(id, value); - } - return false; - } - - bool remove(const Id &id) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->remove(id); - } - return false; - } - - bool contains(const Id &id) const noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->contains(id); - } - return false; - } -}; -} - -struct StorageSettings { - bool allow_overwrite = false; -}; - -template Codec = DefaultCodecFor> -class Storage_ { -public: - using value_type = T; - using codec_type = Codec; - using load_error = typename Codec::load_error; - using save_error = typename Codec::save_error; - - explicit Storage_(RawStorage_ inner) - : _inner(std::move(inner)) {} - explicit Storage_(RawStorage_ inner, IndexMap index) - : _inner(std::move(inner)), _index(detail::MaybeIndex(std::move(index))) {} - - Storage_ &operator=(const Storage_ &) = delete; - Storage_(const Storage_ &) = delete; - Storage_(Storage_ &&) = default; - Storage_ &operator=(Storage_ &&) = default; - - virtual ~Storage_() { - if (this->_index.map.has_value() && this->_index.dirty) { - auto result = helpers::save_index_map(this->_index.map.value(), this->_inner.layout()); - if (!result.has_value()) { - LOG_ERROR("Failed to automatically save index when closing storage: {}", result.error()); - } - } - } - - std::expected load(const Id &id) const noexcept { - if (const auto value_opt = this->_cache.get(id)) { - return value_opt.value(); - } - - if (!this->_index.contains(id, true)) { - return std::unexpected(Codec::file_not_found()); - } - - const auto result = this->_inner.load(id); - if (result.has_value()) { - this->_cache.put(id, result.value()); - } - return result; - } - - std::expected save(const Id &id, const value_type &value) noexcept { - if (this->check_overwrite(id)) { - LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); - } - - const auto result = this->_inner.save(id, value); - if (result.has_value()) { - this->_cache.put(id, value); - this->_index.add(id); - } - return result; - } - - std::expected copy_from(const Id &id, const Storage_ &source) noexcept { - if (!source._index.contains(id, true)) { - return std::unexpected(CopyErrorKind::FileNotFound); - } - - if (this->check_overwrite(id)) { - // TODO: proper error - LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); - } - - const auto result = this->_inner.copy_from(id, source._inner); - if (result.has_value()) { - this->_index.add(id); - } - return result; - } - - std::expected copy_to(const Id &id, Storage_ &target) const noexcept { - return target.copy_from(id, *this); - } - - bool remove(const Id &id) noexcept { - this->_cache.remove(id); - this->_index.remove(id); - return this->_inner.remove(id); - } - - bool has(const Id &id) const noexcept { - if (this->is_indexed()) { - return this->_index.contains(id, false); - } else { - return this->_inner.has(id); - } - } - - std::filesystem::path path_for(const Id &id) const noexcept { - return this->_inner.path_for(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_inner.base_path(); - } - - bool is_indexed() const noexcept { - return this->_index.map.has_value(); - } - - void ensure_indexed() noexcept { - if (this->is_indexed()) { - return; - } - - LOG_TRACE("Index not present, creating empty index"); - IndexMap map; - helpers::update_index_map(map, this->_inner.layout()); - LOG_TRACE("Index created with {} entries", map.size()); - this->_index.map = std::move(map); - } - - std::optional> index() const noexcept { - // I miss Option::as_ref - if (this->_index.map.has_value()) { - return this->_index.map.value(); - } else { - return std::nullopt; - } - } - - std::optional>> &cache() noexcept { - return this->_cache.cache; - } - - std::optional>> cache() const noexcept { - if (this->_cache.cache.has_value()) { - return *this->_cache.cache.value(); - } else { - return std::nullopt; - } - } - - std::expected save_or_create_index() noexcept { - if (this->is_indexed() && !this->_index.dirty) { - return {}; - } - - this->ensure_indexed(); - - auto result = helpers::save_index_map(this->index().value(), this->_inner.layout()); - if (result.has_value()) { - this->_index.dirty = false; - } - return result; - } - - const octree::disk::Layout &layout() const noexcept { - return this->_inner.layout(); - } - - const StorageSettings& settings() const noexcept { - return this->_settings; - } - StorageSettings& settings() noexcept { - return this->_settings; - } - -protected: - bool check_overwrite(const Id &id) noexcept { - return !this->_settings.allow_overwrite && this->has(id); - } - - std::optional &index_mut() noexcept { - return this->_index.map; - } - - bool is_index_dirty() const noexcept { - return this->_index.dirty; - } - void set_index_dirty(const bool val = true) const noexcept { - this->_index.dirty = val; - } - - void update_index() noexcept { - if (!this->is_indexed()) { - return; - } - - helpers::update_index_map(this->index_mut().value(), this->_inner.layout()); - this->_index.dirty = false; - } - -private: - RawStorage_ _inner; - detail::MaybeIndex _index = detail::MaybeIndex(); - mutable detail::MaybeCache _cache = detail::MaybeCache(); - StorageSettings _settings = {}; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/cache/Dummy.h b/src/terrainlib/octree/storage/cache/Dummy.h index ecf6cff8..159bc32d 100644 --- a/src/terrainlib/octree/storage/cache/Dummy.h +++ b/src/terrainlib/octree/storage/cache/Dummy.h @@ -1,27 +1,11 @@ #pragma once -#include - -#include "octree/Id.h" -#include "octree/storage/cache/ICache.h" +#include "octree/StoreTraits.h" +#include "store/cache/Dummy.h" namespace octree::cache { - template -class Dummy : public ICache { -public: - std::optional get(const Id &) noexcept override { - return std::nullopt; - } - bool put(const Id &, const T &) noexcept override { - return false; - } - bool remove(const Id &) noexcept override { - return false; - } - bool contains(const Id &) const noexcept override { - return false; - } -}; +template +using Dummy = store::cache::Dummy; -} +} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/cache/ICache.h b/src/terrainlib/octree/storage/cache/ICache.h index bd0258bc..b14bd0b0 100644 --- a/src/terrainlib/octree/storage/cache/ICache.h +++ b/src/terrainlib/octree/storage/cache/ICache.h @@ -1,20 +1,11 @@ #pragma once -#include - -#include "octree/Id.h" +#include "octree/StoreTraits.h" +#include "store/cache/Interface.h" namespace octree::cache { -template -class ICache { -public: - virtual ~ICache() = default; +template +using ICache = store::cache::Interface; - virtual std::optional get(const Id& id) noexcept = 0; - virtual bool put(const Id &id, const T &value) noexcept = 0; - virtual bool remove(const Id &id) noexcept = 0; - virtual bool contains(const Id &id) const noexcept = 0; -}; - -} +} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/cache/LruCache.h b/src/terrainlib/octree/storage/cache/LruCache.h index 1c705e05..167f19d2 100644 --- a/src/terrainlib/octree/storage/cache/LruCache.h +++ b/src/terrainlib/octree/storage/cache/LruCache.h @@ -1,71 +1,11 @@ #pragma once -#include -#include -#include - -#include "octree/Id.h" -#include "octree/RawStorage.h" -#include "octree/cache/ICache.h" +#include "octree/StoreTraits.h" +#include "store/cache/Lru.h" namespace octree::cache { -// TODO: UNTESTED -// Do not use this cache for NodeLoader's ancestor lookup until cached ancestors -// are clipped to the requested node bounds. NodeLoader currently returns a -// cached ancestor unchanged, producing incorrect geometry for child requests. -template -class Lru_ : public ICache { -public: - explicit Lru_(const size_t capacity) : _capacity(capacity) {} - - std::optional get(const Key& key) { - auto it = this->_map.find(key); - if (it == this->_map.end()) return std::nullopt; - - // Move to front (most recently used) - this->_usage.splice(this->_usage.begin(), this->_usage, it->second.second); - return it->second.first; - } - - void put(const Key& key, const Value& value) { - auto it = this->_map.find(key); - if (it != this->_map.end()) { - // Update value and move to front - it->second.first = value; - this->_usage.splice(this->_usage.begin(), this->_usage, it->second.second); - } else { - // Insert new entry - if (this->_map.size() == _capacity) { - // Evict least recently used - const Key& lru_key = this->_usage.back(); - this->_map.erase(lru_key); - this->_usage.pop_back(); - } - - this->_usage.push_front(key); - this->_map[key] = { value, this->_usage.begin() }; - } - } - - void remove(const Key& key) { - auto it = this->_map.find(key); - if (it != this->_map.end()) { - this->_usage.erase(it->second.second); - this->_map.erase(it); - } - } - - bool contains(const Key& key) const { - return this->_map.contains(key); - } - -private: - size_t _capacity; - std::list _usage; - std::unordered_map::iterator>> _map; -}; - -using Lru = Lru; +template +using Lru = store::cache::Lru; -} +} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/codec/Codec.h b/src/terrainlib/octree/storage/codec/Codec.h deleted file mode 100644 index f31b6f45..00000000 --- a/src/terrainlib/octree/storage/codec/Codec.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -template -concept CodecFor = - requires(const std::filesystem::path &path, const T &value) { - typename Codec::value_type; - typename Codec::load_error; - typename Codec::save_error; - - requires std::same_as; - - { Codec::load_from_path(path) } noexcept - -> std::same_as>; - - { Codec::save_to_path(value, path) } noexcept - -> std::same_as>; - - { Codec::file_not_found() } noexcept - -> std::same_as; - - { Codec::supported_extensions() } -> std::ranges::range; - - requires std::convertible_to< - std::ranges::range_value_t, - std::string_view>; - }; diff --git a/src/terrainlib/octree/storage/codec/MeshCodec.h b/src/terrainlib/octree/storage/codec/MeshCodec.h deleted file mode 100644 index 7a15400a..00000000 --- a/src/terrainlib/octree/storage/codec/MeshCodec.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "mesh/io.h" - -namespace octree { - -struct MeshCodec { - using value_type = mesh::Simple; - using load_error = mesh::io::LoadMeshError; - using save_error = mesh::io::SaveMeshError; - - static std::expected load_from_path(const std::filesystem::path& path) noexcept { - return mesh::io::load_from_path(path); - } - - static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { - return mesh::io::save_to_path(value, path); - } - - static load_error file_not_found() noexcept { - return mesh::io::LoadMeshErrorKind::FileNotFound; - } - - static std::vector supported_extensions() { - return {".terrain", ".glb", ".gltf"}; - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h b/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h deleted file mode 100644 index 3ea68ba9..00000000 --- a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include "log.h" - -namespace octree { - -// Wraps a codec so saving always fails loudly instead of writing, for storages that must stay -// read-only (e.g. a view that aliases another storage's files). -template -struct ReadOnlyCodec : Codec { - static std::expected save_to_path( - const typename Codec::value_type &, const std::filesystem::path &) noexcept { - LOG_ERROR_AND_EXIT("Attempted to write through a read-only codec"); - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h b/src/terrainlib/octree/storage/codec/ZppBitsCodec.h deleted file mode 100644 index dc4acd5a..00000000 --- a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "Codec.h" -#include "io/serialize.h" -#include "io/Error.h" - -namespace octree { - -template -struct ZppBitsCodec { - using value_type = T; - using load_error = io::Error; - using save_error = io::Error; - - static std::expected load_from_path(const std::filesystem::path& path) noexcept { - return io::read_from_path(path); - } - - static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { - return io::write_to_path(value, path); - } - - static load_error file_not_found() noexcept { - return io::Error::Value::OpenFile; - } - - static std::vector supported_extensions() { - return {".bin"}; - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/defaults.h b/src/terrainlib/octree/storage/defaults.h deleted file mode 100644 index fb4981c4..00000000 --- a/src/terrainlib/octree/storage/defaults.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "mesh/SimpleMesh.h" -#include "codec/Codec.h" -#include "codec/MeshCodec.h" -#include "codec/ZppBitsCodec.h" - -namespace octree { - -using DefaultT = mesh::Simple; - -template -struct DefaultCodec { - using type = ZppBitsCodec; -}; - -template <> -struct DefaultCodec { - using type = MeshCodec; -}; - -template -using DefaultCodecFor = typename DefaultCodec>::type; - -} // namespace octree \ No newline at end of file diff --git a/src/terrainlib/octree/storage/helpers.cpp b/src/terrainlib/octree/storage/helpers.cpp deleted file mode 100644 index ed817515..00000000 --- a/src/terrainlib/octree/storage/helpers.cpp +++ /dev/null @@ -1,136 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include "octree/storage/helpers.h" -#include "io/serialize.h" -#include "octree/NodeStatus.h" -#include "octree/IndexMap.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/Strategy.h" -#include "octree/disk/layout/StrategyRegister.h" - -namespace octree::helpers { -namespace { -template -std::optional find_max_key(const std::unordered_map& map) { - if (map.empty()) { - return std::nullopt; - } - - auto max_it = map.begin(); - for (auto it = std::next(max_it); it != map.end(); it++) { - if (it->second > max_it->second) { - max_it = it; - } - } - - return max_it->first; -} -} - -std::optional guess_layout_strategy( - const std::filesystem::path &base_path, - size_t max_files_to_check) { - - if (!std::filesystem::is_directory(base_path)) { - return std::nullopt; - } - - std::vector candidate_paths; - std::unordered_map extension_counters; - for (const auto &entry : std::filesystem::recursive_directory_iterator(base_path)) { - if (max_files_to_check == 0) { - break; - } - if (!entry.is_regular_file() && !entry.is_symlink()) { - continue; - } - - const auto ext = entry.path().extension(); - if (ext == disk::v1::index_extension()) { - continue; - } - extension_counters[ext]++; - - candidate_paths.emplace_back(std::filesystem::relative(entry.path(), base_path)); - max_files_to_check--; - } - std::optional most_common_ext = find_max_key(extension_counters); - - std::unique_ptr best_strategy; - size_t best_match_count = 0; - const auto &factories = disk::layout::StrategyRegister::instance().factories(); - for (const auto &[_, make_strategy] : factories) { - auto strategy = make_strategy(); // assume returns unique_ptr - - size_t match_count = 0; - for (const auto &rel_path : candidate_paths) { - if (strategy->get_id_from_relative_node_path(rel_path).has_value()) { - match_count++; - } - } - - if (match_count > best_match_count) { - best_match_count = match_count; - best_strategy = std::move(strategy); - } - } - - if (best_strategy && most_common_ext.has_value()) { - return LayoutWithoutBase(std::move(best_strategy), most_common_ext.value()); - } - - return std::nullopt; -} - -std::expected save_index_map(const IndexMap& index, const disk::Layout& layout) { - const auto index_path = layout.base_path() / disk::v1::index_file_name(); - LOG_TRACE("Saving octree storage index to {}", index_path); - - disk::v1::IndexFile index_file; - index_file.map = index; - index_file.preferred_extension = layout.extension_with_dot(); - index_file.layout_strategy_id = disk::layout::StrategyRegister::instance().get_id(layout.strategy()); - - const auto result = io::write_to_path(index_file, index_path); - if (!result.has_value()) { - LOG_ERROR("Failed to save octree storage index to {}", index_path); - return std::unexpected(result.error()); - } - return {}; -} - -void update_index_map(IndexMap &index, const disk::Layout &layout) { - index.clear(); - const auto &base_path = layout.base_path(); - - std::filesystem::create_directories(base_path); - - for (const auto &entry : std::filesystem::recursive_directory_iterator(base_path)) { - if (!entry.is_regular_file() && !entry.is_symlink()) { - continue; - } - - const auto ext = entry.path().extension(); - if (ext == disk::v1::index_extension()) { - continue; - } - - auto id_opt = layout.get_id_from_node_path(entry.path()); - if (!id_opt) { - continue; - } - - const Id id = *id_opt; - index.add(id); - } -} - -} diff --git a/src/terrainlib/octree/storage/helpers.h b/src/terrainlib/octree/storage/helpers.h deleted file mode 100644 index 288096cf..00000000 --- a/src/terrainlib/octree/storage/helpers.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "io/Error.h" -#include "octree/IndexMap.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/Strategy.h" - -namespace octree::helpers { - -struct LayoutWithoutBase { - std::unique_ptr strategy; - std::string extension_with_dot; -}; - -std::optional guess_layout_strategy( - const std::filesystem::path &base_path, - size_t max_files_to_check = 100); -std::expected save_index_map(const IndexMap &index, const disk::Layout &layout); -void update_index_map(IndexMap &index, const disk::Layout &layout); - -} diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h index b21ede08..b8f1a6d0 100644 --- a/src/terrainlib/octree/storage/open.h +++ b/src/terrainlib/octree/storage/open.h @@ -1,37 +1,31 @@ #pragma once #include -#include -#include +#include -#include - -#include "io/Error.h" -#include "octree/disk/layout/strategy/Default.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/Storage.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" +#include "mesh/storage.h" +#include "octree/Storage.h" namespace octree { -struct OpenOptions { - std::unique_ptr default_layout_strategy = {}; - std::optional preferred_extension_with_dot = {}; -}; +using OpenOptions = mesh::storage::OpenOptions; + +inline std::expected> open_index( + const std::filesystem::path &path) { + return mesh::storage::open_index(path); +} -template Codec = DefaultCodecFor> -std::expected, io::Error> open_index(const std::filesystem::path &index_path); -template Codec = DefaultCodecFor> -Storage_ open_folder( - const std::filesystem::path &base_path, +inline std::expected> open_folder( + const std::filesystem::path &path, const bool create_index = false, - OpenOptions options = {}); -template Codec = DefaultCodecFor> -IndexedStorage_ open_folder_indexed( - const std::filesystem::path &base_path, - OpenOptions options = {}); + OpenOptions options = {}) { + return mesh::storage::open_folder(path, create_index, std::move(options)); } -#include "open.inl" +inline std::expected> open_folder_indexed( + const std::filesystem::path &path, + OpenOptions options = {}) { + return mesh::storage::open_folder_indexed(path, std::move(options)); +} +} // namespace octree diff --git a/src/terrainlib/octree/storage/open.inl b/src/terrainlib/octree/storage/open.inl deleted file mode 100644 index c60c3699..00000000 --- a/src/terrainlib/octree/storage/open.inl +++ /dev/null @@ -1,111 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include "io/serialize.h" -#include "log.h" -#include "io/Error.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/layout/StrategyRegister.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/helpers.h" -#include "octree/storage/defaults.h" -#include "range_utils.h" - -namespace octree { - -template Codec> -std::expected, io::Error> open_index(const std::filesystem::path &index_path) { - LOG_TRACE("Opening storage index {}", index_path); - - const auto result = io::read_from_path(index_path); - if (!result.has_value()) { - LOG_TRACE("Failed to open storage index due to {}", result.error()); - return std::unexpected(result.error()); - } - auto index_file = result.value(); - LOG_TRACE("Successfully read storage index with {} entries.", index_file.map.size()); - auto index_map = std::move(index_file.map); - const auto base_path = index_path.parent_path(); - auto layout_strategy = disk::layout::StrategyRegister::instance().create(index_file.layout_strategy_id); - disk::Layout layout(base_path, std::move(layout_strategy), index_file.preferred_extension); - RawStorage_ raw_storage(std::move(layout)); - return IndexedStorage_(std::move(raw_storage), std::move(index_map)); -} - -template Codec> -Storage_ open_folder( - const std::filesystem::path &base_path, - bool create_index, - OpenOptions options) { - LOG_TRACE("Opening storage folder {}", base_path); - - if (!std::filesystem::is_directory(base_path)) { - if (std::filesystem::exists(base_path)) { - LOG_ERROR_AND_EXIT("Base path {} exists but is not a directory", base_path); - } - - LOG_TRACE("Base path {} does not exist, creating it", base_path); - std::filesystem::create_directories(base_path); - } - - const std::filesystem::path index_path = base_path / disk::v1::index_file_name(); - auto storage_opt = open_index(index_path); - if (storage_opt.has_value()) { - LOG_TRACE("Loaded existing index"); - return Storage_(std::move(storage_opt.value())); - } - - auto layout_info_opt = helpers::guess_layout_strategy(base_path); - if (layout_info_opt.has_value()) { - const octree::helpers::LayoutWithoutBase &layout_info = layout_info_opt.value(); - LOG_TRACE("Guessed layout strategy of dataset as {} and extension as {}", - disk::layout::StrategyRegister::instance().get_id(*layout_info.strategy), - layout_info.extension_with_dot); - } else { - auto default_layout_strategy = std::move(options.default_layout_strategy); - if (!default_layout_strategy) { - default_layout_strategy = disk::layout::strategy::make_default(); - } - std::string default_extension_with_dot; - const std::vector supported_extensions = Codec::supported_extensions(); - if (options.preferred_extension_with_dot.has_value()) { - ASSERT(contains(supported_extensions, options.preferred_extension_with_dot.value())); - default_extension_with_dot = options.preferred_extension_with_dot.value(); - } else { - default_extension_with_dot = supported_extensions[0]; - } - LOG_WARN("Unable to determine layout of dataset, using layout strategy {} and extension {}", - disk::layout::StrategyRegister::instance().get_id(*default_layout_strategy), - default_extension_with_dot); - layout_info_opt = octree::helpers::LayoutWithoutBase(std::move(default_layout_strategy), default_extension_with_dot); - } - - disk::Layout layout(base_path, std::move(layout_info_opt->strategy), layout_info_opt->extension_with_dot); - if (!create_index) { - return Storage_(RawStorage_(std::move(layout))); - } - - IndexMap map; - helpers::update_index_map(map, layout); - if (!map.empty()) { - const auto save_result = helpers::save_index_map(map, layout); - if (!save_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to create storage index in {}: {}", base_path, save_result.error()); - } - } - return Storage_(RawStorage_(std::move(layout)), std::move(map)); -} - -template Codec> -IndexedStorage_ open_folder_indexed( - const std::filesystem::path &base_path, - OpenOptions options) { - return IndexedStorage_(open_folder(base_path, true, std::move(options))); -} - -} // namespace octree diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h new file mode 100644 index 00000000..64070609 --- /dev/null +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -0,0 +1,196 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "store/open.h" + +namespace octree::storage { + +struct OpenOptions { + std::optional> default_mapping; + std::optional preferred_extension; +}; + +struct DiscoveredLayout { + store::PathMapping mapping; + std::string codec_selector; +}; + +template +std::optional discover_layout( + const std::filesystem::path &base_path, + CodecResolver &resolve_codec) { + struct Candidate { + std::filesystem::path path; + std::string extension; + }; + std::vector candidates; + std::unordered_map extension_counts; + std::error_code error; + for (std::filesystem::recursive_directory_iterator iterator(base_path, error), end; + !error && iterator != end; + iterator.increment(error)) { + if (!iterator->is_regular_file(error) && !iterator->is_symlink(error)) { + if (error) { + break; + } + continue; + } + if (iterator->path().filename() == disk::v1::index_file_name()) { + continue; + } + const std::string extension = iterator->path().extension().string(); + if (extension.empty() || !resolve_codec(extension).has_value()) { + continue; + } + candidates.push_back({iterator->path(), extension}); + ++extension_counts[extension]; + } + if (error || candidates.empty()) { + return std::nullopt; + } + + std::optional> best_mapping; + size_t best_match_count = 0; + for (const auto mapping : store_layout::all()) { + const store::Layout layout(base_path, mapping); + size_t matches = 0; + for (const Candidate &candidate : candidates) { + std::filesystem::path node_path = candidate.path; + node_path.replace_extension(); + matches += layout.key_from_node_path(store::NodePath(node_path)).has_value(); + } + if (matches > best_match_count) { + best_mapping = mapping; + best_match_count = matches; + } + } + if (!best_mapping.has_value()) { + return std::nullopt; + } + + const auto extension = std::max_element( + extension_counts.begin(), + extension_counts.end(), + [](const auto &left, const auto &right) { return left.second < right.second; }); + return DiscoveredLayout{best_mapping.value(), extension->first}; +} + +template +std::expected, store::OpenError> open_folder( + const std::filesystem::path &base_path, + const bool create_index, + std::string default_codec_selector, + CodecResolver resolve_codec, + OpenOptions options = {}) { + std::error_code error; + const bool base_exists = std::filesystem::exists(base_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "exists", + error, + })); + } + if (base_exists && !std::filesystem::is_directory(base_path, error)) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "is_directory", + error ? error : std::make_error_code(std::errc::not_a_directory), + })); + } + if (!base_exists) { + std::filesystem::create_directories(base_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + base_path, + "create_directories", + error, + })); + } + } + + const auto format = index_format(); + const std::filesystem::path index_path = base_path / format.index_filename; + const bool index_exists = std::filesystem::exists(index_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + index_path, + "exists", + error, + })); + } + if (index_exists) { + auto indexed = store::open_index( + index_path, + format, + resolve_codec); + if (!indexed.has_value()) { + return std::unexpected(indexed.error()); + } + return store::Storage(std::move(indexed.value())); + } + + const auto discovered = discover_layout(base_path, resolve_codec); + const store::PathMapping mapping = discovered.has_value() + ? discovered->mapping + : options.default_mapping.value_or(format.default_mapping()); + std::string codec_selector = discovered.has_value() + ? discovered->codec_selector + : options.preferred_extension.value_or(std::move(default_codec_selector)); + auto codec = resolve_codec(codec_selector); + if (!codec.has_value()) { + return std::unexpected(store::OpenError(codec.error())); + } + + auto storage_result = store::make_storage( + base_path, + format, + mapping, + std::move(codec_selector), + std::move(codec.value()), + false); + if (!storage_result.has_value()) { + return std::unexpected(storage_result.error()); + } + auto storage = std::move(storage_result.value()); + if (create_index) { + const auto index_result = storage.save_or_create_index(); + if (!index_result.has_value()) { + return std::unexpected(store::OpenError(index_result.error())); + } + } + return storage; +} + +template +std::expected, store::OpenError> +open_folder_indexed( + const std::filesystem::path &base_path, + std::string default_codec_selector, + CodecResolver resolve_codec, + OpenOptions options = {}) { + auto result = open_folder( + base_path, + true, + std::move(default_codec_selector), + std::move(resolve_codec), + std::move(options)); + if (!result.has_value()) { + return std::unexpected(result.error()); + } + return store::IndexedStorage(std::move(result.value())); +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/traverse.h b/src/terrainlib/octree/traverse.h index 6d0af8db..cc5b9435 100644 --- a/src/terrainlib/octree/traverse.h +++ b/src/terrainlib/octree/traverse.h @@ -6,6 +6,7 @@ #include #include "octree/IndexMap.h" +#include "octree/StoreTraits.h" #include "store/traverse.h" namespace octree { @@ -32,4 +33,20 @@ void traverse( DEBUG_ASSERT(result.has_value()); } +template> +void traverse( + const store::Index &index, + VisitFn &&visit, + RefineFn &&refine = always_refine, + const Id &root = Id::root(), + const TraversalOrder order = TraversalOrder::DepthFirst) { + const auto result = store::traverse( + index, + std::forward(visit), + std::forward(refine), + root, + order); + DEBUG_ASSERT(result.has_value()); +} + } // namespace octree diff --git a/src/terrainlib/store/OpenError.h b/src/terrainlib/store/OpenError.h new file mode 100644 index 00000000..2622b8a5 --- /dev/null +++ b/src/terrainlib/store/OpenError.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "store/IndexFormat.h" +#include "store/StorageError.h" + +namespace store { + +struct UnknownLayout { + std::string id; + bool operator==(const UnknownLayout &) const = default; +}; + +template +using OpenError = std::variant< + IndexFormatError, + FilesystemError, + UnknownLayout, + CodecError, + InvalidKey>; + +} // namespace store diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index fa99b6a4..e3cf3a85 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -18,7 +20,10 @@ template class Storage { public: using Key = typename Traits::Key; + using key_type = Key; using value_type = NodeData; + using load_error = LoadError; + using save_error = SaveError; struct Persistence { IndexFormat format; @@ -29,6 +34,9 @@ class Storage { explicit Storage(RawStorage raw) : _raw(std::move(raw)) {} + Storage(RawStorage raw, Persistence persistence) + : _raw(std::move(raw)), _persistence(std::move(persistence)) {} + Storage( RawStorage raw, Index index, @@ -195,6 +203,17 @@ class Storage { return _raw.paths(key); } + std::expected> path_for( + const Key &key) const { + const auto node_paths = paths(key); + if (!node_paths.has_value()) { + return std::unexpected(node_paths.error()); + } + return node_paths->empty() + ? _raw.layout().node_path(key).path() + : node_paths->front(); + } + const std::filesystem::path &base_path() const { return _raw.layout().base_path(); } @@ -204,6 +223,12 @@ class Storage { const Codec &codec() const { return _raw.codec(); } + std::optional codec_selector() const { + if (!_persistence.has_value()) { + return std::nullopt; + } + return _persistence->codec_selector; + } bool is_indexed() const { return _index.has_value(); @@ -247,6 +272,18 @@ class Storage { return result; } + std::expected save_or_create_index() { + if (!_index.has_value()) { + const auto scan_result = scan_index(); + if (!scan_result.has_value()) { + return std::unexpected(scan_result.error()); + } + _index = std::move(scan_result.value()); + _dirty = true; + } + return save_index(); + } + const StorageSettings &settings() const { return _settings; } @@ -269,6 +306,85 @@ class Storage { } private: + std::expected, IndexFormatError> scan_index() const { + std::error_code error; + std::filesystem::create_directories(base_path(), error); + if (error) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Open, + base_path(), + error.message(), + }); + } + + const NodePath probe("__codec_probe__/node"); + const std::string probe_text = probe.path().generic_string(); + std::vector suffixes; + for (const auto &codec_path : _raw.codec().paths(probe)) { + const std::string text = codec_path.generic_string(); + if (text.starts_with(probe_text)) { + suffixes.push_back(text.substr(probe_text.size())); + } + } + + std::unordered_set candidates; + for (std::filesystem::recursive_directory_iterator iterator(base_path(), error), end; + !error && iterator != end; + iterator.increment(error)) { + if (!iterator->is_regular_file(error) && !iterator->is_symlink(error)) { + if (error) { + break; + } + continue; + } + const std::string path_text = iterator->path().generic_string(); + for (const std::string &suffix : suffixes) { + if (path_text.size() < suffix.size() + || !path_text.ends_with(suffix)) { + continue; + } + const NodePath node_path( + std::filesystem::path(path_text.substr(0, path_text.size() - suffix.size()))); + const auto key = _raw.layout().key_from_node_path(node_path); + if (key.has_value() && Traits::is_valid(key.value())) { + candidates.insert(key.value()); + } + } + } + if (error) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Open, + base_path(), + error.message(), + }); + } + + Index result; + for (const Key &key : candidates) { + const auto present = _raw.has(key); + if (!present.has_value()) { + const FilesystemError &filesystem_error = + std::get(present.error()); + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Open, + filesystem_error.path, + filesystem_error.error.message(), + }); + } + if (present.value()) { + const auto added = result.add(key); + if (!added.has_value()) { + return std::unexpected(IndexFormatError{ + IndexFormatErrorCategory::Malformed, + base_path(), + "codec path resolved to an invalid hierarchy key", + }); + } + } + } + return result; + } + void finalize_displaced_state() noexcept { if (!_dirty || !_index.has_value()) { return; diff --git a/src/terrainlib/store/cache/Dummy.h b/src/terrainlib/store/cache/Dummy.h new file mode 100644 index 00000000..bcb7c7e7 --- /dev/null +++ b/src/terrainlib/store/cache/Dummy.h @@ -0,0 +1,25 @@ +#pragma once + +#include "store/cache/Interface.h" + +namespace store::cache { + +template +class Dummy final : public Interface { +public: + using Key = typename Traits::Key; + std::optional get(const Key &) noexcept override { + return std::nullopt; + } + bool put(const Key &, const NodeData &) noexcept override { + return false; + } + bool remove(const Key &) noexcept override { + return false; + } + bool contains(const Key &) const noexcept override { + return false; + } +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/cache/Interface.h b/src/terrainlib/store/cache/Interface.h new file mode 100644 index 00000000..4ffd070d --- /dev/null +++ b/src/terrainlib/store/cache/Interface.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "store/Traits.h" + +namespace store::cache { + +template +class Interface { +public: + using Key = typename Traits::Key; + virtual ~Interface() = default; + + virtual std::optional get(const Key &key) noexcept = 0; + virtual bool put(const Key &key, const NodeData &value) noexcept = 0; + virtual bool remove(const Key &key) noexcept = 0; + virtual bool contains(const Key &key) const noexcept = 0; +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/cache/Lru.h b/src/terrainlib/store/cache/Lru.h new file mode 100644 index 00000000..0e8d8abf --- /dev/null +++ b/src/terrainlib/store/cache/Lru.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include + +#include "store/cache/Interface.h" + +namespace store::cache { + +template +class Lru final : public Interface { +public: + using Key = typename Traits::Key; + + explicit Lru(const size_t capacity) : _capacity(capacity) {} + + std::optional get(const Key &key) noexcept override { + const auto iterator = _values.find(key); + if (iterator == _values.end()) { + return std::nullopt; + } + _usage.splice(_usage.begin(), _usage, iterator->second.second); + return iterator->second.first; + } + + bool put(const Key &key, const NodeData &value) noexcept override { + const auto iterator = _values.find(key); + if (iterator != _values.end()) { + iterator->second.first = value; + _usage.splice(_usage.begin(), _usage, iterator->second.second); + return false; + } + if (_capacity == 0) { + return false; + } + if (_values.size() == _capacity) { + _values.erase(_usage.back()); + _usage.pop_back(); + } + _usage.push_front(key); + _values.emplace(key, std::pair::iterator>{ + value, + _usage.begin(), + }); + return true; + } + + bool remove(const Key &key) noexcept override { + const auto iterator = _values.find(key); + if (iterator == _values.end()) { + return false; + } + _usage.erase(iterator->second.second); + _values.erase(iterator); + return true; + } + + bool contains(const Key &key) const noexcept override { + return _values.contains(key); + } + +private: + size_t _capacity; + std::list _usage; + std::unordered_map< + Key, + std::pair::iterator>, + typename Traits::Hasher> + _values; +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/describe_error.h b/src/terrainlib/store/describe_error.h new file mode 100644 index 00000000..aa61d2a1 --- /dev/null +++ b/src/terrainlib/store/describe_error.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#include "store/IndexFormat.h" +#include "store/OpenError.h" + +namespace store { + +inline std::string describe_error(const CodecError &error) { + return error.message; +} + +inline std::string describe_error(const IndexFormatError &error) { + return error.path.empty() + ? error.message + : error.path.string() + ": " + error.message; +} + +inline std::string describe_error(const FilesystemError &error) { + return error.operation + " " + error.path.string() + ": " + error.error.message(); +} + +inline std::string describe_error(const UnknownLayout &error) { + return "unknown layout: " + error.id; +} + +inline std::string describe_error(const AlreadyExists &error) { + return "path already exists: " + error.path.string(); +} + +template +std::string describe_error(const InvalidKey &) { + return "invalid hierarchy key"; +} + +template +std::string describe_error(const MissingSource &) { + return "source node is missing"; +} + +template +std::string describe_error(const std::variant &error) { + return std::visit( + [](const auto &value) { return describe_error(value); }, + error); +} + +} // namespace store diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h new file mode 100644 index 00000000..70bf3647 --- /dev/null +++ b/src/terrainlib/store/open.h @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include + +#include + +#include "store/IndexedStorage.h" +#include "store/OpenError.h" + +namespace store { + +template +std::expected, OpenError> open_index( + const std::filesystem::path &index_path, + const IndexFormat format, + CodecResolver &&resolve_codec) { + using Key = typename Traits::Key; + const auto metadata_result = format.read(index_path); + if (!metadata_result.has_value()) { + return std::unexpected(OpenError(metadata_result.error())); + } + IndexMetadata metadata = std::move(metadata_result.value()); + for (const auto &[key, status] : metadata.index) { + static_cast(status); + if (!Traits::is_valid(key)) { + return std::unexpected(OpenError(InvalidKey{key})); + } + } + + const auto mapping = format.mapping_from_id(metadata.layout_id); + if (!mapping.has_value()) { + return std::unexpected(OpenError(UnknownLayout{metadata.layout_id})); + } + auto codec = resolve_codec(metadata.codec_selector); + if (!codec.has_value()) { + return std::unexpected(OpenError(codec.error())); + } + + const std::filesystem::path base_path = index_path.parent_path(); + using Storage = store::Storage; + return IndexedStorage( + RawStorage( + Layout(base_path, mapping.value()), + std::move(codec.value())), + std::move(metadata.index), + typename Storage::Persistence{ + format, + index_path, + std::move(metadata.layout_id), + std::move(metadata.codec_selector), + }); +} + +template +std::expected, OpenError> make_storage( + const std::filesystem::path &base_path, + const IndexFormat format, + const PathMapping mapping, + std::string codec_selector, + std::unique_ptr> codec, + const bool indexed) { + using Key = typename Traits::Key; + std::error_code error; + std::filesystem::create_directories(base_path, error); + if (error) { + return std::unexpected(OpenError(FilesystemError{ + base_path, + "create_directories", + error, + })); + } + + typename Storage::Persistence persistence{ + format, + base_path / format.index_filename, + std::string(mapping.id), + std::move(codec_selector), + }; + RawStorage raw( + Layout(base_path, mapping), + std::move(codec)); + if (indexed) { + return Storage( + std::move(raw), + Index{}, + std::move(persistence)); + } + return Storage(std::move(raw), std::move(persistence)); +} + +} // namespace store diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 1b53b9fa..45a3d351 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -13,6 +13,7 @@ #include "octree/traverse.h" #include "storage.h" #include "store/codec/ZppBits.h" +#include "thread_safe_storage.h" namespace { @@ -63,7 +64,9 @@ TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { }); CHECK(visited == std::vector{root, parent, leaf}); - const auto batch_storage = octree::open_folder_indexed(path); + auto batch_storage_result = dag::storage::open_folder_indexed(path); + REQUIRE(batch_storage_result.has_value()); + const auto batch_storage = std::move(batch_storage_result.value()); const auto batch = batch_storage.load(root); REQUIRE(batch.has_value()); CHECK(batch->metadata.group_assignment == std::vector{0}); @@ -73,7 +76,9 @@ TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { REQUIRE(batch->clustering.clusters.size() == 1); CHECK(batch->clustering.clusters.front().id == 17); - const auto metadata_storage = octree::open_folder_indexed(path); + auto metadata_storage_result = dag::storage::open_metadata_indexed(path); + REQUIRE(metadata_storage_result.has_value()); + const auto metadata_storage = std::move(metadata_storage_result.value()); const auto metadata = metadata_storage.load(root); REQUIRE(metadata.has_value()); CHECK(metadata->group_assignment == batch->metadata.group_assignment); @@ -126,3 +131,46 @@ TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { REQUIRE(operation.get()); } } + +TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") { + const auto batch_codec = dag::codec::from_extension(".bin"); + REQUIRE(batch_codec.has_value()); + CHECK(batch_codec.value()->paths(store::NodePath("node")) + == std::vector{"node.bin"}); + + const auto metadata_codec = dag::codec::metadata_from_extension(".bin"); + REQUIRE(metadata_codec.has_value()); + const auto write_result = metadata_codec.value()->write( + store::NodePath("node"), + dag::NodeMetadata{}); + REQUIRE_FALSE(write_result.has_value()); + CHECK(write_result.error().operation == store::CodecOperation::Write); + CHECK(write_result.error().category == store::CodecErrorCategory::UnsupportedOperation); + + const auto unknown = dag::codec::from_extension(".unknown"); + REQUIRE_FALSE(unknown.has_value()); + CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); +} + +TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") { + const std::filesystem::path fixture = + std::filesystem::path(ALP_TEST_DATA_DIR) + / "raster-store-refactor/dag/0/0/0/0.bin"; + const auto batch = io::read_from_path(fixture); + REQUIRE(batch.has_value()); + + TemporaryDirectory output; + auto storage_result = dag::storage::open_folder_indexed(output.path()); + REQUIRE(storage_result.has_value()); + dag::ThreadSafeStorage synchronized(std::move(storage_result.value())); + REQUIRE(synchronized.save(octree::Id::root(), batch.value()).has_value()); + + auto released = std::move(synchronized).release(); + REQUIRE(released.save_index().has_value()); + CHECK(std::filesystem::is_regular_file(output.path() / "terrain.index")); + + auto reopened_result = dag::storage::open_folder_indexed(output.path()); + REQUIRE(reopened_result.has_value()); + CHECK(reopened_result->has(octree::Id::root()).value()); + CHECK(reopened_result->load(octree::Id::root()).has_value()); +} diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 84e28a85..40a3f61f 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,12 +9,14 @@ #include #include "io/serialize.h" +#include "io/bytes.h" #include "mesh/SimpleMesh.h" #include "octree/Storage.h" #include "octree/disk/IndexFile.h" -#include "octree/disk/layout/strategy/Flat.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" -#include "octree/storage/RawStorage.h" +#include "mesh/codec/from_extension.h" +#include "mesh/storage.h" +#include "octree/store_layout/Mappings.h" +#include "octree/storage/IndexFile.h" #include "octree/storage/open.h" #include "octree/traverse.h" @@ -55,10 +58,21 @@ mesh::Simple triangle_mesh(const double offset) { octree::IndexedStorage make_storage( const std::filesystem::path &path, - std::unique_ptr strategy, + const store::PathMapping mapping, const std::string &extension) { - octree::disk::Layout layout(path, std::move(strategy), extension); - return octree::IndexedStorage(octree::RawStorage_(std::move(layout)), octree::IndexMap{}); + auto codec = mesh::codec::from_extension(extension); + REQUIRE(codec.has_value()); + return octree::IndexedStorage( + store::RawStorage( + store::Layout(path, mapping), + std::move(codec.value())), + store::Index{}, + octree::Storage::Persistence{ + octree::storage::index_format(), + path / "terrain.index", + std::string(mapping.id), + extension, + }); } } // namespace @@ -76,7 +90,9 @@ TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { CHECK(index_file->map.is(octree::NodeStatus::Leaf, root)); CHECK(std::filesystem::is_regular_file(path / "0-0.terrain")); - const octree::IndexedStorage storage = octree::open_folder_indexed(path); + auto storage_result = octree::open_folder_indexed(path); + REQUIRE(storage_result.has_value()); + const octree::IndexedStorage storage = std::move(storage_result.value()); const auto mesh = storage.load(root); REQUIRE(mesh.has_value()); CHECK(mesh->face_count() == 1); @@ -114,14 +130,14 @@ TEST_CASE("pre-refactor layouts round-trip boundary IDs") { octree::Id(octree::Id::max_level(), octree::Id::max_index_on_level(octree::Id::max_level())), }; - octree::disk::layout::strategy::Flat flat; - octree::disk::layout::strategy::LevelAndCoordinateDirectories coordinates; + const auto flat = octree::store_layout::flat(); + const auto coordinates = octree::store_layout::level_and_coordinate_directories(); for (const octree::Id id : ids) { - const auto flat_path = flat.get_relative_node_path(id, ".terrain"); - CHECK(flat.get_id_from_relative_node_path(flat_path) == id); + const auto flat_path = flat.key_to_node_path(id); + CHECK(flat.node_path_to_key(flat_path) == id); - const auto coordinate_path = coordinates.get_relative_node_path(id, ".terrain"); - CHECK(coordinates.get_id_from_relative_node_path(coordinate_path) == id); + const auto coordinate_path = coordinates.key_to_node_path(id); + CHECK(coordinates.node_path_to_key(coordinate_path) == id); } } @@ -130,17 +146,17 @@ TEST_CASE("pre-refactor storage hard-links matching payload formats") { TemporaryDirectory target_directory("hard-link-target"); auto source = make_storage( source_directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".terrain"); auto target = make_storage( target_directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".terrain"); const octree::Id root = octree::Id::root(); REQUIRE(source.save(root, triangle_mesh(0.0)).has_value()); REQUIRE(target.copy_from(root, source).has_value()); - CHECK(std::filesystem::equivalent(source.path_for(root), target.path_for(root))); + CHECK(std::filesystem::equivalent(source.path_for(root).value(), target.path_for(root).value())); REQUIRE(source.save_index().has_value()); REQUIRE(target.save_index().has_value()); } @@ -150,17 +166,19 @@ TEST_CASE("pre-refactor storage re-encodes differing payload formats") { TemporaryDirectory target_directory("reencode-target"); auto source = make_storage( source_directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".terrain"); auto target = make_storage( target_directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".glb"); const octree::Id root = octree::Id::root(); REQUIRE(source.save(root, triangle_mesh(4.0)).has_value()); REQUIRE(target.copy_from(root, source).has_value()); - CHECK_FALSE(std::filesystem::equivalent(source.path_for(root), target.path_for(root))); + CHECK_FALSE(std::filesystem::equivalent( + source.path_for(root).value(), + target.path_for(root).value())); const auto loaded = target.load(root); REQUIRE(loaded.has_value()); CHECK(loaded->face_count() == 1); @@ -172,7 +190,7 @@ TEST_CASE("pre-refactor storage supports enabled overwrites") { TemporaryDirectory directory("overwrite"); auto storage = make_storage( directory.path(), - std::make_unique(), + octree::store_layout::flat(), ".terrain"); storage.settings().allow_overwrite = true; @@ -189,24 +207,75 @@ TEST_CASE("pre-refactor storage supports enabled overwrites") { TEST_CASE("pre-refactor folder opening scans payloads and creates an index") { TemporaryDirectory directory("folder-open"); octree::OpenOptions options; - options.default_layout_strategy = std::make_unique(); - options.preferred_extension_with_dot = ".terrain"; + options.default_mapping = octree::store_layout::flat(); + options.preferred_extension = ".terrain"; { - auto storage = octree::open_folder(directory.path(), false, std::move(options)); + auto storage_result = octree::open_folder(directory.path(), false, std::move(options)); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); CHECK_FALSE(storage.is_indexed()); REQUIRE(storage.save(octree::Id::root(), triangle_mesh(2.0)).has_value()); CHECK_FALSE(std::filesystem::exists(directory.path() / "terrain.index")); } { - const auto storage = octree::open_folder_indexed(directory.path()); + auto storage_result = octree::open_folder_indexed(directory.path()); + REQUIRE(storage_result.has_value()); + const auto storage = std::move(storage_result.value()); CHECK(storage.is_indexed()); - CHECK(storage.has(octree::Id::root())); + CHECK(storage.has(octree::Id::root()).value()); CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); } - const auto reopened = octree::open_index<>(directory.path() / "terrain.index"); + const auto reopened = octree::open_index(directory.path() / "terrain.index"); REQUIRE(reopened.has_value()); - CHECK(reopened->has(octree::Id::root())); + CHECK(reopened->has(octree::Id::root()).value()); +} + +TEST_CASE("mesh codec resolver dispatches every supported legacy selector", "[store][open]") { + for (const std::string extension : {".terrain", ".glb", ".gltf"}) { + const auto codec = mesh::codec::from_extension(extension); + REQUIRE(codec.has_value()); + const auto paths = codec.value()->paths(store::NodePath("node")); + REQUIRE_FALSE(paths.empty()); + CHECK(paths.front().extension() == extension); + } + + const auto unknown = mesh::codec::from_extension(".unknown"); + REQUIRE_FALSE(unknown.has_value()); + CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); + CHECK(unknown.error().operation == store::CodecOperation::Resolve); +} + +TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") { + TemporaryDirectory directory("open-errors"); + const std::filesystem::path index_path = directory.path() / "terrain.index"; + + octree::disk::v1::IndexFile index_file; + index_file.layout_strategy_id = "unknown-layout"; + index_file.preferred_extension = ".terrain"; + REQUIRE(io::write_to_path(index_file, index_path).has_value()); + + const auto unknown_layout = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(unknown_layout.has_value()); + REQUIRE(std::holds_alternative(unknown_layout.error())); + CHECK(std::get(unknown_layout.error()).id == "unknown-layout"); + + index_file.layout_strategy_id = "flat"; + index_file.preferred_extension = ".unknown"; + REQUIRE(io::write_to_path(index_file, index_path).has_value()); + const auto unknown_codec = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(unknown_codec.has_value()); + REQUIRE(std::holds_alternative(unknown_codec.error())); + CHECK(std::get(unknown_codec.error()).category + == store::CodecErrorCategory::UnsupportedCodec); + + const std::array malformed_bytes{0xff, 0x00, 0x01}; + REQUIRE(io::write_bytes_to_path(malformed_bytes, index_path).has_value()); + const auto malformed = octree::open_folder_indexed(directory.path()); + REQUIRE_FALSE(malformed.has_value()); + REQUIRE(std::holds_alternative(malformed.error())); + CHECK(std::get(malformed.error()).category + == store::IndexFormatErrorCategory::Malformed); } From 30863369c92be2967867a6144a9784b36cf882b7 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:29:56 +0200 Subject: [PATCH 08/43] Harden SF storage boundaries --- docs/raster-store/refactor-status.md | 45 ++- src/dag_builder/build.cpp | 16 +- src/dag_builder/build.h | 6 +- src/dag_builder/main.cpp | 11 +- src/sf_builder/main.cpp | 9 +- src/sf_builder/terrainbuilder.cpp | 15 +- src/sf_builder/terrainbuilder.h | 7 +- src/sf_merger/NodeWriter.h | 45 ++- src/sf_merger/cut.h | 78 ++++-- src/sf_merger/main.cpp | 19 +- src/sf_merger/merge.h | 86 ++++-- src/terrainlib/sf/Error.h | 42 +++ src/terrainlib/sf/InvalidTopology.h | 13 + src/terrainlib/sf/finalize_storage.h | 28 ++ src/terrainlib/sf/validate_index.h | 21 ++ src/terrainlib/store/RawStorage.h | 25 ++ src/terrainlib/store/Storage.h | 10 +- unittests/CMakeLists.txt | 11 + .../dag_builder/storage_compatibility.cpp | 34 +++ unittests/sf_builder/storage_finalization.cpp | 80 ++++++ unittests/sf_merger/storage_boundaries.cpp | 196 +++++++++++++ unittests/terrainlib/sf_validate_index.cpp | 24 ++ unittests/terrainlib/store_storage.cpp | 265 +++++++++++++++++- 23 files changed, 998 insertions(+), 88 deletions(-) create mode 100644 src/terrainlib/sf/Error.h create mode 100644 src/terrainlib/sf/InvalidTopology.h create mode 100644 src/terrainlib/sf/finalize_storage.h create mode 100644 src/terrainlib/sf/validate_index.h create mode 100644 unittests/sf_builder/storage_finalization.cpp create mode 100644 unittests/sf_merger/storage_boundaries.cpp create mode 100644 unittests/terrainlib/sf_validate_index.cpp diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 1bfd3b4a..132225d9 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 3 — Generalize storage and index lifecycle +- Phase: 4 — Harden node reuse and enforce SF topology - State: complete; completion commit pending. -- Next action: commit and tag Phase 3, then begin Phase 4 copy hardening and - SF topology validation. +- Next action: commit and tag Phase 4, then begin Phase 5 cleanup and + documentation. ## Phase log @@ -91,6 +91,29 @@ This file is the resumable implementation log for index no-fallback, and DAG synchronized move/release coverage. - State: complete. +### Phase 4 — Harden node reuse and enforce SF topology + +- Hardened `copy_from()` around the fixed codec probe path: matching path + lists hard-link every physical file, differing lists decode/re-encode, and + actual path-count mismatches return a typed codec error. +- Moved overwrite index removal to the mutation boundary. Missing-source and + decode failures leave the old logical target indexed; encode and partial + multi-file hard-link failures leave it unindexed for safety. +- Added one-file and multi-file hard-link, re-encode, overwrite, + missing-source, decode, encode, and partial-hard-link failure coverage. +- Added `sf::InvalidTopology`, `sf::validate_index()`, typed SF processing and + finalization errors, and an SF finalization boundary that writes the index + before validating it. +- Applied SF validation before SF merge/cut and DAG processing and after SF + builder/merger output finalization. The diagnostic index browser remains + exempt. +- Propagated SF merger save, copy, open, index, and validation failures through + `std::expected` to the command-line boundary. +- Added SF builder, SF merger, SF cut, and DAG builder boundary tests, + including invalid-input rejection, inspectable invalid output, unchanged + hard links, and newly written changed/clipped nodes. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -130,6 +153,20 @@ This file is the resumable implementation log for - Phase 3 all-application configuration built `sf-builder`, `sf-merger`, `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. - Phase 3 `git diff --check`: passed. +- Phase 4 focused storage-copy tests: 70 assertions in 5 test cases passed. +- Phase 4 SF validator tests: 6 assertions in 2 test cases passed. +- Phase 4 DAG SF-input boundary test: 12 assertions passed. +- Phase 4 SF merger boundary tests: 66 assertions in 4 test cases passed. +- Phase 4 SF builder finalization tests: 19 assertions in 2 test cases passed. +- Phase 4 `unittests_terrainlib`: 24,049 assertions in 429 test cases passed. +- Phase 4 `unittests_dagbuilder`: 471 assertions in 74 test cases passed. +- Phase 4 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 4 all-application configuration built `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. +- The pre-existing `unittests_sfbuilder` compile failure involving removed + `radix::tile::Scheme` remains isolated; the new finalization tests build and + pass in `unittests_sfbuilder_finalization`. +- Phase 4 `git diff --check`: passed. ## Commit and tag log @@ -145,3 +182,5 @@ This file is the resumable implementation log for - `fbc02d0` / `refactor_phase_2` — runtime layouts/codecs and Phase 2 completion tag. - `61acb74` — Phase 3 shared runtime-codec storage core. +- `4ffcdea` / `refactor_phase_3` — Phase 3 application migration and + completion tag. diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 203dca83..7653df1b 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -41,6 +41,7 @@ #include "vertex_lock.h" #include "parallel.h" #include "ContinuationMode.h" +#include "sf/validate_index.h" namespace dag { @@ -584,11 +585,15 @@ std::unordered_set build_level( // Builds the DAG from input_storage into output_storage, restricted to levels within level_range. // Iterates octree levels from finest to coarsest, simplifying and re-clustering geometry at each // level from its children. -void build_levels( +std::expected build_levels( const octree::IndexedMeshStorage &input_storage, octree::IndexedDagStorage &output_storage, const BuildOptions &options, const AnyRange &level_range) { + const auto validation = sf::validate_index(input_storage.index()); + if (!validation.has_value()) { + return std::unexpected(validation.error()); + } const octree::OddLevelShifted shifted_space = octree::OddLevelShifted::earth(); const octree::Space space = octree::Space::earth(); const octree::Id root_node = options.root_node; @@ -599,7 +604,7 @@ void build_levels( auto max_input_level_opt = find_max_input_level(input_by_level); if (!max_input_level_opt.has_value()) { LOG_WARN("No input nodes found for root {}", root_node); - return; + return {}; } const uint32_t max_input_level = max_input_level_opt.value(); @@ -607,7 +612,7 @@ void build_levels( const Range range = valid_range.intersect(level_range).to_range(max_input_level + 1); if (range.is_empty()) { LOG_WARN("Requested level range does not overlap with buildable levels {}-{}", root_node.level(), max_input_level); - return; + return {}; } // Seed prev_level_built with any already-built nodes one level finer than the first level. @@ -638,14 +643,15 @@ void build_levels( } output_storage = std::move(ctx.output_storage).release(); + return {}; } // Builds the complete DAG from input_storage into output_storage. -void build_full( +std::expected build_full( const octree::IndexedMeshStorage &input_storage, octree::IndexedDagStorage &output_storage, const BuildOptions &options) { - build_levels(input_storage, output_storage, options, RangeFull{}); + return build_levels(input_storage, output_storage, options, RangeFull{}); } } // namespace dag diff --git a/src/dag_builder/build.h b/src/dag_builder/build.h index d39491d6..b87528e7 100644 --- a/src/dag_builder/build.h +++ b/src/dag_builder/build.h @@ -2,6 +2,7 @@ #include #include +#include #include "ContinuationMode.h" #include "Range.h" @@ -9,6 +10,7 @@ #include "texturing.h" #include "octree/storage/MeshStorage.h" #include "storage.h" +#include "sf/InvalidTopology.h" namespace dag { @@ -30,12 +32,12 @@ struct BuildOptions { ContinuationMode continuation_mode = ContinuationMode::Error; }; -void build_full( +std::expected build_full( const octree::IndexedMeshStorage &input_storage, octree::IndexedDagStorage &output_storage, const BuildOptions &options); -void build_levels( +std::expected build_levels( const octree::IndexedMeshStorage &input_storage, octree::IndexedDagStorage &output_storage, const BuildOptions &options, diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index 495bbf09..4fb1d70a 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -8,6 +8,7 @@ #include "octree/storage/open.h" #include "storage.h" #include "store/describe_error.h" +#include "sf/Error.h" #include "ContinuationMode.h" int main(int argc, char **argv) { @@ -58,7 +59,15 @@ int main(int argc, char **argv) { options.relative_target_error = 0.001f; } - dag::build_levels(input_storage, output_storage, options, args.level_range); + const auto build_result = dag::build_levels( + input_storage, + output_storage, + options, + args.level_range); + if (!build_result.has_value()) { + LOG_ERROR("Invalid Structura Fundamentalis input: {}", sf::describe_error(build_result.error())); + return EXIT_FAILURE; + } const auto index_result = output_storage.save_index(); if (!index_result.has_value()) { LOG_ERROR( diff --git a/src/sf_builder/main.cpp b/src/sf_builder/main.cpp index e7216346..6bef9d68 100644 --- a/src/sf_builder/main.cpp +++ b/src/sf_builder/main.cpp @@ -18,6 +18,7 @@ #include "terrainbuilder.h" #include "octree/Space.h" #include "tile_provider.h" +#include "sf/Error.h" #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" @@ -317,7 +318,7 @@ int run(std::span args) { const auto maybe_s = actual_num_threads == 1 ? "" : "s"; LOG_INFO("Using {} thread{} for batch processing.", actual_num_threads, maybe_s); - terrainbuilder::build_all_patches( + const auto build_result = terrainbuilder::build_all_patches( dataset, target_level, texture_srs, @@ -326,6 +327,12 @@ int run(std::span args) { output_base_path, output_format, overwrite_existing); + if (!build_result.has_value()) { + LOG_ERROR( + "Failed to finalize Structura Fundamentalis output: {}", + sf::describe_error(build_result.error())); + return 1; + } } return 0; diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index a5780999..438da793 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -45,6 +45,7 @@ #include "octree/storage/open.h" #include "octree/utils.h" #include "store/describe_error.h" +#include "sf/finalize_storage.h" namespace terrainbuilder { @@ -173,7 +174,11 @@ T expect(const std::optional &opt, const std::string &msg) { } } -void build_all_patches( +std::expected finalize_storage(octree::Storage &storage) { + return sf::finalize_storage(storage); +} + +std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, @@ -347,12 +352,6 @@ void build_all_patches( LOG_ERROR_AND_EXIT("Failed to build all terrain patches"); } - const auto index_result = storage.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT( - "Failed to save output index in {}: {}", - storage.base_path(), - store::describe_error(index_result.error())); - } + return finalize_storage(storage); } } diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index b4c0f581..35d51b29 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -8,6 +9,8 @@ #include "octree/Id.h" #include "mesh/SimpleMesh.h" #include "tile_provider.h" +#include "octree/Storage.h" +#include "sf/Error.h" namespace terrainbuilder { @@ -28,7 +31,9 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -void build_all_patches( +std::expected finalize_storage(octree::Storage &storage); + +std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 6c5bf290..9c51359e 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -1,5 +1,8 @@ #pragma once +#include +#include + #include #include "NodeLoader.h" @@ -14,34 +17,58 @@ class NodeWriter { public: NodeWriter(octree::Storage &storage) : _storage(storage) {} - bool has_node(const octree::Id &id) { - return DEBUG_ASSERT_VAL(this->_storage.has(id)).value(); + std::expected> has_node( + const octree::Id &id) { + return this->_storage.has(id); } - void write_node(const octree::Id &id, const SimpleMesh &mesh) { + std::expected> write_node( + const octree::Id &id, + const SimpleMesh &mesh) { mesh::validate(mesh); - DEBUG_ASSERT_VAL(this->_storage.save(id, mesh)); - auto p = DEBUG_ASSERT_VAL(this->_storage.path_for(id)).value(); + const auto save_result = this->_storage.save(id, mesh); + if (!save_result.has_value()) { + return save_result; + } + const auto path_result = this->_storage.path_for(id); + if (!path_result.has_value()) { + return std::unexpected(store::SaveError(path_result.error())); + } + auto p = path_result.value(); // change extension to .png p.replace_extension(".png"); - cv::imwrite(p, mesh.texture.value_or(cv::Mat())); + if (mesh.texture.has_value()) { + cv::imwrite(p, mesh.texture.value()); + } + return {}; } - void copy_subtree_to_output( + std::expected> copy_subtree_to_output( const octree::Id &id, const NodeLoader &loader) { + std::optional> error; octree::traverse( loader.storage().index(), [&](const octree::Id &child_id, const octree::NodeStatus &status) { + if (error.has_value()) { + return; + } if (status == octree::NodeStatus::Virtual) { return; } DEBUG_ASSERT(status == octree::NodeStatus::Leaf); - DEBUG_ASSERT_VAL(this->_storage.copy_from(child_id, loader.storage())); + const auto result = this->_storage.copy_from(child_id, loader.storage()); + if (!result.has_value()) { + error = result.error(); + } }, - octree::always_refine, + [&](const octree::Id &) { return !error.has_value(); }, id); + if (error.has_value()) { + return std::unexpected(std::move(error.value())); + } + return {}; } private: diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 1e785618..3c6ce4d6 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include "mesh/SimpleMesh.h" @@ -14,6 +16,9 @@ #include "utils.h" #include "containers/Cow.h" #include "store/describe_error.h" +#include "sf/Error.h" +#include "sf/finalize_storage.h" +#include "sf/validate_index.h" struct Context { const octree::IndexedStorage& input; @@ -22,12 +27,12 @@ struct Context { const bool keep_inside; }; -void cut_node( +std::expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask); -inline void cut_leaf_node( +inline std::expected cut_leaf_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -41,20 +46,27 @@ inline void cut_leaf_node( const Cow clipped = clip_on_mask(mesh, mask, ctx.keep_inside); if (clipped.is_ref()) { LOG_TRACE("Mesh was fully inside the mask"); - DEBUG_ASSERT_VAL(ctx.output.copy_from(id, ctx.input)); + const auto copy_result = ctx.output.copy_from(id, ctx.input); + if (!copy_result.has_value()) { + return std::unexpected(sf::ProcessingError(copy_result.error())); + } } else { const SimpleMesh &clipped_mesh = clipped; if (!clipped_mesh.is_empty()) { LOG_TRACE("Mesh was clipped from {} vertices and {} triangles to {} vertices and {} triangles", mesh.vertex_count(), mesh.face_count(), clipped_mesh.vertex_count(), clipped_mesh.face_count()); - DEBUG_ASSERT_VAL(ctx.output.save(id, clipped_mesh)); + const auto save_result = ctx.output.save(id, clipped_mesh); + if (!save_result.has_value()) { + return std::unexpected(sf::ProcessingError(save_result.error())); + } } else { LOG_TRACE("Mesh was fully outside the mask"); } } + return {}; } -inline void cut_virtual_node( +inline std::expected cut_virtual_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -69,18 +81,22 @@ inline void cut_virtual_node( const auto child_bounds = geometry::pad_bounds_relative(ctx.space.get_node_bounds(child_id), 0.125); LOG_TRACE("Clipping mask for {}", child_id); const auto child_mask = MeshMask(mesh::clip_on_bounds_and_cap(mask.mesh, child_bounds)); - cut_node(ctx, child_id, child_mask); + const auto child_result = cut_node(ctx, child_id, child_mask); + if (!child_result.has_value()) { + return child_result; + } } + return {}; } -inline void cut_node( +inline std::expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask ) { if (ctx.keep_inside && mask.mesh.is_empty()) { LOG_TRACE("Mesh was fully outside the mask"); - return; + return {}; } /* @@ -94,40 +110,46 @@ inline void cut_node( const auto status_result = ctx.input.index().get(id); DEBUG_ASSERT(status_result.has_value()); if (!status_result->has_value()) { - return; + return {}; } const octree::NodeStatus status = status_result->value(); switch (status) { case octree::NodeStatus::Virtual: - cut_virtual_node(ctx, id, mask); - break; + return cut_virtual_node(ctx, id, mask); case octree::NodeStatus::Leaf: - cut_leaf_node(ctx, id, mask); - break; + return cut_leaf_node(ctx, id, mask); default: UNREACHABLE(); - break; + return {}; } + return {}; } -inline void cut_dataset( +inline std::expected cut_dataset( const octree::IndexedStorage &input, const MeshMask& mask, octree::Storage &output, const bool keep_inside) { + const auto validation = sf::validate_index(input.index()); + if (!validation.has_value()) { + return std::unexpected(sf::ProcessingError(validation.error())); + } Context ctx(input, output, octree::Space::earth(), keep_inside); - cut_node(ctx, octree::Id::root(), mask); - const auto index_result = output.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT( - "Failed to save output index in {}: {}", - output.base_path(), - store::describe_error(index_result.error())); + const auto cut_result = cut_node(ctx, octree::Id::root(), mask); + if (!cut_result.has_value()) { + return cut_result; + } + const auto finalization = sf::finalize_storage(output); + if (!finalization.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> sf::ProcessingError { return error; }, + finalization.error())); } + return {}; } -inline void cut_dataset( +inline std::expected cut_dataset( const octree::IndexedStorage &input_dataset, const MeshMask& mask, const std::filesystem::path &output_path, @@ -140,15 +162,13 @@ inline void cut_dataset( options.preferred_extension = ".glb"; auto output_result = octree::open_folder_indexed(output_path, std::move(options)); if (!output_result.has_value()) { - LOG_ERROR_AND_EXIT( - "Failed to open output dataset {}: {}", - output_path, - store::describe_error(output_result.error())); + return std::unexpected(sf::ProcessingError(output_result.error())); } octree::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { - LOG_ERROR_AND_EXIT("Output dataset has to be empty."); + return std::unexpected(sf::ProcessingError(store::SaveError( + store::AlreadyExists{output_path}))); } - cut_dataset(input_dataset, mask, output_dataset, keep_inside); + return cut_dataset(input_dataset, mask, output_dataset, keep_inside); } diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 76894680..bae30669 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -12,6 +12,7 @@ #include "optional_utils.h" #include "earth.h" #include "store/describe_error.h" +#include "sf/Error.h" std::optional load_mask_from_path(const std::filesystem::path& path) { if (std::filesystem::exists(path)) { @@ -70,7 +71,14 @@ void run(const cli::MergeArgs& args) { std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); - return merge_datasets(base_dataset, new_dataset, output_dataset, mask); + const auto merge_result = merge_datasets( + base_dataset, + new_dataset, + output_dataset, + mask); + if (!merge_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to merge datasets: {}", sf::describe_error(merge_result.error())); + } } void run(const cli::CutArgs& args) { @@ -84,7 +92,14 @@ void run(const cli::CutArgs& args) { } const octree::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); - cut_dataset(input_dataset, mask, args.output_path, args.keep_inside); + const auto cut_result = cut_dataset( + input_dataset, + mask, + args.output_path, + args.keep_inside); + if (!cut_result.has_value()) { + LOG_ERROR_AND_EXIT("Failed to cut dataset: {}", sf::describe_error(cut_result.error())); + } } void run(const cli::Args &args) { diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index b84fe38d..6987d14d 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -15,6 +16,9 @@ #include "octree/NodeStatusOrMissing.h" #include "octree/storage/cache/Dummy.h" #include "store/describe_error.h" +#include "sf/Error.h" +#include "sf/finalize_storage.h" +#include "sf/validate_index.h" inline std::string get_dataset_name(const octree::Storage &storage) { return storage.layout().base_path().filename().string(); @@ -33,6 +37,7 @@ class Merger { using Status = octree::NodeStatusOrMissing; using Context = Visitor::Context; using Result = merge::Result; + using Expected = std::expected; Merger( Visitor& visitor, @@ -41,62 +46,86 @@ class Merger { NodeWriter output) : _visitor(visitor), _left(left), _right(right), _output(output) { } - void merge_root() { + Expected merge_root() { const octree::Id id = octree::Id::root(); const Context ctx = this->_visitor.make_root_context(); - merge_node(id, ctx); + return merge_node(id, ctx); } - void merge_node(const octree::Id &id, const Context& ctx) { + Expected merge_node(const octree::Id &id, const Context& ctx) { const Status left_status = this->_left.get_status(id); const Status right_status = this->_right.get_status(id); return this->merge_node(id, left_status, right_status, ctx); } - void merge_node( + Expected merge_node( const octree::Id &id, const Status left_status, const Status right_status, const Context& ctx ) { LOG_DEBUG("[{}] Start merging (left = {}, right = {})", id, left_status, right_status); - if (this->_output.has_node(id)) { + const auto has_result = this->_output.has_node(id); + if (!has_result.has_value()) { + return std::unexpected(sf::ProcessingError(has_result.error())); + } + if (has_result.value()) { LOG_DEBUG("[{}] Already merged, skipping...", id); - return; // Already merged previously + return {}; // Already merged previously } const auto merge_result = this->call_merge(id, left_status, right_status, ctx); - std::visit([&](const auto &result) { + return std::visit([&](const auto &result) -> Expected { using Result = std::decay_t; if constexpr (std::is_same_v>) { LOG_DEBUG("[{}] needs recursion", id); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); for (const auto &child_id : children) { - this->merge_node(child_id, result.context); + const auto child_result = this->merge_node(child_id, result.context); + if (!child_result.has_value()) { + return child_result; + } } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] remains unchanged (same as {})", id, (result.source == merge::Source::Left ? "left" : "right")); // If the node should remain unchanged, but its not present on disk we cannot copy it. if (result.source == merge::Source::Left && left_status == Status::Missing) { auto mesh_opt = this->_left.load_node(id); if (mesh_opt.has_value()) { - this->_output.write_node(id, mesh_opt.value()); + const auto write_result = this->_output.write_node(id, mesh_opt.value()); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } } } else if (result.source == merge::Source::Right && right_status == Status::Missing) { auto mesh_opt = this->_right.load_node(id); if (mesh_opt.has_value()) { - this->_output.write_node(id, mesh_opt.value()); + const auto write_result = this->_output.write_node(id, mesh_opt.value()); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } } } else { - this->_output.copy_subtree_to_output(id, result.source == merge::Source::Left ? this->_left : this->_right); + const auto copy_result = this->_output.copy_subtree_to_output( + id, + result.source == merge::Source::Left ? this->_left : this->_right); + if (!copy_result.has_value()) { + return std::unexpected(sf::ProcessingError(copy_result.error())); + } } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was merged", id); - this->_output.write_node(id, result.mesh); + const auto write_result = this->_output.write_node(id, result.mesh); + if (!write_result.has_value()) { + return std::unexpected(sf::ProcessingError(write_result.error())); + } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was ignored", id); - // do nothing + return {}; } }, merge_result); } @@ -152,11 +181,18 @@ class Merger { NodeWriter _output; }; -inline void merge_datasets( +inline std::expected merge_datasets( const octree::IndexedStorage &left_dataset, const octree::IndexedStorage &right_dataset, octree::Storage &output_dataset, const std::optional mask = std::nullopt) { + for (const octree::IndexedStorage *input : {&left_dataset, &right_dataset}) { + const auto validation = sf::validate_index(input->index()); + if (!validation.has_value()) { + return std::unexpected(sf::ProcessingError(validation.error())); + } + } + LOG_TRACE("Merging {} and {} into {}", get_dataset_name(left_dataset), get_dataset_name(right_dataset), @@ -171,18 +207,24 @@ inline void merge_datasets( if (mask.has_value()) { merge::visitor::Masked visitor {mask.value(), space}; Merger merger(visitor, left, right, output); - merger.merge_root(); + const auto result = merger.merge_root(); + if (!result.has_value()) { + return result; + } } else { merge::visitor::Simple visitor; Merger merger(visitor, left, right, output); - merger.merge_root(); + const auto result = merger.merge_root(); + if (!result.has_value()) { + return result; + } } - const auto index_result = output_dataset.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT( - "Failed to save output index in {}: {}", - output_dataset.base_path(), - store::describe_error(index_result.error())); + const auto finalization = sf::finalize_storage(output_dataset); + if (!finalization.has_value()) { + return std::unexpected(std::visit( + [](const auto &error) -> sf::ProcessingError { return error; }, + finalization.error())); } + return {}; } diff --git a/src/terrainlib/sf/Error.h b/src/terrainlib/sf/Error.h new file mode 100644 index 00000000..a8274c72 --- /dev/null +++ b/src/terrainlib/sf/Error.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +#include "octree/Id.h" +#include "sf/InvalidTopology.h" +#include "store/OpenError.h" +#include "store/describe_error.h" + +namespace sf { + +using FinalizeError = std::variant; + +using ProcessingError = std::variant< + InvalidTopology, + store::OpenError, + store::FileOperationError, + store::SaveError, + store::CopyError, + store::IndexFormatError>; + +inline std::string describe_error(const InvalidTopology &error) { + return "Structura Fundamentalis topology contains Inner node " + + error.key.to_string(); +} + +template +std::string describe_error(const std::variant &error) { + return std::visit( + [](const auto &value) -> std::string { + if constexpr (std::is_same_v, InvalidTopology>) { + return describe_error(value); + } else { + return store::describe_error(value); + } + }, + error); +} + +} // namespace sf diff --git a/src/terrainlib/sf/InvalidTopology.h b/src/terrainlib/sf/InvalidTopology.h new file mode 100644 index 00000000..643d769d --- /dev/null +++ b/src/terrainlib/sf/InvalidTopology.h @@ -0,0 +1,13 @@ +#pragma once + +#include "octree/Id.h" + +namespace sf { + +struct InvalidTopology { + octree::Id key; + + bool operator==(const InvalidTopology &) const = default; +}; + +} // namespace sf diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h new file mode 100644 index 00000000..8ee9a06c --- /dev/null +++ b/src/terrainlib/sf/finalize_storage.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include + +#include "octree/Storage.h" +#include "sf/Error.h" +#include "sf/validate_index.h" + +namespace sf { + +inline std::expected finalize_storage(octree::Storage &storage) { + const auto save_result = storage.save_or_create_index(); + if (!save_result.has_value()) { + return std::unexpected(FinalizeError(save_result.error())); + } + + const auto index = storage.index(); + DEBUG_ASSERT(index.has_value()); + const auto validation = validate_index(index->get()); + if (!validation.has_value()) { + return std::unexpected(FinalizeError(validation.error())); + } + return {}; +} + +} // namespace sf diff --git a/src/terrainlib/sf/validate_index.h b/src/terrainlib/sf/validate_index.h new file mode 100644 index 00000000..1cf33719 --- /dev/null +++ b/src/terrainlib/sf/validate_index.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "octree/StoreTraits.h" +#include "sf/InvalidTopology.h" +#include "store/Index.h" + +namespace sf { + +inline std::expected validate_index( + const store::Index &index) { + for (const auto &[key, status] : index) { + if (status == store::NodeStatus::Inner) { + return std::unexpected(InvalidTopology{key}); + } + } + return {}; +} + +} // namespace sf diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index 487a3e97..c4bb1abc 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -108,6 +108,16 @@ class RawStorage { std::expected> copy_from( const Key &key, const RawStorage &source) const { + return copy_from(key, source, []() -> std::expected> { + return {}; + }); + } + + template + std::expected> copy_from( + const Key &key, + const RawStorage &source, + BeforeModify &&before_modify) const { if (!Traits::is_valid(key)) { return std::unexpected(CopyError(InvalidKey{key})); } @@ -127,6 +137,10 @@ class RawStorage { if (!loaded.has_value()) { return std::unexpected(CopyError(loaded.error())); } + const auto prepared = before_modify(); + if (!prepared.has_value()) { + return prepared; + } const auto written = _codec->write(_layout.node_path(key), loaded.value()); if (!written.has_value()) { return std::unexpected(CopyError(written.error())); @@ -136,6 +150,17 @@ class RawStorage { const auto source_paths = source._codec->paths(source._layout.node_path(key)); const auto target_paths = _codec->paths(_layout.node_path(key)); + if (source_paths.size() != target_paths.size()) { + return std::unexpected(CopyError(CodecError{ + CodecOperation::Write, + CodecErrorCategory::Domain, + "matching codec probes produced different actual path counts", + })); + } + const auto prepared = before_modify(); + if (!prepared.has_value()) { + return prepared; + } for (size_t index = 0; index < source_paths.size(); ++index) { std::error_code error; std::filesystem::remove(target_paths[index], error); diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index e3cf3a85..e6a4f211 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -147,15 +147,19 @@ class Storage { node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), })); } - if (exists.value() && _index.has_value()) { + const auto prepare_target = [&]() -> std::expected> { + if (!exists.value() || !_index.has_value()) { + return {}; + } const auto removed = _index->remove(key); if (!removed.has_value()) { return std::unexpected(CopyError(removed.error())); } _dirty = _dirty || removed.value(); - } + return {}; + }; - const auto result = _raw.copy_from(key, source._raw); + const auto result = _raw.copy_from(key, source._raw, prepare_target); if (!result.has_value()) { return result; } diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 26db9b5d..3cb01c33 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -29,6 +29,7 @@ add_executable(unittests_terrainlib terrainlib/hybrid_vector.cpp terrainlib/offset_table.cpp terrainlib/segmented_buffer.cpp + terrainlib/sf_validate_index.cpp terrainlib/stamp_set.cpp terrainlib/store_compatibility.cpp terrainlib/store_codec.cpp @@ -97,6 +98,15 @@ if(TARGET sfbuilderlib) ) target_link_libraries(unittests_sfbuilder PRIVATE sfbuilderlib Catch2::Catch2WithMain) atb_configure_test(unittests_sfbuilder) + + add_executable(unittests_sfbuilder_finalization + catch2_helpers.h + sf_builder/storage_finalization.cpp + ) + target_link_libraries( + unittests_sfbuilder_finalization + PRIVATE sfbuilderlib Catch2::Catch2WithMain) + atb_configure_test(unittests_sfbuilder_finalization) endif() if(TARGET tile-downloader) @@ -137,6 +147,7 @@ if(TARGET sfmergerlib) catch2_helpers.h sf_merger/mask.cpp sf_merger/sphere_projector.cpp + sf_merger/storage_boundaries.cpp ) target_link_libraries(unittests_sfmerger PRIVATE sfmergerlib Catch2::Catch2WithMain) atb_configure_test(unittests_sfmerger) diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 45a3d351..c2854a22 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -6,6 +6,7 @@ #include #include "encoded.h" +#include "build.h" #include "io/bytes.h" #include "io/serialize.h" #include "octree/disk/IndexFile.h" @@ -174,3 +175,36 @@ TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[s CHECK(reopened_result->has(octree::Id::root()).value()); CHECK(reopened_result->load(octree::Id::root()).has_value()); } + +TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") { + const std::filesystem::path fixture_path = + std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor/sf-flat"; + auto fixture_result = octree::open_folder_indexed(fixture_path); + REQUIRE(fixture_result.has_value()); + const auto mesh_result = fixture_result->load(octree::Id::root()); + REQUIRE(mesh_result.has_value()); + + TemporaryDirectory input_directory; + TemporaryDirectory output_directory; + auto input_result = octree::open_folder_indexed(input_directory.path()); + REQUIRE(input_result.has_value()); + auto input = std::move(input_result.value()); + const octree::Id root = octree::Id::root(); + REQUIRE(input.save(root, mesh_result.value()).has_value()); + REQUIRE(input.save(root.child(0).value(), mesh_result.value()).has_value()); + REQUIRE(input.index().is(store::NodeStatus::Inner, root).value()); + + auto output_result = dag::storage::open_folder_indexed(output_directory.path()); + REQUIRE(output_result.has_value()); + auto output = std::move(output_result.value()); + const dag::BuildOptions options{ + .clusters_per_partition = 1, + .target_ratio = 1.0f, + .relative_target_error = std::nullopt, + }; + + const auto built = dag::build_full(input, output, options); + REQUIRE_FALSE(built.has_value()); + CHECK(built.error().key == root); + CHECK(output.index().empty()); +} diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp new file mode 100644 index 00000000..3ef11068 --- /dev/null +++ b/unittests/sf_builder/storage_finalization.cpp @@ -0,0 +1,80 @@ +#include +#include +#include + +#include + +#include "octree/storage/open.h" +#include "terrainbuilder.h" + +namespace { + +class TemporaryDirectory { +public: + TemporaryDirectory() { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-sfbuilder-finalize-" + std::to_string(timestamp) + "-" + + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple fixture_mesh() { + const std::filesystem::path fixture_path = + std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor/sf-flat"; + auto fixture = octree::open_folder_indexed(fixture_path); + REQUIRE(fixture.has_value()); + auto loaded = fixture->load(octree::Id::root()); + REQUIRE(loaded.has_value()); + return std::move(loaded.value()); +} + +} // namespace + +TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuilder][sf]") { + TemporaryDirectory directory; + auto storage_result = octree::open_folder(directory.path()); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + REQUIRE(storage.save(octree::Id::root(), fixture_mesh()).has_value()); + + CHECK(terrainbuilder::finalize_storage(storage).has_value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); +} + +TEST_CASE( + "SF builder finalization retains an invalid written index for diagnosis", + "[sfbuilder][sf]") { + TemporaryDirectory directory; + auto storage_result = octree::open_folder(directory.path()); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + const octree::Id root = octree::Id::root(); + const mesh::Simple mesh = fixture_mesh(); + REQUIRE(storage.save(root, mesh).has_value()); + REQUIRE(storage.save(root.child(0).value(), mesh).has_value()); + + const auto finalized = terrainbuilder::finalize_storage(storage); + REQUIRE_FALSE(finalized.has_value()); + REQUIRE(std::holds_alternative(finalized.error())); + CHECK(std::get(finalized.error()).key == root); + CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); + + auto reopened = octree::open_index(directory.path() / "terrain.index"); + REQUIRE(reopened.has_value()); + CHECK(reopened->index().is(store::NodeStatus::Inner, root).value()); +} diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp new file mode 100644 index 00000000..cbc96ed6 --- /dev/null +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -0,0 +1,196 @@ +#include +#include +#include + +#include + +#include "cut.h" +#include "merge.h" +#include "octree/storage/open.h" + +namespace { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label) { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + _path = std::filesystem::temp_directory_path() + / ("atb-sfmerger-" + std::string(label) + "-" + + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(_path)); + } + + ~TemporaryDirectory() { + std::error_code error; + std::filesystem::remove_all(_path, error); + } + + const std::filesystem::path &path() const { + return _path; + } + +private: + std::filesystem::path _path; +}; + +mesh::Simple triangle(const double x_offset = 0.0) { + return mesh::Simple( + {{0, 1, 2}}, + { + {-1.0 + x_offset, 0.0, 0.0}, + {1.0 + x_offset, 0.0, 0.0}, + {x_offset, 2.0, 0.0}, + }); +} + +mesh::Simple clipping_box() { + return mesh::Simple( + { + {0, 1, 2}, {0, 2, 3}, + {1, 5, 6}, {1, 6, 2}, + {5, 4, 7}, {5, 7, 6}, + {4, 0, 3}, {4, 3, 7}, + {3, 2, 6}, {3, 6, 7}, + {4, 5, 1}, {4, 1, 0}, + }, + { + {0.0, -1.0, -1.0}, + {2.0, -1.0, -1.0}, + {2.0, 3.0, -1.0}, + {0.0, 3.0, -1.0}, + {0.0, -1.0, 1.0}, + {2.0, -1.0, 1.0}, + {2.0, 3.0, 1.0}, + {0.0, 3.0, 1.0}, + }); +} + +octree::IndexedStorage indexed_storage(const std::filesystem::path &path) { + auto result = octree::open_folder_indexed(path); + REQUIRE(result.has_value()); + return std::move(result.value()); +} + +octree::Storage unindexed_storage(const std::filesystem::path &path) { + auto result = octree::open_folder(path); + REQUIRE(result.has_value()); + return std::move(result.value()); +} + +} // namespace + +TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][validation]") { + TemporaryDirectory left_directory("invalid-left"); + TemporaryDirectory right_directory("invalid-right"); + TemporaryDirectory output_directory("invalid-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + const octree::Id root = octree::Id::root(); + REQUIRE(left.save(root, triangle()).has_value()); + REQUIRE(left.save(root.child(0).value(), triangle()).has_value()); + + const auto merged = merge_datasets(left, right, output); + REQUIRE_FALSE(merged.has_value()); + REQUIRE(std::holds_alternative(merged.error())); + CHECK(std::get(merged.error()).key == root); + CHECK_FALSE(std::filesystem::exists(output_directory.path() / "terrain.index")); +} + +TEST_CASE( + "SF merge retains an invalid output index for diagnosis", + "[sfmerger][sf][validation]") { + TemporaryDirectory left_directory("valid-left"); + TemporaryDirectory right_directory("valid-right"); + TemporaryDirectory output_directory("bad-final-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + const octree::Id root = octree::Id::root(); + REQUIRE(output.save(root, triangle()).has_value()); + REQUIRE(output.save(root.child(0).value(), triangle()).has_value()); + + const auto merged = merge_datasets(left, right, output); + REQUIRE_FALSE(merged.has_value()); + REQUIRE(std::holds_alternative(merged.error())); + CHECK(std::get(merged.error()).key == root); + CHECK(std::filesystem::is_regular_file(output_directory.path() / "terrain.index")); +} + +TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfmerger][sf][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("unchanged") { + TemporaryDirectory left_directory("unchanged-left"); + TemporaryDirectory right_directory("unchanged-right"); + TemporaryDirectory output_directory("unchanged-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(left.save(root, triangle()).has_value()); + + REQUIRE(merge_datasets(left, right, output).has_value()); + REQUIRE(output.has(root).value()); + CHECK(std::filesystem::equivalent( + left.paths(root)->front(), + output.paths(root)->front())); + } + + SECTION("changed") { + TemporaryDirectory left_directory("changed-left"); + TemporaryDirectory right_directory("changed-right"); + TemporaryDirectory output_directory("changed-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(left.save(root, triangle()).has_value()); + REQUIRE(right.save(root, triangle(10.0)).has_value()); + + REQUIRE(merge_datasets(left, right, output).has_value()); + REQUIRE(output.has(root).value()); + CHECK_FALSE(std::filesystem::equivalent( + left.paths(root)->front(), + output.paths(root)->front())); + CHECK_FALSE(std::filesystem::equivalent( + right.paths(root)->front(), + output.paths(root)->front())); + REQUIRE(output.load(root).has_value()); + CHECK(output.load(root)->face_count() == 2); + } +} + +TEST_CASE("SF cut hard-links unchanged leaves and writes clipped leaves", "[sfmerger][sf][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("unchanged") { + TemporaryDirectory input_directory("cut-unchanged-input"); + TemporaryDirectory output_directory("cut-unchanged-output"); + auto input = indexed_storage(input_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(input.save(root, triangle()).has_value()); + + REQUIRE(cut_dataset(input, MeshMask{}, output, false).has_value()); + REQUIRE(output.has(root).value()); + CHECK(std::filesystem::equivalent( + input.paths(root)->front(), + output.paths(root)->front())); + } + + SECTION("clipped") { + TemporaryDirectory input_directory("cut-clipped-input"); + TemporaryDirectory output_directory("cut-clipped-output"); + auto input = indexed_storage(input_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(input.save(root, triangle()).has_value()); + + REQUIRE(cut_dataset(input, MeshMask{clipping_box()}, output, true).has_value()); + REQUIRE(output.has(root).value()); + CHECK_FALSE(std::filesystem::equivalent( + input.paths(root)->front(), + output.paths(root)->front())); + const auto clipped = output.load(root); + REQUIRE(clipped.has_value()); + CHECK_FALSE(clipped->is_empty()); + } +} diff --git a/unittests/terrainlib/sf_validate_index.cpp b/unittests/terrainlib/sf_validate_index.cpp new file mode 100644 index 00000000..b8f7e2ec --- /dev/null +++ b/unittests/terrainlib/sf_validate_index.cpp @@ -0,0 +1,24 @@ +#include + +#include "octree/StoreTraits.h" +#include "sf/validate_index.h" + +TEST_CASE("Structura Fundamentalis validation accepts Leaf and Virtual nodes", "[sf][store]") { + store::Index index; + const octree::Id leaf = octree::Id::root().child(2).value().child(5).value(); + REQUIRE(index.add(leaf).has_value()); + + CHECK(sf::validate_index(index).has_value()); +} + +TEST_CASE("Structura Fundamentalis validation reports the first Inner node", "[sf][store]") { + store::Index index; + const octree::Id root = octree::Id::root(); + const octree::Id child = root.child(3).value(); + REQUIRE(index.add(root).has_value()); + REQUIRE(index.add(child).has_value()); + + const auto result = sf::validate_index(index); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().key == root); +} diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index 04125969..3065a6d2 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -96,6 +96,91 @@ class MultiFileCodec final : public store::Codec { second += ".metadata"; return {first, second}; } + + std::expected read( + const store::NodePath &node_path) const override { + const auto value = io::read_from_path(paths(node_path).front()); + if (!value.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Io, + "multi-file read failed", + }); + } + return value.value(); + } + + std::expected write( + const store::NodePath &node_path, + const int &value) const override { + const auto node_paths = paths(node_path); + if (!io::write_to_path(value, node_paths[0]).has_value() + || !io::write_to_path(value + 1, node_paths[1]).has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + "multi-file write failed", + }); + } + return {}; + } +}; + +class ConfigurableCodec final : public store::Codec { +public: + explicit ConfigurableCodec(std::string extension) + : extension(std::move(extension)) {} + + std::vector paths( + const store::NodePath &node_path) const override { + std::filesystem::path path = node_path.path(); + path += extension; + return {path}; + } + + std::expected read( + const store::NodePath &node_path) const override { + if (fail_read) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Domain, + "injected decode failure", + }); + } + const auto value = io::read_from_path(paths(node_path).front()); + if (!value.has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Io, + "configurable read failed", + }); + } + return value.value(); + } + + std::expected write( + const store::NodePath &node_path, + const int &value) const override { + if (fail_write) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + "injected encode failure", + }); + } + if (!io::write_to_path(value, paths(node_path).front()).has_value()) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + "configurable write failed", + }); + } + return {}; + } + + std::string extension; + bool fail_read = false; + bool fail_write = false; }; std::atomic_uint32_t index_writes = 0; @@ -138,12 +223,14 @@ store::IndexFormat counting_index_format() { } store::IndexedStorage make_indexed_storage( - const std::filesystem::path &path) { + const std::filesystem::path &path, + std::unique_ptr> codec = + std::make_unique>()) { using Storage = store::Storage; return store::IndexedStorage( store::RawStorage( store::Layout(path, octree::store_layout::flat()), - std::make_unique>()), + std::move(codec)), store::Index{}, Storage::Persistence{ counting_index_format(), @@ -249,3 +336,177 @@ TEST_CASE("storage moves transfer and finalize dirty index state", "[store][stor } CHECK(index_writes == 3); } + +TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("one file") { + TemporaryDirectory source_directory("copy-one-source"); + TemporaryDirectory target_directory("copy-one-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 41).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + REQUIRE(target.has(root).value()); + CHECK(target.load(root).value() == 41); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); + } + + SECTION("multiple files") { + TemporaryDirectory source_directory("copy-many-source"); + TemporaryDirectory target_directory("copy-many-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage( + target_directory.path(), std::make_unique()); + REQUIRE(source.save(root, 42).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + const auto source_paths = source.paths(root).value(); + const auto target_paths = target.paths(root).value(); + REQUIRE(source_paths.size() == 2); + REQUIRE(target_paths.size() == 2); + CHECK(std::filesystem::equivalent(source_paths[0], target_paths[0])); + CHECK(std::filesystem::equivalent(source_paths[1], target_paths[1])); + } +} + +TEST_CASE("copy_from re-encodes differing path lists", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-reencode-source"); + TemporaryDirectory target_directory("copy-reencode-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 43).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(target.load(root).value() == 43); + CHECK(target.paths(root)->size() == 1); + CHECK(source.paths(root)->size() == 2); + CHECK_FALSE(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); +} + +TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-overwrite-source"); + TemporaryDirectory target_directory("copy-overwrite-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 44).has_value()); + REQUIRE(target.save(root, 1).has_value()); + + const auto rejected = target.copy_from(root, source); + REQUIRE_FALSE(rejected.has_value()); + CHECK(std::holds_alternative(rejected.error())); + CHECK(target.load(root).value() == 1); + + target.settings().allow_overwrite = true; + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(target.load(root).value() == 44); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target.paths(root)->front())); +} + +TEST_CASE( + "failed multi-file overwrite leaves the target node unindexed", + "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-partial-source"); + TemporaryDirectory target_directory("copy-partial-target"); + auto source = make_indexed_storage( + source_directory.path(), std::make_unique()); + auto target = make_indexed_storage( + target_directory.path(), std::make_unique()); + REQUIRE(source.save(root, 45).has_value()); + REQUIRE(target.save(root, 1).has_value()); + target.settings().allow_overwrite = true; + + const auto target_paths = target.paths(root).value(); + REQUIRE(std::filesystem::remove(target_paths[1])); + REQUIRE(std::filesystem::create_directory(target_paths[1])); + REQUIRE(io::write_to_path(9, target_paths[1] / "blocker.bin").has_value()); + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(std::holds_alternative(copied.error())); + CHECK_FALSE(target.has(root).value()); + CHECK_FALSE(target.index().get(root).value().has_value()); + CHECK(std::filesystem::equivalent( + source.paths(root)->front(), + target_paths.front())); + CHECK(std::filesystem::is_directory(target_paths[1])); +} + +TEST_CASE("copy_from propagates source and codec failures", "[store][storage][copy]") { + const octree::Id root = octree::Id::root(); + + SECTION("missing source") { + TemporaryDirectory source_directory("copy-missing-source"); + TemporaryDirectory target_directory("copy-missing-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(std::holds_alternative>( + copied.error())); + CHECK(target.has(root).value()); + CHECK(target.load(root).value() == 7); + } + + SECTION("decode failure") { + TemporaryDirectory source_directory("copy-decode-source"); + TemporaryDirectory target_directory("copy-decode-target"); + auto source_codec = std::make_unique(".source"); + ConfigurableCodec *source_codec_ptr = source_codec.get(); + auto source = make_indexed_storage( + source_directory.path(), std::move(source_codec)); + auto target = make_indexed_storage( + target_directory.path(), + std::make_unique(".target")); + REQUIRE(source.save(root, 46).has_value()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + source_codec_ptr->fail_read = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + REQUIRE(std::holds_alternative(copied.error())); + CHECK(std::get(copied.error()).operation + == store::CodecOperation::Read); + CHECK(target.has(root).value()); + CHECK(target.load(root).value() == 7); + } + + SECTION("encode failure") { + TemporaryDirectory source_directory("copy-encode-source"); + TemporaryDirectory target_directory("copy-encode-target"); + auto source = make_indexed_storage( + source_directory.path(), + std::make_unique(".source")); + auto target_codec = std::make_unique(".target"); + ConfigurableCodec *target_codec_ptr = target_codec.get(); + auto target = make_indexed_storage( + target_directory.path(), std::move(target_codec)); + REQUIRE(source.save(root, 47).has_value()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + target_codec_ptr->fail_write = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + REQUIRE(std::holds_alternative(copied.error())); + CHECK(std::get(copied.error()).operation + == store::CodecOperation::Write); + CHECK_FALSE(target.has(root).value()); + } +} From bca0b5cd591e4e0af13cfcd27c702bd202ba4406 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:40:03 +0200 Subject: [PATCH 09/43] Remove octree migration shims --- docs/raster-store/README.md | 1 + docs/raster-store/architecture.md | 59 ++++++++ docs/raster-store/before-refactor/README.md | 7 + docs/raster-store/refactor-status.md | 34 ++++- docs/raster-store/status-quo.md | 13 ++ docs/raster-store/storage-format.md | 12 ++ src/dag_builder/build.cpp | 30 ++-- src/dag_builder/build.h | 10 +- src/dag_builder/main.cpp | 4 +- src/dag_builder/storage.h | 22 ++- src/dag_builder/thread_safe_storage.h | 4 +- src/dag_convert_debug/main.cpp | 8 +- src/sf_builder/terrainbuilder.cpp | 5 +- src/sf_builder/terrainbuilder.h | 4 +- src/sf_index_browser/main.cpp | 30 ++-- src/sf_merger/NodeLoader.h | 22 +-- src/sf_merger/NodeWriter.h | 20 +-- src/sf_merger/cut.h | 26 ++-- src/sf_merger/main.cpp | 9 +- src/sf_merger/merge.h | 21 +-- src/sf_merger/merge/NodeData.h | 15 +- src/sf_merger/merge/visitor/Masked.h | 4 +- src/sf_merger/merge/visitor/Simple.h | 4 +- src/sf_merger/merge/visitor/Visitor.h | 8 +- src/terrainlib/octree/IndexMap.h | 94 ------------- src/terrainlib/octree/NodeStatus.h | 7 - src/terrainlib/octree/NodeStatusOrMissing.h | 7 - src/terrainlib/octree/Storage.h | 13 -- src/terrainlib/octree/disk/IndexFile.h | 5 +- src/terrainlib/octree/storage/IndexFile.h | 5 +- src/terrainlib/octree/storage/MeshStorage.h | 3 - src/terrainlib/octree/storage/cache/Dummy.h | 11 -- src/terrainlib/octree/storage/cache/ICache.h | 11 -- .../octree/storage/cache/LruCache.h | 11 -- src/terrainlib/octree/storage/open.h | 7 +- src/terrainlib/octree/traverse.h | 52 ------- src/terrainlib/sf/finalize_storage.h | 4 +- unittests/CMakeLists.txt | 2 - .../dag_builder/storage_compatibility.cpp | 16 ++- unittests/sf_merger/storage_boundaries.cpp | 4 +- unittests/terrainlib/index_map.cpp | 100 ------------- unittests/terrainlib/index_traverse.cpp | 132 ------------------ unittests/terrainlib/store_compatibility.cpp | 29 ++-- 43 files changed, 279 insertions(+), 606 deletions(-) delete mode 100644 src/terrainlib/octree/IndexMap.h delete mode 100644 src/terrainlib/octree/NodeStatus.h delete mode 100644 src/terrainlib/octree/NodeStatusOrMissing.h delete mode 100644 src/terrainlib/octree/Storage.h delete mode 100644 src/terrainlib/octree/storage/MeshStorage.h delete mode 100644 src/terrainlib/octree/storage/cache/Dummy.h delete mode 100644 src/terrainlib/octree/storage/cache/ICache.h delete mode 100644 src/terrainlib/octree/storage/cache/LruCache.h delete mode 100644 src/terrainlib/octree/traverse.h delete mode 100644 unittests/terrainlib/index_map.cpp delete mode 100644 unittests/terrainlib/index_traverse.cpp diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md index 04429875..e78c6fd0 100644 --- a/docs/raster-store/README.md +++ b/docs/raster-store/README.md @@ -11,6 +11,7 @@ baseline, not a finalized binary-format specification. - [Architecture](architecture.md) - [Storage format](storage-format.md) - [Sampling and pyramid generation](sampling-and-generation.md) +- [Implemented refactor status](refactor-status.md) ## Plans diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md index 204c7ac7..9373d754 100644 --- a/docs/raster-store/architecture.md +++ b/docs/raster-store/architecture.md @@ -1,5 +1,64 @@ # Architecture +## Implementation status + +This document describes the intended raster-fundamentalis and tile-base +architecture. Its RF format, snapshot-publication, builder, merger, pyramid, +and server requirements are future work; they were not acceptance criteria +for the completed 2D/3D hierarchy-store refactor. The implemented boundary is +recorded in [refactor-status.md](refactor-status.md), and the original plan is +kept in [refactor-plan.md](refactor-plan.md). + +The refactor delivered the dimension-neutral mechanisms needed by that future +work: + +- `store::Index`, `store::traverse`, layouts, runtime codecs, storage, + typed errors, and cache interfaces; +- `octree::StoreTraits` plus legacy 3D layout/index/open adapters; +- `raster_store::StoreTraits` for in-memory topology keyed by + `radix::tile::Id`; and +- SF-only topology validation under `sf`. + +It deliberately did not define a persistent RF index, RF payload codec, RF +opening API, `rf_builder`, `rf_merger`, tile-base generator, or tile server. + +### Final public names + +The shared API uses `store::NodeStatus`, `store::NodeStatusOrMissing`, +`store::Index`, `store::traverse`, `store::RawStorage`, +`store::Storage`, and `store::IndexedStorage`. Mesh storage aliases live under +`mesh::storage`; DAG batch and metadata aliases live under `dag::storage`. +The `octree` namespace retains the 3D key, traits, path mappings, legacy index +DTO, and mesh-opening compatibility functions. + +A legacy 3D mesh dataset is opened through the 3D adapter: + +```cpp +#include "octree/storage/open.h" + +auto opened = octree::open_folder_indexed(dataset_path); +if (!opened.has_value()) { + return std::unexpected(opened.error()); +} +mesh::storage::IndexedStorage storage = std::move(opened.value()); +``` + +The 2D traits adapter can exercise the shared topology without implying a +persistent RF format: + +```cpp +#include "raster_store/StoreTraits.h" +#include "store/Index.h" +#include "store/traverse.h" + +store::Index index; +const radix::tile::Id tile{2, {1, 3}}; +auto added = index.add(tile); +auto walked = store::traverse(index, [](const auto &id, store::NodeStatus status) { + // In-memory hierarchy processing only. +}); +``` + ## System boundary The design separates authoritative data management from delivery generation: diff --git a/docs/raster-store/before-refactor/README.md b/docs/raster-store/before-refactor/README.md index f9d3d9f0..5a5e146d 100644 --- a/docs/raster-store/before-refactor/README.md +++ b/docs/raster-store/before-refactor/README.md @@ -1,5 +1,12 @@ # SF storage and merge architecture before the 2D/3D refactor +> **Historical report:** this file describes the pre-refactor tree and keeps +> its original names, paths, diagrams, and conclusions for review history. +> Some linked source paths were intentionally removed by the completed +> migration. For the implemented boundary and verification results, see +> [refactor-status.md](../refactor-status.md); for current public names and +> examples, see [architecture.md](../architecture.md#implementation-status). + This report describes the existing Structura Fundamentalis (SF) builder, sparse octree index, storage stack, and mask-based dataset merger. Its purpose is to identify the mechanisms that can be generalised from 3D octrees to a diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 132225d9..7f4f4b27 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 4 — Harden node reuse and enforce SF topology +- Phase: 5 — Cleanup and documentation - State: complete; completion commit pending. -- Next action: commit and tag Phase 4, then begin Phase 5 cleanup and - documentation. +- Next action: commit and tag Phase 5, then remove migration-only fixtures in + Phase 6. ## Phase log @@ -114,6 +114,23 @@ This file is the resumable implementation log for hard links, and newly written changed/clipped nodes. - State: complete. +### Phase 5 — Cleanup and documentation + +- Migrated repository callers from the temporary octree topology, traversal, + storage, and cache aliases to the final `store`, `mesh::storage`, and + `dag::storage` public names. +- Removed the temporary `octree::IndexMap`, status, traversal, mesh-storage, + and cache forwarding headers and their wrapper-only unit tests. +- Changed the unchanged legacy `terrain.index` DTO to serialize the shared + `store::Index` directly. +- Kept only 3D identity/geometry, traits, mappings, legacy index-format, and + opening adapters under `octree`. +- Updated the architecture, storage-format, status-quo, historical + before-refactor report, and raster-store index documentation. The docs now + distinguish the implemented shared boundary from future RF formats and + tools and include final public names plus 3D-open and in-memory 2D examples. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -167,6 +184,15 @@ This file is the resumable implementation log for `radix::tile::Scheme` remains isolated; the new finalization tests build and pass in `unittests_sfbuilder_finalization`. - Phase 4 `git diff --check`: passed. +- Phase 5 `unittests_terrainlib`: 24,017 assertions in 426 test cases passed. +- Phase 5 `unittests_dagbuilder`: 472 assertions in 74 test cases passed. +- Phase 5 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 5 SF builder finalization: 19 assertions in 2 test cases passed. +- Phase 5 application builds passed for `sf-builder`, `sf-merger`, + `sf-index-browser`, `dag-builder`, and `dag-convert-debug`. +- Phase 5 repository searches found no remaining caller of a removed octree + forwarding name and no `octree::Id` dependency under `terrainlib/store`. +- Phase 5 `git diff --check`: passed. ## Commit and tag log @@ -184,3 +210,5 @@ This file is the resumable implementation log for - `61acb74` — Phase 3 shared runtime-codec storage core. - `4ffcdea` / `refactor_phase_3` — Phase 3 application migration and completion tag. +- `3086336` / `refactor_phase_4` — hardened node reuse, SF topology + enforcement, and Phase 4 completion tag. diff --git a/docs/raster-store/status-quo.md b/docs/raster-store/status-quo.md index 9d2b5701..461faf46 100644 --- a/docs/raster-store/status-quo.md +++ b/docs/raster-store/status-quo.md @@ -1,5 +1,18 @@ # Status quo and reuse assessment +> **Historical assessment:** this document records the repository before the +> shared-store refactor and is intentionally not rewritten as current code +> documentation. The completed implementation and verification record is in +> [refactor-status.md](refactor-status.md); the resulting public boundary is +> summarized in [architecture.md](architecture.md#implementation-status). + +The refactor chose the generalization option discussed below. Sparse topology, +traversal, layouts, runtime codecs, storage, index lifecycle, and cache +interfaces now live under `store` and are parameterized by hierarchy traits. +The legacy 3D format remains an adapter under `octree`, while +`raster_store::StoreTraits` proves the shared in-memory mechanisms with +`radix::tile::Id`. Persistent RF formats and RF tools remain future work. + This document evaluates the current repository after commit `9cf9065` (`Consolidate raster handling and use std::expected`). It distinguishes reusable mechanisms from interfaces that encode assumptions unsuitable for diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 5767e0f8..0332e8e5 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -1,5 +1,17 @@ # Storage format +> **Status:** future raster-store design. The completed 2D/3D store refactor +> did not implement or finalize the RF/TB formats described below. These +> requirements were not acceptance criteria for that refactor. See +> [refactor-status.md](refactor-status.md) for the implemented shared boundary. + +The refactor preserves the existing 3D `terrain.index` and payload contracts +through the adapters under `octree` and `mesh::storage`. It supplies only an +in-memory `raster_store::StoreTraits` adapter for `radix::tile::Id`; there is +currently no persistent RF index, `.amort` codec, snapshot publisher, or RF +opening API. Names and formats in this document remain proposals until their +own implementation plans approve them. + This document specifies the logical format and required invariants for the following data stores: - raster-fundamentalis (our authoritative raster store) - tile-base (the tile-pyramid for our tile-server) diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 7653df1b..d8e265a6 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -26,9 +26,8 @@ #include "octree/IdRect.h" #include "octree/OddLevelShifted.h" #include "octree/Space.h" -#include "octree/Storage.h" #include "octree/storage/open.h" -#include "octree/traverse.h" +#include "store/traverse.h" #include "ProgressIndicator.h" #include "partition.h" #include "simplify.h" @@ -49,7 +48,7 @@ namespace { // Load a mesh from storage and clusterize it. std::optional load_and_clusterize_mesh( - const octree::MeshStorage &storage, + const mesh::storage::Storage &storage, const octree::Id &id) { const auto result = storage.load(id); @@ -127,7 +126,7 @@ LodResult build_lod( // Load input meshes, clusterize, and filter them to the target region. std::vector load_input_clusters( const std::span input_ids, - const octree::MeshStorage &input_storage, + const mesh::storage::Storage &input_storage, const RegionFilter& filter) { std::vector result; @@ -154,8 +153,8 @@ std::vector load_input_clusters( // Dependencies shared across the whole dag building pipeline. struct BuildContext { - const octree::IndexedMeshStorage &input_storage; - ThreadSafeStorage output_storage; + const mesh::storage::IndexedStorage &input_storage; + ThreadSafeStorage output_storage; const BuildOptions &options; const octree::Space &space; const octree::OddLevelShifted &shifted_space; @@ -462,10 +461,10 @@ std::vector> gather_relevant_input_leaves( const auto start = space.find_smallest_node_encompassing_bounds(root_bounds) .value_or(octree::Id::root()); std::vector> result(octree::Id::max_level() + 1); - octree::traverse( + const auto traversal = store::traverse( index, - [&](const octree::Id &id, const octree::NodeStatus status) { - if (status == octree::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { + [&](const octree::Id &id, const store::NodeStatus status) { + if (status == store::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { result[id.level()].push_back(id); } }, @@ -473,6 +472,7 @@ std::vector> gather_relevant_input_leaves( return radix::geometry::intersect(root_bounds, space.get_node_bounds(id)); }, start); + DEBUG_ASSERT(traversal.has_value()); return result; } @@ -518,7 +518,7 @@ std::unordered_set build_level( } // Initialize debug storage if requested (contains .glb meshes) - std::optional debug_storage; + std::optional debug_storage; if (ctx.options.write_debug_meshes) { octree::OpenOptions options; options.preferred_extension = ".glb"; @@ -586,8 +586,8 @@ std::unordered_set build_level( // Iterates octree levels from finest to coarsest, simplifying and re-clustering geometry at each // level from its children. std::expected build_levels( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range) { const auto validation = sf::validate_index(input_storage.index()); @@ -619,7 +619,7 @@ std::expected build_levels( // TODO: use hierachical lookup here based on root_bounds and IdRect std::unordered_set prev_level_built; for (const auto &[id, status] : output_storage.index()) { - if (id.level() == range.end && status != octree::NodeStatus::Virtual) { + if (id.level() == range.end && status != store::NodeStatus::Virtual) { prev_level_built.insert(id); } } @@ -648,8 +648,8 @@ std::expected build_levels( // Builds the complete DAG from input_storage into output_storage. std::expected build_full( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options) { return build_levels(input_storage, output_storage, options, RangeFull{}); } diff --git a/src/dag_builder/build.h b/src/dag_builder/build.h index b87528e7..a2b774a4 100644 --- a/src/dag_builder/build.h +++ b/src/dag_builder/build.h @@ -8,7 +8,7 @@ #include "Range.h" #include "build_config.h" #include "texturing.h" -#include "octree/storage/MeshStorage.h" +#include "mesh/storage.h" #include "storage.h" #include "sf/InvalidTopology.h" @@ -33,13 +33,13 @@ struct BuildOptions { }; std::expected build_full( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options); std::expected build_levels( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range); diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index 4fb1d70a..b733c3c6 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -33,8 +33,8 @@ int main(int argc, char **argv) { store::describe_error(output_result.error())); return EXIT_FAILURE; } - const octree::IndexedMeshStorage input_storage = std::move(input_result.value()); - octree::IndexedDagStorage output_storage = std::move(output_result.value()); + const mesh::storage::IndexedStorage input_storage = std::move(input_result.value()); + dag::storage::IndexedStorage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = args.continuation_mode == ContinuationMode::Overwrite; dag::BuildOptions options{ diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 92bccec9..b3638172 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -10,20 +10,16 @@ #include "serialization.h" #include "store/IndexedStorage.h" -namespace octree { - -using DagStorage = store::Storage; -using IndexedDagStorage = store::IndexedStorage; -using DagMetaStorage = store::Storage; -using IndexedDagMetaStorage = store::IndexedStorage; - -} // namespace octree - namespace dag::storage { using OpenOptions = octree::storage::OpenOptions; +using Storage = store::Storage; +using IndexedStorage = store::IndexedStorage; +using MetadataStorage = store::Storage; +using IndexedMetadataStorage = + store::IndexedStorage; -inline std::expected> open_index( +inline std::expected> open_index( const std::filesystem::path &path) { return store::open_index( path, @@ -31,7 +27,7 @@ inline std::expected> op dag::codec::from_extension); } -inline std::expected> open_folder( +inline std::expected> open_folder( const std::filesystem::path &path, const bool create_index = false, OpenOptions options = {}) { @@ -43,7 +39,7 @@ inline std::expected> open_fold std::move(options)); } -inline std::expected> +inline std::expected> open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { return octree::storage::open_folder_indexed( path, @@ -52,7 +48,7 @@ open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) std::move(options)); } -inline std::expected> +inline std::expected> open_metadata_indexed(const std::filesystem::path &path) { const std::filesystem::path index_path = path.filename() == octree::disk::v1::index_file_name() diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h index da54f8a0..9a4c483d 100644 --- a/src/dag_builder/thread_safe_storage.h +++ b/src/dag_builder/thread_safe_storage.h @@ -8,8 +8,8 @@ namespace dag { -// A mutex-based thread-safe wrapper around a Storage_ instance. -// TODO: move this locking into Storage_ itself and remove this wrapper once that lands. +// A mutex-based thread-safe wrapper around a Storage instance. +// TODO: move this locking into Storage itself and remove this wrapper once that lands. template class ThreadSafeStorage { static_assert(!std::is_const_v, "ThreadSafeStorage requires a non-const Storage"); diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index d72be3a2..f4901974 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -7,7 +7,7 @@ #include "io/serialize.h" #include "log.h" #include "mesh/io.h" -#include "octree/storage/MeshStorage.h" +#include "mesh/storage.h" #include "octree/storage/open.h" #include "ProgressIndicator.h" #include "storage.h" @@ -41,7 +41,7 @@ void export_storage(const cli::Args &args) { store::describe_error(input_result.error())); return; } - const octree::IndexedDagStorage input_storage = std::move(input_result.value()); + const dag::storage::IndexedStorage input_storage = std::move(input_result.value()); octree::OpenOptions options; options.preferred_extension = ".glb"; @@ -56,7 +56,7 @@ void export_storage(const cli::Args &args) { store::describe_error(output_result.error())); return; } - octree::MeshStorage output_storage = std::move(output_result.value()); + mesh::storage::Storage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = true; size_t exported_count = 0; @@ -65,7 +65,7 @@ void export_storage(const cli::Args &args) { auto progress_thread = progress.start_monitoring(); for (const auto &[id, status] : input_storage.index()) { - if (status == octree::NodeStatus::Virtual) { + if (status == store::NodeStatus::Virtual) { progress.task_finished(); continue; } diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 438da793..310e384d 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -41,7 +41,6 @@ #include "octree/Id.h" #include "octree/Space.h" -#include "octree/Storage.h" #include "octree/storage/open.h" #include "octree/utils.h" #include "store/describe_error.h" @@ -174,7 +173,7 @@ T expect(const std::optional &opt, const std::string &msg) { } } -std::expected finalize_storage(octree::Storage &storage) { +std::expected finalize_storage(mesh::storage::Storage &storage) { return sf::finalize_storage(storage); } @@ -207,7 +206,7 @@ std::expected build_all_patches( output_base_path, store::describe_error(storage_result.error())); } - octree::Storage storage = std::move(storage_result.value()); + mesh::storage::Storage storage = std::move(storage_result.value()); storage.settings().allow_overwrite = overwrite_existing; const auto dataset_srs = dataset.srs(); diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index 35d51b29..8ddbeba0 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -9,7 +9,7 @@ #include "octree/Id.h" #include "mesh/SimpleMesh.h" #include "tile_provider.h" -#include "octree/Storage.h" +#include "mesh/storage.h" #include "sf/Error.h" namespace terrainbuilder { @@ -31,7 +31,7 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -std::expected finalize_storage(octree::Storage &storage); +std::expected finalize_storage(mesh::storage::Storage &storage); std::expected build_all_patches( Dataset &dataset, diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index 0c6e6ddd..f1008ff3 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -13,18 +13,16 @@ #include "log.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/Storage.h" +#include "mesh/storage.h" #include "octree/storage/open.h" -#include "octree/traverse.h" #include "cli.h" #include "store/describe_error.h" +#include "store/NodeStatusOrMissing.h" namespace { struct IndexNode { octree::Id id; - octree::NodeStatusOrMissing status; + store::NodeStatusOrMissing status; bool expanded = false; }; struct MeshNode { @@ -44,8 +42,8 @@ class TreeView { using iterator = std::list::iterator; using const_iterator = std::list::const_iterator; - explicit TreeView(const octree::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { - const octree::NodeStatusOrMissing status( + explicit TreeView(const mesh::storage::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { + const store::NodeStatusOrMissing status( DEBUG_ASSERT_VAL(this->_storage.index().get(root)).value()); this->_view.emplace_back(IndexNode{root, status, false}); } @@ -82,7 +80,7 @@ class TreeView { } parent.expanded = true; - if (parent.status == octree::NodeStatusOrMissing::Leaf) { + if (parent.status == store::NodeStatusOrMissing::Leaf) { auto result = this->_storage.load(parent.id); DisplayEntry entry; if (result.has_value()) { @@ -106,7 +104,7 @@ class TreeView { if (!status_result->has_value()) { continue; } - const octree::NodeStatus status = status_result->value(); + const store::NodeStatus status = status_result->value(); IndexNode child_node{child_id, status, false}; it++; it = this->_view.emplace(it, child_node); @@ -128,7 +126,7 @@ class TreeView { } parent.expanded = false; - if (parent.status == octree::NodeStatusOrMissing::Leaf) { + if (parent.status == store::NodeStatusOrMissing::Leaf) { it++; it = this->_view.erase(it); return; @@ -150,16 +148,16 @@ class TreeView { private: std::list _view; - const octree::IndexedStorage& _storage; + const mesh::storage::IndexedStorage& _storage; }; const octree::Id find_deepest_root( const store::Index &index, const octree::Id &root = octree::Id::root()) { - switch (octree::NodeStatusOrMissing(DEBUG_ASSERT_VAL(index.get(root)).value())) { - case octree::NodeStatusOrMissing::Missing: + switch (store::NodeStatusOrMissing(DEBUG_ASSERT_VAL(index.get(root)).value())) { + case store::NodeStatusOrMissing::Missing: return octree::Id::root(); - case octree::NodeStatusOrMissing::Leaf: + case store::NodeStatusOrMissing::Leaf: return root; default: break; @@ -191,7 +189,7 @@ const octree::Id find_deepest_root( } // namespace -octree::IndexedStorage open_path_indexed(const std::filesystem::path& path) { +mesh::storage::IndexedStorage open_path_indexed(const std::filesystem::path& path) { auto result = std::filesystem::is_directory(path) ? octree::open_folder_indexed(path) : octree::open_index(path); @@ -232,7 +230,7 @@ ftxui::Element render_line(const DisplayEntry &entry, size_t base_level, bool is } int run(const cli::Args &args) { - const octree::IndexedStorage storage = open_path_indexed(args.dataset_path); + const mesh::storage::IndexedStorage storage = open_path_indexed(args.dataset_path); const store::Index &index = storage.index(); const octree::Id root_id = args.full_view ? octree::Id::root() : find_deepest_root(index); diff --git a/src/sf_merger/NodeLoader.h b/src/sf_merger/NodeLoader.h index c7c44b90..23a2cbd1 100644 --- a/src/sf_merger/NodeLoader.h +++ b/src/sf_merger/NodeLoader.h @@ -9,19 +9,21 @@ #include "mesh/clip.h" #include "mesh/texture_trim.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" #include "octree/Space.h" -#include "octree/Storage.h" -#include "octree/storage/cache/Dummy.h" -#include "octree/storage/cache/ICache.h" +#include "mesh/storage.h" +#include "store/NodeStatusOrMissing.h" +#include "store/cache/Interface.h" class NodeLoader { public: - NodeLoader(const octree::IndexedStorage &storage, octree::cache::ICache &cache, octree::Space space) + NodeLoader( + const mesh::storage::IndexedStorage &storage, + store::cache::Interface &cache, + octree::Space space) : _storage(storage), _cache(cache), _space(space) {} - octree::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { - return octree::NodeStatusOrMissing( + store::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { + return store::NodeStatusOrMissing( DEBUG_ASSERT_VAL(this->_storage.index().get(id)).value()); } @@ -66,12 +68,12 @@ class NodeLoader { return std::nullopt; } - const octree::IndexedStorage &storage() const { + const mesh::storage::IndexedStorage &storage() const { return this->_storage; } private: - const octree::IndexedStorage &_storage; - octree::cache::ICache &_cache; + const mesh::storage::IndexedStorage &_storage; + store::cache::Interface &_cache; const octree::Space _space; }; diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 9c51359e..6a29a556 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -8,14 +8,13 @@ #include "NodeLoader.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/Storage.h" -#include "octree/traverse.h" +#include "mesh/storage.h" +#include "store/traverse.h" // TODO: make thread safe class NodeWriter { public: - NodeWriter(octree::Storage &storage) : _storage(storage) {} + NodeWriter(mesh::storage::Storage &storage) : _storage(storage) {} std::expected> has_node( const octree::Id &id) { @@ -47,16 +46,16 @@ class NodeWriter { const octree::Id &id, const NodeLoader &loader) { std::optional> error; - octree::traverse( + const auto traversal = store::traverse( loader.storage().index(), - [&](const octree::Id &child_id, const octree::NodeStatus &status) { + [&](const octree::Id &child_id, const store::NodeStatus &status) { if (error.has_value()) { return; } - if (status == octree::NodeStatus::Virtual) { + if (status == store::NodeStatus::Virtual) { return; } - DEBUG_ASSERT(status == octree::NodeStatus::Leaf); + DEBUG_ASSERT(status == store::NodeStatus::Leaf); const auto result = this->_storage.copy_from(child_id, loader.storage()); if (!result.has_value()) { @@ -65,6 +64,9 @@ class NodeWriter { }, [&](const octree::Id &) { return !error.has_value(); }, id); + if (!traversal.has_value()) { + return std::unexpected(store::CopyError(traversal.error())); + } if (error.has_value()) { return std::unexpected(std::move(error.value())); } @@ -72,5 +74,5 @@ class NodeWriter { } private: - octree::Storage &_storage; + mesh::storage::Storage &_storage; }; diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 3c6ce4d6..6ecca34c 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -8,11 +8,9 @@ #include "log.h" #include "mask.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/Storage.h" +#include "mesh/storage.h" #include "octree/storage/open.h" #include "octree/Space.h" -#include "octree/traverse.h" #include "utils.h" #include "containers/Cow.h" #include "store/describe_error.h" @@ -21,8 +19,8 @@ #include "sf/validate_index.h" struct Context { - const octree::IndexedStorage& input; - octree::Storage& output; + const mesh::storage::IndexedStorage& input; + mesh::storage::Storage& output; const octree::Space space; const bool keep_inside; }; @@ -38,7 +36,7 @@ inline std::expected cut_leaf_node( const MeshMask& mask ) { DEBUG_ASSERT(DEBUG_ASSERT_VAL( - ctx.input.index().is(octree::NodeStatus::Leaf, id)).value()); + ctx.input.index().is(store::NodeStatus::Leaf, id)).value()); const SimpleMesh mesh = DEBUG_ASSERT_VAL(ctx.input.load(id)).value(); LOG_TRACE("Cutting mesh at {} using mask with {} vertices and {} triangles", @@ -72,7 +70,7 @@ inline std::expected cut_virtual_node( const MeshMask& mask ) { DEBUG_ASSERT(DEBUG_ASSERT_VAL( - ctx.input.index().is(octree::NodeStatus::Virtual, id)).value()); + ctx.input.index().is(store::NodeStatus::Virtual, id)).value()); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); @@ -113,11 +111,11 @@ inline std::expected cut_node( return {}; } - const octree::NodeStatus status = status_result->value(); + const store::NodeStatus status = status_result->value(); switch (status) { - case octree::NodeStatus::Virtual: + case store::NodeStatus::Virtual: return cut_virtual_node(ctx, id, mask); - case octree::NodeStatus::Leaf: + case store::NodeStatus::Leaf: return cut_leaf_node(ctx, id, mask); default: UNREACHABLE(); @@ -127,9 +125,9 @@ inline std::expected cut_node( } inline std::expected cut_dataset( - const octree::IndexedStorage &input, + const mesh::storage::IndexedStorage &input, const MeshMask& mask, - octree::Storage &output, + mesh::storage::Storage &output, const bool keep_inside) { const auto validation = sf::validate_index(input.index()); if (!validation.has_value()) { @@ -150,7 +148,7 @@ inline std::expected cut_dataset( } inline std::expected cut_dataset( - const octree::IndexedStorage &input_dataset, + const mesh::storage::IndexedStorage &input_dataset, const MeshMask& mask, const std::filesystem::path &output_path, const bool keep_inside) { @@ -164,7 +162,7 @@ inline std::expected cut_dataset( if (!output_result.has_value()) { return std::unexpected(sf::ProcessingError(output_result.error())); } - octree::IndexedStorage output_dataset = std::move(output_result.value()); + mesh::storage::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { return std::unexpected(sf::ProcessingError(store::SaveError( store::AlreadyExists{output_path}))); diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index bae30669..3d4f5bcd 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -8,7 +8,6 @@ #include "log.h" #include "mask.h" #include "merge.h" -#include "octree/Storage.h" #include "optional_utils.h" #include "earth.h" #include "store/describe_error.h" @@ -43,7 +42,7 @@ void run(const cli::MergeArgs& args) { args.base_path, store::describe_error(base_result.error())); } - octree::IndexedStorage base_dataset = std::move(base_result.value()); + mesh::storage::IndexedStorage base_dataset = std::move(base_result.value()); LOG_TRACE("Loading new dataset from {}", args.new_path); auto new_result = octree::open_folder_indexed(args.new_path); @@ -53,7 +52,7 @@ void run(const cli::MergeArgs& args) { args.new_path, store::describe_error(new_result.error())); } - octree::IndexedStorage new_dataset = std::move(new_result.value()); + mesh::storage::IndexedStorage new_dataset = std::move(new_result.value()); LOG_TRACE("Creating output dataset at {}", args.output_path); std::filesystem::create_directories(args.output_path); @@ -67,7 +66,7 @@ void run(const cli::MergeArgs& args) { args.output_path, store::describe_error(output_result.error())); } - octree::Storage output_dataset = std::move(output_result.value()); + mesh::storage::Storage output_dataset = std::move(output_result.value()); std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); @@ -90,7 +89,7 @@ void run(const cli::CutArgs& args) { args.input_path, store::describe_error(input_result.error())); } - const octree::IndexedStorage input_dataset = std::move(input_result.value()); + const mesh::storage::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); const auto cut_result = cut_dataset( input_dataset, diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index 6987d14d..bef384ac 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -13,14 +13,15 @@ #include "merge/visitor/Simple.h" #include "merge/visitor/Visitor.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/storage/cache/Dummy.h" +#include "mesh/storage.h" +#include "store/NodeStatusOrMissing.h" +#include "store/cache/Dummy.h" #include "store/describe_error.h" #include "sf/Error.h" #include "sf/finalize_storage.h" #include "sf/validate_index.h" -inline std::string get_dataset_name(const octree::Storage &storage) { +inline std::string get_dataset_name(const mesh::storage::Storage &storage) { return storage.layout().base_path().filename().string(); } @@ -34,7 +35,7 @@ using smallest_uint_t = template class Merger { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; using Context = Visitor::Context; using Result = merge::Result; using Expected = std::expected; @@ -182,11 +183,11 @@ class Merger { }; inline std::expected merge_datasets( - const octree::IndexedStorage &left_dataset, - const octree::IndexedStorage &right_dataset, - octree::Storage &output_dataset, + const mesh::storage::IndexedStorage &left_dataset, + const mesh::storage::IndexedStorage &right_dataset, + mesh::storage::Storage &output_dataset, const std::optional mask = std::nullopt) { - for (const octree::IndexedStorage *input : {&left_dataset, &right_dataset}) { + for (const mesh::storage::IndexedStorage *input : {&left_dataset, &right_dataset}) { const auto validation = sf::validate_index(input->index()); if (!validation.has_value()) { return std::unexpected(sf::ProcessingError(validation.error())); @@ -199,9 +200,9 @@ inline std::expected merge_datasets( get_dataset_name(output_dataset)); octree::Space space = octree::Space::earth(); - octree::cache::Dummy left_cache; + store::cache::Dummy left_cache; NodeLoader left(left_dataset, left_cache, space); - octree::cache::Dummy right_cache; + store::cache::Dummy right_cache; NodeLoader right(right_dataset, right_cache, space); NodeWriter output(output_dataset); if (mask.has_value()) { diff --git a/src/sf_merger/merge/NodeData.h b/src/sf_merger/merge/NodeData.h index c3c9de13..2bdd72f1 100644 --- a/src/sf_merger/merge/NodeData.h +++ b/src/sf_merger/merge/NodeData.h @@ -6,18 +6,17 @@ #include "log.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/Storage.h" #include "optional_utils.h" +#include "store/NodeStatusOrMissing.h" namespace merge { -template +template class NodeData { public: constexpr explicit NodeData(octree::Id id, const NodeLoader &loader) : _id(id), _loader(loader) {} - static constexpr octree::NodeStatusOrMissing status() { + static constexpr store::NodeStatusOrMissing status() { return Status; } @@ -26,8 +25,8 @@ class NodeData { } // Status::Leaf or Status::Inner guarantuess a node is present - template - std::enable_if_t + template + std::enable_if_t mesh() const { auto mesh = this->load_mesh(); if (mesh.has_value()) { @@ -38,8 +37,8 @@ class NodeData { } // Status::Missing and Status::Virtual do not exist on disk, but may be the child of a leaf or inner node - template - std::enable_if_t>> + template + std::enable_if_t>> mesh() const { return this->load_mesh(); } diff --git a/src/sf_merger/merge/visitor/Masked.h b/src/sf_merger/merge/visitor/Masked.h index 69936bfd..3c095c40 100644 --- a/src/sf_merger/merge/visitor/Masked.h +++ b/src/sf_merger/merge/visitor/Masked.h @@ -11,7 +11,7 @@ #include "mesh/merging/helpers.h" #include "mesh/merging/mapping.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" #include "polygon/Polygon.h" #include "polygon/triangulate.h" #include "spatial_lookup/Hashmap.h" @@ -26,7 +26,7 @@ namespace merge::visitor { class Masked { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; struct Context { MeshMask mask; bool has_left_parent; diff --git a/src/sf_merger/merge/visitor/Simple.h b/src/sf_merger/merge/visitor/Simple.h index 531cd686..9305ce7b 100644 --- a/src/sf_merger/merge/visitor/Simple.h +++ b/src/sf_merger/merge/visitor/Simple.h @@ -5,13 +5,13 @@ #include "merge/visitor/Visitor.h" #include "mesh/merge.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" namespace merge::visitor { class Simple { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; struct Context{}; using Result = merge::Result; diff --git a/src/sf_merger/merge/visitor/Visitor.h b/src/sf_merger/merge/visitor/Visitor.h index 35206fe4..6607e222 100644 --- a/src/sf_merger/merge/visitor/Visitor.h +++ b/src/sf_merger/merge/visitor/Visitor.h @@ -3,7 +3,7 @@ #include "merge/NodeData.h" #include "merge/Result.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" namespace merge { template @@ -15,10 +15,10 @@ concept Visitor = requires(T t, const octree::Id &id) { { t.make_root_context() } -> std::same_as; { - t.template visit( + t.template visit( id, - std::declval &>(), - std::declval &>(), + std::declval &>(), + std::declval &>(), std::declval()) } -> std::same_as>; }; diff --git a/src/terrainlib/octree/IndexMap.h b/src/terrainlib/octree/IndexMap.h deleted file mode 100644 index c45d7da1..00000000 --- a/src/terrainlib/octree/IndexMap.h +++ /dev/null @@ -1,94 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/NodeStatus.h" -#include "octree/StoreTraits.h" -#include "store/Index.h" - -namespace octree { - -class IndexMap { -public: - using SharedIndex = store::Index; - using iterator = SharedIndex::iterator; - using const_iterator = SharedIndex::const_iterator; - - IndexMap() = default; - explicit IndexMap(SharedIndex index) : _index(std::move(index)) {} - - std::optional get(const Id id) const { - return _index.get(id).value(); - } - bool add(const Id id) { - return _index.add(id).value(); - } - bool remove(const Id id) { - return _index.remove(id).value(); - } - bool is_present(const Id id) const { - return _index.is_present(id).value(); - } - bool is_absent(const Id id) const { - return _index.is_absent(id).value(); - } - bool is(const NodeStatus status, const Id id) const { - return _index.is(status, id).value(); - } - - void clear() { - _index.clear(); - } - bool empty() const { - return _index.empty(); - } - size_t size() const { - return _index.size(); - } - - const_iterator begin() const { - return _index.begin(); - } - const_iterator end() const { - return _index.end(); - } - const_iterator cbegin() const { - return _index.cbegin(); - } - const_iterator cend() const { - return _index.cend(); - } - - NodeStatus *get_raw(const Id id) { - return _index.get_raw(id).value(); - } - const NodeStatus *get_raw(const Id id) const { - return _index.get_raw(id).value(); - } - void set_raw(const Id id, const NodeStatus status) { - _index.set_raw(id, status).value(); - } - void remove_raw(const Id id) { - _index.remove_raw(id).value(); - } - - SharedIndex &shared() { - return _index; - } - const SharedIndex &shared() const { - return _index; - } - SharedIndex take_shared() && { - return std::move(_index); - } - - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; - -private: - SharedIndex _index; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/NodeStatus.h b/src/terrainlib/octree/NodeStatus.h deleted file mode 100644 index 185f4fdd..00000000 --- a/src/terrainlib/octree/NodeStatus.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#include "store/NodeStatus.h" - -namespace octree { -using NodeStatus = store::NodeStatus; -} diff --git a/src/terrainlib/octree/NodeStatusOrMissing.h b/src/terrainlib/octree/NodeStatusOrMissing.h deleted file mode 100644 index 1c71a67d..00000000 --- a/src/terrainlib/octree/NodeStatusOrMissing.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#include "store/NodeStatusOrMissing.h" - -namespace octree { -using NodeStatusOrMissing = store::NodeStatusOrMissing; -} diff --git a/src/terrainlib/octree/Storage.h b/src/terrainlib/octree/Storage.h deleted file mode 100644 index adb5181e..00000000 --- a/src/terrainlib/octree/Storage.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "mesh/storage.h" - -namespace octree { - -using MeshStorage = mesh::storage::Storage; -using IndexedMeshStorage = mesh::storage::IndexedStorage; -using Storage = MeshStorage; -using IndexedStorage = IndexedMeshStorage; -using StorageSettings = store::StorageSettings; - -} // namespace octree diff --git a/src/terrainlib/octree/disk/IndexFile.h b/src/terrainlib/octree/disk/IndexFile.h index 478be69f..92baacd9 100644 --- a/src/terrainlib/octree/disk/IndexFile.h +++ b/src/terrainlib/octree/disk/IndexFile.h @@ -4,7 +4,8 @@ #include -#include "octree/IndexMap.h" +#include "octree/StoreTraits.h" +#include "store/Index.h" namespace octree::disk::v1 { @@ -20,6 +21,6 @@ struct IndexFile { std::string layout_strategy_id; std::string preferred_extension; - IndexMap map; + store::Index map; }; } diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 7d70fbde..b296a755 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -5,7 +5,6 @@ #include #include "io/serialize.h" -#include "octree/IndexMap.h" #include "octree/StoreTraits.h" #include "octree/disk/IndexFile.h" #include "octree/store_layout/Mappings.h" @@ -27,7 +26,7 @@ read_index_file(const std::filesystem::path &path) { } disk::v1::IndexFile file = std::move(result.value()); return store::IndexMetadata{ - std::move(file.map).take_shared(), + std::move(file.map), std::move(file.layout_strategy_id), std::move(file.preferred_extension), }; @@ -39,7 +38,7 @@ inline std::expected write_index_file( disk::v1::IndexFile file; file.layout_strategy_id = metadata.layout_id; file.preferred_extension = metadata.codec_selector; - file.map = IndexMap(metadata.index); + file.map = metadata.index; const auto result = io::write_to_path(file, path); if (!result.has_value()) { return std::unexpected(store::IndexFormatError{ diff --git a/src/terrainlib/octree/storage/MeshStorage.h b/src/terrainlib/octree/storage/MeshStorage.h deleted file mode 100644 index e2d8b585..00000000 --- a/src/terrainlib/octree/storage/MeshStorage.h +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include "octree/Storage.h" diff --git a/src/terrainlib/octree/storage/cache/Dummy.h b/src/terrainlib/octree/storage/cache/Dummy.h deleted file mode 100644 index 159bc32d..00000000 --- a/src/terrainlib/octree/storage/cache/Dummy.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "octree/StoreTraits.h" -#include "store/cache/Dummy.h" - -namespace octree::cache { - -template -using Dummy = store::cache::Dummy; - -} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/cache/ICache.h b/src/terrainlib/octree/storage/cache/ICache.h deleted file mode 100644 index b14bd0b0..00000000 --- a/src/terrainlib/octree/storage/cache/ICache.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "octree/StoreTraits.h" -#include "store/cache/Interface.h" - -namespace octree::cache { - -template -using ICache = store::cache::Interface; - -} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/cache/LruCache.h b/src/terrainlib/octree/storage/cache/LruCache.h deleted file mode 100644 index 167f19d2..00000000 --- a/src/terrainlib/octree/storage/cache/LruCache.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "octree/StoreTraits.h" -#include "store/cache/Lru.h" - -namespace octree::cache { - -template -using Lru = store::cache::Lru; - -} // namespace octree::cache diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h index b8f1a6d0..c15b1a0b 100644 --- a/src/terrainlib/octree/storage/open.h +++ b/src/terrainlib/octree/storage/open.h @@ -4,25 +4,24 @@ #include #include "mesh/storage.h" -#include "octree/Storage.h" namespace octree { using OpenOptions = mesh::storage::OpenOptions; -inline std::expected> open_index( +inline std::expected> open_index( const std::filesystem::path &path) { return mesh::storage::open_index(path); } -inline std::expected> open_folder( +inline std::expected> open_folder( const std::filesystem::path &path, const bool create_index = false, OpenOptions options = {}) { return mesh::storage::open_folder(path, create_index, std::move(options)); } -inline std::expected> open_folder_indexed( +inline std::expected> open_folder_indexed( const std::filesystem::path &path, OpenOptions options = {}) { return mesh::storage::open_folder_indexed(path, std::move(options)); diff --git a/src/terrainlib/octree/traverse.h b/src/terrainlib/octree/traverse.h deleted file mode 100644 index cc5b9435..00000000 --- a/src/terrainlib/octree/traverse.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "octree/IndexMap.h" -#include "octree/StoreTraits.h" -#include "store/traverse.h" - -namespace octree { - -using TraversalOrder = store::TraversalOrder; - -constexpr bool always_refine(const Id &) { - return true; -} - -template> -void traverse( - const IndexMap &index, - VisitFn &&visit, - RefineFn &&refine = always_refine, - const Id &root = Id::root(), - const TraversalOrder order = TraversalOrder::DepthFirst) { - const auto result = store::traverse( - index.shared(), - std::forward(visit), - std::forward(refine), - root, - order); - DEBUG_ASSERT(result.has_value()); -} - -template> -void traverse( - const store::Index &index, - VisitFn &&visit, - RefineFn &&refine = always_refine, - const Id &root = Id::root(), - const TraversalOrder order = TraversalOrder::DepthFirst) { - const auto result = store::traverse( - index, - std::forward(visit), - std::forward(refine), - root, - order); - DEBUG_ASSERT(result.has_value()); -} - -} // namespace octree diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h index 8ee9a06c..157d9852 100644 --- a/src/terrainlib/sf/finalize_storage.h +++ b/src/terrainlib/sf/finalize_storage.h @@ -4,13 +4,13 @@ #include -#include "octree/Storage.h" +#include "mesh/storage.h" #include "sf/Error.h" #include "sf/validate_index.h" namespace sf { -inline std::expected finalize_storage(octree::Storage &storage) { +inline std::expected finalize_storage(mesh::storage::Storage &storage) { const auto save_result = storage.save_or_create_index(); if (!save_result.has_value()) { return std::unexpected(FinalizeError(save_result.error())); diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 3cb01c33..9853fe1a 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -37,8 +37,6 @@ add_executable(unittests_terrainlib terrainlib/store_layout.cpp terrainlib/store_storage.cpp terrainlib/store_traverse.cpp - terrainlib/index_map.cpp - terrainlib/index_traverse.cpp terrainlib/mesh_boundary.cpp terrainlib/mesh_bounds.cpp terrainlib/mesh_cleanup.cpp diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index c2854a22..22545e2f 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -11,7 +11,7 @@ #include "io/serialize.h" #include "octree/disk/IndexFile.h" #include "octree/storage/open.h" -#include "octree/traverse.h" +#include "store/traverse.h" #include "storage.h" #include "store/codec/ZppBits.h" #include "thread_safe_storage.h" @@ -53,16 +53,18 @@ TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { REQUIRE(index_file.has_value()); CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); CHECK(index_file->preferred_extension == ".bin"); - CHECK(index_file->map.is(octree::NodeStatus::Inner, root)); - CHECK(index_file->map.is(octree::NodeStatus::Virtual, parent)); - CHECK(index_file->map.is(octree::NodeStatus::Leaf, leaf)); + CHECK(index_file->map.is(store::NodeStatus::Inner, root).value()); + CHECK(index_file->map.is(store::NodeStatus::Virtual, parent).value()); + CHECK(index_file->map.is(store::NodeStatus::Leaf, leaf).value()); CHECK(std::filesystem::is_regular_file(path / "0/0/0/0.bin")); CHECK(std::filesystem::is_regular_file(path / "2/2/3/1.bin")); std::vector visited; - octree::traverse(index_file->map, [&](const octree::Id id, const octree::NodeStatus) { - visited.push_back(id); - }); + REQUIRE(store::traverse( + index_file->map, + [&](const octree::Id id, const store::NodeStatus) { + visited.push_back(id); + }).has_value()); CHECK(visited == std::vector{root, parent, leaf}); auto batch_storage_result = dag::storage::open_folder_indexed(path); diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp index cbc96ed6..26f5eba7 100644 --- a/unittests/sf_merger/storage_boundaries.cpp +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -66,13 +66,13 @@ mesh::Simple clipping_box() { }); } -octree::IndexedStorage indexed_storage(const std::filesystem::path &path) { +mesh::storage::IndexedStorage indexed_storage(const std::filesystem::path &path) { auto result = octree::open_folder_indexed(path); REQUIRE(result.has_value()); return std::move(result.value()); } -octree::Storage unindexed_storage(const std::filesystem::path &path) { +mesh::storage::Storage unindexed_storage(const std::filesystem::path &path) { auto result = octree::open_folder(path); REQUIRE(result.has_value()); return std::move(result.value()); diff --git a/unittests/terrainlib/index_map.cpp b/unittests/terrainlib/index_map.cpp deleted file mode 100644 index 5d595c23..00000000 --- a/unittests/terrainlib/index_map.cpp +++ /dev/null @@ -1,100 +0,0 @@ -#include "../catch2_helpers.h" - -#include "octree/IndexMap.h" - -using namespace octree; - -TEST_CASE("IndexMap basic operations") { - IndexMap index; - - SECTION("Add root node") { - Id root = Id::root(); - index.add(root); - CHECK(index.is_present(root)); - CHECK(index.is(NodeStatus::Leaf, root)); - CHECK(index.get(root) == std::optional(NodeStatus::Leaf)); - } - - SECTION("Add child node creates virtual parent") { - Id root = Id::root(); - Id child = root.child(3).value(); - index.add(child); - CHECK(index.is_present(root)); - CHECK(index.is(NodeStatus::Virtual, root)); - CHECK(index.is_present(child)); - CHECK(index.is(NodeStatus::Leaf, child)); - } - - SECTION("Add child note promotes parent to inner") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - index.add(child1); - index.add(root); - CHECK(index.is(NodeStatus::Leaf, child1)); - CHECK(index.is(NodeStatus::Inner, root)); - } - - SECTION("Removing a leaf node cleans up correctly") { - Id root = Id::root(); - Id child = root.child(2).value(); - index.add(child); - CHECK(index.is_present(child)); - index.remove(child); - CHECK(index.is_absent(child)); - CHECK(index.is_absent(root)); - } - - SECTION("Removing one of two children of an inner keeps it inner") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - Id child2 = root.child(1).value(); - index.add(root); - index.add(child1); - index.add(child2); - index.remove(child1); - - CHECK(index.is_absent(child1)); - CHECK(index.is(NodeStatus::Leaf, child2)); - CHECK(index.is(NodeStatus::Inner, root)); - } - - SECTION("Removing last child of inner converts it to leaf") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - index.add(child1); - index.add(root); - - index.remove(child1); - - CHECK(index.is_absent(child1)); - CHECK(index.is(NodeStatus::Leaf, root)); - } - - SECTION("Clear works") { - Id a = Id::root().child(1).value(); - Id b = Id::root().child(2).value(); - index.add(a); - index.add(b); - CHECK(index.size() == 3); // root + a + b - index.clear(); - CHECK(index.empty()); - } - - SECTION("Serialization round-trip") { - IndexMap original; - original.add(Id::root().child(3).value()); - - const std::errc success {}; - - std::vector buffer; - zpp::bits::out out(buffer); - CHECK(out(original).code == success); - - IndexMap loaded; - zpp::bits::in in(buffer); - CHECK(in(loaded).code == success); - - CHECK(original.size() == loaded.size()); - CHECK(loaded.is_present(Id::root().child(3).value())); - } -} diff --git a/unittests/terrainlib/index_traverse.cpp b/unittests/terrainlib/index_traverse.cpp deleted file mode 100644 index 5ada7575..00000000 --- a/unittests/terrainlib/index_traverse.cpp +++ /dev/null @@ -1,132 +0,0 @@ -#include -#include -#include -#include - -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/traverse.h" - -using namespace octree; - -TEST_CASE("octree::traverse basic") { - IndexMap index; - - const Id root = Id::root(); - index.add(root); - - const auto children = root.children().value(); - for (const auto& child : children) { - index.add(child); - } - - SECTION("traversal visits root then children") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return true; }, // refine always - root, - TraversalOrder::DepthFirst); - - CHECK(visited.front() == root); - CHECK(visited.size() == 9); // root + 8 children - std::vector expected(children.begin(), children.end()); - visited.erase(visited.begin()); - CHECK(visited == expected); - } - - SECTION("Refine function disables recursion") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return false; }, // don't refine - root, - TraversalOrder::DepthFirst); - - CHECK(visited.size() == 1); - CHECK(visited.front() == root); - } - - SECTION("Traversal skips when root is missing") { - IndexMap empty; - bool visited = false; - - traverse( - empty, - [&](const Id&, const NodeStatus&) { - visited = true; - }, - [](const Id&) { return true; }, - root, - TraversalOrder::DepthFirst - ); - - CHECK_FALSE(visited); - } -} - -TEST_CASE("octree::traverse with depth") { - IndexMap index; - - const Id root = Id::root(); - index.add(root); - - const auto children = root.children().value(); - const Id c0 = children[0]; - const Id c1 = children[1]; - const Id c2 = children[2]; - - index.add(c0); - index.add(c1); - index.add(c2); - - // Add grandchildren - const Id gc0 = c0.child(0).value(); // grandchild of c0 - const Id gc1 = c2.child(1).value(); // grandchild of c2 - - index.add(gc0); - index.add(gc1); - - SECTION("DFS visits deeply before siblings") { - std::vector visited; - traverse( - index, - [&](const Id& id, const NodeStatus&) { - visited.push_back(id); - }, - [](const Id&) { return true; }, - root, - TraversalOrder::DepthFirst - ); - - CHECK(visited.size() == 6); - - // DFS order should be: root, c0, gc0, c1, c2, gc1 - std::vector expected{root, c0, gc0, c1, c2, gc1}; - CHECK(visited == expected); - } - - SECTION("BFS visits all nodes level by level") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return true; }, - root, - TraversalOrder::BreadthFirst); - - CHECK(visited.size() == 6); - - // BFS order should be: root, c0, c1, c2, gc0, gc1 - std::vector expected{root, c0, c1, c2, gc0, gc1}; - CHECK(visited == expected); - } -} diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 40a3f61f..574ef8fc 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -11,14 +11,13 @@ #include "io/serialize.h" #include "io/bytes.h" #include "mesh/SimpleMesh.h" -#include "octree/Storage.h" #include "octree/disk/IndexFile.h" #include "mesh/codec/from_extension.h" #include "mesh/storage.h" #include "octree/store_layout/Mappings.h" #include "octree/storage/IndexFile.h" #include "octree/storage/open.h" -#include "octree/traverse.h" +#include "store/traverse.h" namespace { @@ -56,18 +55,18 @@ mesh::Simple triangle_mesh(const double offset) { {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); } -octree::IndexedStorage make_storage( +mesh::storage::IndexedStorage make_storage( const std::filesystem::path &path, const store::PathMapping mapping, const std::string &extension) { auto codec = mesh::codec::from_extension(extension); REQUIRE(codec.has_value()); - return octree::IndexedStorage( + return mesh::storage::IndexedStorage( store::RawStorage( store::Layout(path, mapping), std::move(codec.value())), store::Index{}, - octree::Storage::Persistence{ + mesh::storage::Storage::Persistence{ octree::storage::index_format(), path / "terrain.index", std::string(mapping.id), @@ -87,12 +86,12 @@ TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { CHECK(index_file->layout_strategy_id == "flat"); CHECK(index_file->preferred_extension == ".terrain"); CHECK(index_file->map.size() == 1); - CHECK(index_file->map.is(octree::NodeStatus::Leaf, root)); + CHECK(index_file->map.is(store::NodeStatus::Leaf, root).value()); CHECK(std::filesystem::is_regular_file(path / "0-0.terrain")); auto storage_result = octree::open_folder_indexed(path); REQUIRE(storage_result.has_value()); - const octree::IndexedStorage storage = std::move(storage_result.value()); + const mesh::storage::IndexedStorage storage = std::move(storage_result.value()); const auto mesh = storage.load(root); REQUIRE(mesh.has_value()); CHECK(mesh->face_count() == 1); @@ -108,17 +107,19 @@ TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { REQUIRE(index_file.has_value()); CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); CHECK(index_file->preferred_extension == ".terrain"); - CHECK(index_file->map.is(octree::NodeStatus::Virtual, root)); - CHECK(index_file->map.is(octree::NodeStatus::Leaf, child)); - CHECK(index_file->map.is(octree::NodeStatus::Virtual, deep_parent)); - CHECK(index_file->map.is(octree::NodeStatus::Leaf, deep)); + CHECK(index_file->map.is(store::NodeStatus::Virtual, root).value()); + CHECK(index_file->map.is(store::NodeStatus::Leaf, child).value()); + CHECK(index_file->map.is(store::NodeStatus::Virtual, deep_parent).value()); + CHECK(index_file->map.is(store::NodeStatus::Leaf, deep).value()); CHECK(std::filesystem::is_regular_file(path / "1/0/1/0.terrain")); CHECK(std::filesystem::is_regular_file(path / "2/3/1/3.terrain")); std::vector visited; - octree::traverse(index_file->map, [&](const octree::Id id, const octree::NodeStatus) { - visited.push_back(id); - }); + REQUIRE(store::traverse( + index_file->map, + [&](const octree::Id id, const store::NodeStatus) { + visited.push_back(id); + }).has_value()); CHECK(visited == std::vector{octree::Id{root}, child, deep_parent, deep}); } } From ae87580b9eec19cebdc62a7da903c105bcabca6f Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:46:29 +0200 Subject: [PATCH 10/43] Remove migration compatibility fixtures --- docs/raster-store/refactor-status.md | 29 +++- .../dag_builder/storage_compatibility.cpp | 139 ++++++------------ .../raster-store-refactor/dag/0/0/0/0.bin | Bin 276 -> 0 bytes .../raster-store-refactor/dag/2/2/3/1.bin | Bin 276 -> 0 bytes .../raster-store-refactor/dag/terrain.index | Bin 78 -> 0 bytes .../sf-coordinates/1/0/1/0.terrain | Bin 100 -> 0 bytes .../sf-coordinates/2/3/1/3.terrain | Bin 100 -> 0 bytes .../sf-coordinates/terrain.index | Bin 92 -> 0 bytes .../raster-store-refactor/sf-flat/0-0.terrain | Bin 103 -> 0 bytes .../sf-flat/terrain.index | Bin 34 -> 0 bytes unittests/sf_builder/storage_finalization.cpp | 16 +- unittests/terrainlib/store_compatibility.cpp | 63 +------- unittests/terrainlib/store_layout.cpp | 17 +-- 13 files changed, 91 insertions(+), 173 deletions(-) delete mode 100644 unittests/data/raster-store-refactor/dag/0/0/0/0.bin delete mode 100644 unittests/data/raster-store-refactor/dag/2/2/3/1.bin delete mode 100644 unittests/data/raster-store-refactor/dag/terrain.index delete mode 100644 unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain delete mode 100644 unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain delete mode 100644 unittests/data/raster-store-refactor/sf-coordinates/terrain.index delete mode 100644 unittests/data/raster-store-refactor/sf-flat/0-0.terrain delete mode 100644 unittests/data/raster-store-refactor/sf-flat/terrain.index diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md index 7f4f4b27..3a82dc46 100644 --- a/docs/raster-store/refactor-status.md +++ b/docs/raster-store/refactor-status.md @@ -16,10 +16,10 @@ This file is the resumable implementation log for ## Current position -- Phase: 5 — Cleanup and documentation +- Phase: 6 — Remove migration-only compatibility fixtures - State: complete; completion commit pending. -- Next action: commit and tag Phase 5, then remove migration-only fixtures in - Phase 6. +- Next action: commit and tag Phase 6, then perform the final clean-worktree + and tag audit. ## Phase log @@ -131,6 +131,19 @@ This file is the resumable implementation log for tools and include final public names plus 3D-open and in-memory 2D examples. - State: complete. +### Phase 6 — Remove migration-only compatibility fixtures + +- Removed the golden pre-refactor SF and DAG indexes and payload files. +- Removed fixture-only SF/DAG opening and byte-comparison assertions. +- Converted DAG codec reentrancy, metadata-prefix reading, synchronized + storage, SF-builder finalization, and DAG validation tests to generate + current data in temporary directories. +- Retained layout/path, resolver, topology, storage, error, merge/cut, and + producer/consumer boundary coverage. +- Repository search and filesystem inspection find no remaining pre-refactor + fixture reference or fixture file. +- State: complete. + ## Verification log - Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. @@ -193,6 +206,12 @@ This file is the resumable implementation log for - Phase 5 repository searches found no remaining caller of a removed octree forwarding name and no `octree::Id` dependency under `terrainlib/store`. - Phase 5 `git diff --check`: passed. +- Phase 6 `unittests_terrainlib`: 23,997 assertions in 425 test cases passed. +- Phase 6 `unittests_dagbuilder`: 439 assertions in 73 test cases passed. +- Phase 6 `unittests_sfmerger`: 121 assertions in 8 test cases passed. +- Phase 6 SF builder finalization: 15 assertions in 2 test cases passed. +- Phase 6 fixture-reference search, fixture-file inspection, and + `git diff --check`: passed. ## Commit and tag log @@ -212,3 +231,7 @@ This file is the resumable implementation log for completion tag. - `3086336` / `refactor_phase_4` — hardened node reuse, SF topology enforcement, and Phase 4 completion tag. +- `bca0b5c` / `refactor_phase_5` — removed migration shims, documented final + public boundaries, and tagged Phase 5 completion. +- Phase 6 completion commit / `refactor_phase_6` — removed migration-only + fixtures while retaining current-data coverage. diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 22545e2f..bde98acb 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -5,13 +5,10 @@ #include -#include "encoded.h" #include "build.h" -#include "io/bytes.h" -#include "io/serialize.h" -#include "octree/disk/IndexFile.h" +#include "dag_node.h" +#include "mesh/SimpleMesh.h" #include "octree/storage/open.h" -#include "store/traverse.h" #include "storage.h" #include "store/codec/ZppBits.h" #include "thread_safe_storage.h" @@ -40,81 +37,40 @@ class TemporaryDirectory { std::filesystem::path _path; }; -} // namespace - -TEST_CASE("pre-refactor DAG fixture preserves index and payload contracts") { - const std::filesystem::path path = - std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor" / "dag"; - const octree::Id root = octree::Id::root(); - const octree::Id parent = root.child(3).value(); - const octree::Id leaf = parent.child(6).value(); - - const auto index_file = io::read_from_path(path / "terrain.index"); - REQUIRE(index_file.has_value()); - CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); - CHECK(index_file->preferred_extension == ".bin"); - CHECK(index_file->map.is(store::NodeStatus::Inner, root).value()); - CHECK(index_file->map.is(store::NodeStatus::Virtual, parent).value()); - CHECK(index_file->map.is(store::NodeStatus::Leaf, leaf).value()); - CHECK(std::filesystem::is_regular_file(path / "0/0/0/0.bin")); - CHECK(std::filesystem::is_regular_file(path / "2/2/3/1.bin")); - - std::vector visited; - REQUIRE(store::traverse( - index_file->map, - [&](const octree::Id id, const store::NodeStatus) { - visited.push_back(id); - }).has_value()); - CHECK(visited == std::vector{root, parent, leaf}); - - auto batch_storage_result = dag::storage::open_folder_indexed(path); - REQUIRE(batch_storage_result.has_value()); - const auto batch_storage = std::move(batch_storage_result.value()); - const auto batch = batch_storage.load(root); - REQUIRE(batch.has_value()); - CHECK(batch->metadata.group_assignment == std::vector{0}); - REQUIRE(batch->metadata.groups.size() == 1); - CHECK(batch->metadata.groups.front().error == 12.5); - CHECK(batch->metadata.groups.front().children == std::vector{{root.child(4).value(), 3}}); - REQUIRE(batch->clustering.clusters.size() == 1); - CHECK(batch->clustering.clusters.front().id == 17); - - auto metadata_storage_result = dag::storage::open_metadata_indexed(path); - REQUIRE(metadata_storage_result.has_value()); - const auto metadata_storage = std::move(metadata_storage_result.value()); - const auto metadata = metadata_storage.load(root); - REQUIRE(metadata.has_value()); - CHECK(metadata->group_assignment == batch->metadata.group_assignment); - REQUIRE(metadata->groups.size() == batch->metadata.groups.size()); - CHECK(metadata->groups.front().error == batch->metadata.groups.front().error); - - const auto encoded = io::write_to_bytes(batch.value()); - const auto fixture_bytes = io::read_bytes_from_path(path / "0/0/0/0.bin"); - REQUIRE(encoded.has_value()); - REQUIRE(fixture_bytes.has_value()); - CHECK(encoded.value() == fixture_bytes.value()); - - store::codec::ZppBits runtime_codec; - const auto runtime_batch = runtime_codec.read(store::NodePath(path / "0/0/0/0")); - REQUIRE(runtime_batch.has_value()); - CHECK(runtime_batch->metadata.group_assignment == batch->metadata.group_assignment); - CHECK(runtime_codec.paths(store::NodePath(path / "0/0/0/0")) - == std::vector{path / "0/0/0/0.bin"}); +dag::ClusterBatch sample_batch() { + dag::ClusterBatch batch; + batch.metadata.group_assignment = {0}; + batch.metadata.groups = {{ + .children = {{octree::Id::root().child(4).value(), 3}}, + .error = 12.5, + .bounds = {}, + }}; + batch.clustering.positions = { + {0.0, 0.0, 0.0}, + {1.0, 0.0, 0.0}, + {0.0, 1.0, 0.0}, + }; + batch.clustering.clusters = {{ + .id = 17, + .vertex_indices = {0, 1, 2}, + .local_triangles = {{0, 1, 2}}, + .uvs = {}, + .texture_id = std::nullopt, + .absolute_error = 0.0, + }}; + return batch; +} - TemporaryDirectory output; - const store::NodePath output_path(output.path() / "nested/node"); - REQUIRE(runtime_codec.write(output_path, batch.value()).has_value()); - const auto runtime_bytes = io::read_bytes_from_path(output.path() / "nested/node.bin"); - REQUIRE(runtime_bytes.has_value()); - CHECK(runtime_bytes.value() == fixture_bytes.value()); +mesh::Simple sample_mesh() { + return mesh::Simple( + {{0, 1, 2}}, + {{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}}); } +} // namespace + TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { - const std::filesystem::path fixture = - std::filesystem::path(ALP_TEST_DATA_DIR) - / "raster-store-refactor/dag/0/0/0/0.bin"; - const auto batch = io::read_from_path(fixture); - REQUIRE(batch.has_value()); + const dag::ClusterBatch batch = sample_batch(); TemporaryDirectory output; store::codec::ZppBits codec; @@ -122,12 +78,12 @@ TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { for (int index = 0; index < 8; ++index) { operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { const store::NodePath path(output.path() / std::to_string(index) / "node"); - if (!codec.write(path, batch.value()).has_value()) { + if (!codec.write(path, batch).has_value()) { return false; } const auto loaded = codec.read(path); return loaded.has_value() - && loaded->metadata.group_assignment == batch->metadata.group_assignment; + && loaded->metadata.group_assignment == batch.metadata.group_assignment; })); } for (auto &operation : operations) { @@ -156,17 +112,13 @@ TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[stor } TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") { - const std::filesystem::path fixture = - std::filesystem::path(ALP_TEST_DATA_DIR) - / "raster-store-refactor/dag/0/0/0/0.bin"; - const auto batch = io::read_from_path(fixture); - REQUIRE(batch.has_value()); + const dag::ClusterBatch batch = sample_batch(); TemporaryDirectory output; auto storage_result = dag::storage::open_folder_indexed(output.path()); REQUIRE(storage_result.has_value()); dag::ThreadSafeStorage synchronized(std::move(storage_result.value())); - REQUIRE(synchronized.save(octree::Id::root(), batch.value()).has_value()); + REQUIRE(synchronized.save(octree::Id::root(), batch).has_value()); auto released = std::move(synchronized).release(); REQUIRE(released.save_index().has_value()); @@ -176,24 +128,25 @@ TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[s REQUIRE(reopened_result.has_value()); CHECK(reopened_result->has(octree::Id::root()).value()); CHECK(reopened_result->load(octree::Id::root()).has_value()); + + auto metadata_result = dag::storage::open_metadata_indexed(output.path()); + REQUIRE(metadata_result.has_value()); + const auto metadata = metadata_result->load(octree::Id::root()); + REQUIRE(metadata.has_value()); + CHECK(metadata->group_assignment == batch.metadata.group_assignment); + CHECK(metadata->groups.front().error == batch.metadata.groups.front().error); } TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") { - const std::filesystem::path fixture_path = - std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor/sf-flat"; - auto fixture_result = octree::open_folder_indexed(fixture_path); - REQUIRE(fixture_result.has_value()); - const auto mesh_result = fixture_result->load(octree::Id::root()); - REQUIRE(mesh_result.has_value()); - TemporaryDirectory input_directory; TemporaryDirectory output_directory; auto input_result = octree::open_folder_indexed(input_directory.path()); REQUIRE(input_result.has_value()); auto input = std::move(input_result.value()); const octree::Id root = octree::Id::root(); - REQUIRE(input.save(root, mesh_result.value()).has_value()); - REQUIRE(input.save(root.child(0).value(), mesh_result.value()).has_value()); + const mesh::Simple mesh = sample_mesh(); + REQUIRE(input.save(root, mesh).has_value()); + REQUIRE(input.save(root.child(0).value(), mesh).has_value()); REQUIRE(input.index().is(store::NodeStatus::Inner, root).value()); auto output_result = dag::storage::open_folder_indexed(output_directory.path()); diff --git a/unittests/data/raster-store-refactor/dag/0/0/0/0.bin b/unittests/data/raster-store-refactor/dag/0/0/0/0.bin deleted file mode 100644 index 7c8b1d28a199bea0ce758cdcc3ae51b44eed5bea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 276 zcmZQ%fB;4y1;#8;0cI%8z@X^>VSd;T6*>T+IiR$P1B7n^rR^MGdi8)33s>y|K?Vm2 z1`P=Y1rdna5B3n60jh_?0c<46;Dc dg{v8um|)7#X-SX*bU_$j5XN9&xBzn!0|33_7i<6k diff --git a/unittests/data/raster-store-refactor/dag/2/2/3/1.bin b/unittests/data/raster-store-refactor/dag/2/2/3/1.bin deleted file mode 100644 index 7c8b1d28a199bea0ce758cdcc3ae51b44eed5bea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 276 zcmZQ%fB;4y1;#8;0cI%8z@X^>VSd;T6*>T+IiR$P1B7n^rR^MGdi8)33s>y|K?Vm2 z1`P=Y1rdna5B3n60jh_?0c<46;Dc dg{v8um|)7#X-SX*bU_$j5XN9&xBzn!0|33_7i<6k diff --git a/unittests/data/raster-store-refactor/dag/terrain.index b/unittests/data/raster-store-refactor/dag/terrain.index deleted file mode 100644 index e3f1d112fe64250be60fa4d32d0b944f214aaca5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 78 zcmY#jU|`5eElbUbPs~e+PtMOTO3BPiEJ=+|$t+4uF3B&-Of6;ss?kfz%wq=9OmaX1 P3>ZON5MY8(K#~yvLw642 diff --git a/unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain b/unittests/data/raster-store-refactor/sf-coordinates/1/0/1/0.terrain deleted file mode 100644 index 8b7a579c003dd10778221743462a9aab1ae525ff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 100 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^=mA9+uG+Qg*Q!+vEI=*t Q3=DNZAs|q308=;#08EYz_5c6? diff --git a/unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain b/unittests/data/raster-store-refactor/sf-coordinates/2/3/1/3.terrain deleted file mode 100644 index bebf97b788532b2663c16a2becfe57e15b2126d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 100 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^=mA9+uG+Qg*Q!+vOh9Gw Q3=DNZAs{eu08=;#08Q8q0ssI2 diff --git a/unittests/data/raster-store-refactor/sf-coordinates/terrain.index b/unittests/data/raster-store-refactor/sf-coordinates/terrain.index deleted file mode 100644 index 546dd8bfb81ff88d2e18776d6159eb841eac8f5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 92 zcmY#jU|`5eElbUbPs~e+PtMOTO3BPiEJ=+|$t+4uF3B&-OfBXBs?jS+EhSd6SNA(#jQ698sS5W4^X diff --git a/unittests/data/raster-store-refactor/sf-flat/0-0.terrain b/unittests/data/raster-store-refactor/sf-flat/0-0.terrain deleted file mode 100644 index 8bc0949026e08a52b88a5f372d02ddf2a804eb07..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 103 zcmZQ%U|?VdVlEK~%?M-*0rA5R3}x+M=@l#6QadwdFlI6^7y?BXuKERps~F@J80u;n N7~~lk>VTp|006Aq4+sDN diff --git a/unittests/data/raster-store-refactor/sf-flat/terrain.index b/unittests/data/raster-store-refactor/sf-flat/terrain.index deleted file mode 100644 index c581a3f6f1d99d289ca268fad4d3421bf7cf191a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34 ecmZQ!U|>kgNi5+2(t0JSMMa63d5l0d0ssJM*#y`C diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp index 3ef11068..229b2e3b 100644 --- a/unittests/sf_builder/storage_finalization.cpp +++ b/unittests/sf_builder/storage_finalization.cpp @@ -33,14 +33,10 @@ class TemporaryDirectory { std::filesystem::path _path; }; -mesh::Simple fixture_mesh() { - const std::filesystem::path fixture_path = - std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor/sf-flat"; - auto fixture = octree::open_folder_indexed(fixture_path); - REQUIRE(fixture.has_value()); - auto loaded = fixture->load(octree::Id::root()); - REQUIRE(loaded.has_value()); - return std::move(loaded.value()); +mesh::Simple sample_mesh() { + return mesh::Simple( + {{0, 1, 2}}, + {{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}}); } } // namespace @@ -50,7 +46,7 @@ TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuil auto storage_result = octree::open_folder(directory.path()); REQUIRE(storage_result.has_value()); auto storage = std::move(storage_result.value()); - REQUIRE(storage.save(octree::Id::root(), fixture_mesh()).has_value()); + REQUIRE(storage.save(octree::Id::root(), sample_mesh()).has_value()); CHECK(terrainbuilder::finalize_storage(storage).has_value()); CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); @@ -64,7 +60,7 @@ TEST_CASE( REQUIRE(storage_result.has_value()); auto storage = std::move(storage_result.value()); const octree::Id root = octree::Id::root(); - const mesh::Simple mesh = fixture_mesh(); + const mesh::Simple mesh = sample_mesh(); REQUIRE(storage.save(root, mesh).has_value()); REQUIRE(storage.save(root.child(0).value(), mesh).has_value()); diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 574ef8fc..79a1d3ad 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -17,14 +17,9 @@ #include "octree/store_layout/Mappings.h" #include "octree/storage/IndexFile.h" #include "octree/storage/open.h" -#include "store/traverse.h" namespace { -std::filesystem::path fixture_path(const std::string_view name) { - return std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor" / name; -} - class TemporaryDirectory { public: explicit TemporaryDirectory(const std::string_view label) { @@ -76,55 +71,7 @@ mesh::storage::IndexedStorage make_storage( } // namespace -TEST_CASE("pre-refactor SF fixtures preserve index and path contracts") { - const octree::Id root = octree::Id::root(); - - SECTION("flat") { - const std::filesystem::path path = fixture_path("sf-flat"); - const auto index_file = io::read_from_path(path / "terrain.index"); - REQUIRE(index_file.has_value()); - CHECK(index_file->layout_strategy_id == "flat"); - CHECK(index_file->preferred_extension == ".terrain"); - CHECK(index_file->map.size() == 1); - CHECK(index_file->map.is(store::NodeStatus::Leaf, root).value()); - CHECK(std::filesystem::is_regular_file(path / "0-0.terrain")); - - auto storage_result = octree::open_folder_indexed(path); - REQUIRE(storage_result.has_value()); - const mesh::storage::IndexedStorage storage = std::move(storage_result.value()); - const auto mesh = storage.load(root); - REQUIRE(mesh.has_value()); - CHECK(mesh->face_count() == 1); - } - - SECTION("level and coordinate directories") { - const std::filesystem::path path = fixture_path("sf-coordinates"); - const octree::Id child = root.child(2).value(); - const octree::Id deep = root.child(5).value().child(7).value(); - const octree::Id deep_parent = deep.parent().value(); - - const auto index_file = io::read_from_path(path / "terrain.index"); - REQUIRE(index_file.has_value()); - CHECK(index_file->layout_strategy_id == "level_and_coordinate_directories"); - CHECK(index_file->preferred_extension == ".terrain"); - CHECK(index_file->map.is(store::NodeStatus::Virtual, root).value()); - CHECK(index_file->map.is(store::NodeStatus::Leaf, child).value()); - CHECK(index_file->map.is(store::NodeStatus::Virtual, deep_parent).value()); - CHECK(index_file->map.is(store::NodeStatus::Leaf, deep).value()); - CHECK(std::filesystem::is_regular_file(path / "1/0/1/0.terrain")); - CHECK(std::filesystem::is_regular_file(path / "2/3/1/3.terrain")); - - std::vector visited; - REQUIRE(store::traverse( - index_file->map, - [&](const octree::Id id, const store::NodeStatus) { - visited.push_back(id); - }).has_value()); - CHECK(visited == std::vector{octree::Id{root}, child, deep_parent, deep}); - } -} - -TEST_CASE("pre-refactor layouts round-trip boundary IDs") { +TEST_CASE("octree layouts round-trip boundary IDs") { const std::vector ids{ octree::Id::root(), octree::Id(octree::Id::max_level(), octree::Id::Index{0}), @@ -142,7 +89,7 @@ TEST_CASE("pre-refactor layouts round-trip boundary IDs") { } } -TEST_CASE("pre-refactor storage hard-links matching payload formats") { +TEST_CASE("storage hard-links matching payload formats") { TemporaryDirectory source_directory("hard-link-source"); TemporaryDirectory target_directory("hard-link-target"); auto source = make_storage( @@ -162,7 +109,7 @@ TEST_CASE("pre-refactor storage hard-links matching payload formats") { REQUIRE(target.save_index().has_value()); } -TEST_CASE("pre-refactor storage re-encodes differing payload formats") { +TEST_CASE("storage re-encodes differing payload formats") { TemporaryDirectory source_directory("reencode-source"); TemporaryDirectory target_directory("reencode-target"); auto source = make_storage( @@ -187,7 +134,7 @@ TEST_CASE("pre-refactor storage re-encodes differing payload formats") { REQUIRE(target.save_index().has_value()); } -TEST_CASE("pre-refactor storage supports enabled overwrites") { +TEST_CASE("storage supports enabled overwrites") { TemporaryDirectory directory("overwrite"); auto storage = make_storage( directory.path(), @@ -205,7 +152,7 @@ TEST_CASE("pre-refactor storage supports enabled overwrites") { REQUIRE(storage.save_index().has_value()); } -TEST_CASE("pre-refactor folder opening scans payloads and creates an index") { +TEST_CASE("folder opening scans payloads and creates an index") { TemporaryDirectory directory("folder-open"); octree::OpenOptions options; options.default_mapping = octree::store_layout::flat(); diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp index ec4eaa4b..736df506 100644 --- a/unittests/terrainlib/store_layout.cpp +++ b/unittests/terrainlib/store_layout.cpp @@ -7,29 +7,28 @@ #include "octree/store_layout/Mappings.h" #include "store/Layout.h" -TEST_CASE("extensionless octree mappings preserve physical fixture paths", "[store][layout]") { - const std::filesystem::path fixtures = - std::filesystem::path(ALP_TEST_DATA_DIR) / "raster-store-refactor"; +TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layout]") { + const std::filesystem::path dataset = "dataset"; mesh::codec::Terrain terrain; SECTION("flat") { - const store::Layout layout(fixtures / "sf-flat", octree::store_layout::flat()); + const store::Layout layout(dataset / "sf-flat", octree::store_layout::flat()); const store::NodePath node_path = layout.node_path(octree::Id::root()); - CHECK(node_path.path() == fixtures / "sf-flat/0-0"); - CHECK(terrain.paths(node_path) == std::vector{fixtures / "sf-flat/0-0.terrain"}); + CHECK(node_path.path() == dataset / "sf-flat/0-0"); + CHECK(terrain.paths(node_path) == std::vector{dataset / "sf-flat/0-0.terrain"}); CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); } SECTION("level and coordinate directories") { const store::Layout layout( - fixtures / "sf-coordinates", + dataset / "sf-coordinates", octree::store_layout::level_and_coordinate_directories()); const octree::Id child = octree::Id::root().child(2).value(); const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); CHECK(terrain.paths(layout.node_path(child)) - == std::vector{fixtures / "sf-coordinates/1/0/1/0.terrain"}); + == std::vector{dataset / "sf-coordinates/1/0/1/0.terrain"}); CHECK(terrain.paths(layout.node_path(deep)) - == std::vector{fixtures / "sf-coordinates/2/3/1/3.terrain"}); + == std::vector{dataset / "sf-coordinates/2/3/1/3.terrain"}); CHECK(layout.key_from_node_path(layout.node_path(child)) == child); CHECK(layout.key_from_node_path(layout.node_path(deep)) == deep); } From c1f4bb48e88f9ca36b4d5ee9973b582b04304098 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 16 Aug 2026 15:53:00 +0200 Subject: [PATCH 11/43] Parameterize golden E2E paths --- docs/raster-store/golden-e2e.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/raster-store/golden-e2e.sh b/docs/raster-store/golden-e2e.sh index b919938c..894622cf 100755 --- a/docs/raster-store/golden-e2e.sh +++ b/docs/raster-store/golden-e2e.sh @@ -6,13 +6,13 @@ set -Eeuo pipefail -readonly SOURCE_DIR="/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store" -readonly BUILD_DIR="${SOURCE_DIR}/build/golden-e2e" -readonly RUN_ROOT="/data/scratch/codex/alpine-terrain-builder-golden-e2e" -readonly INPUT_ROOT="${RUN_ROOT}/inputs" -readonly REFERENCE_ROOT="${RUN_ROOT}/reference" -readonly LOG_ROOT="${RUN_ROOT}/logs" -readonly STATE_ROOT="${RUN_ROOT}/state" +readonly SOURCE_DIR="${SOURCE_DIR:-/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store}" +readonly BUILD_DIR="${BUILD_DIR:-${SOURCE_DIR}/build/golden-e2e}" +readonly RUN_ROOT="${RUN_ROOT:-/data/scratch/codex/alpine-terrain-builder-golden-e2e}" +readonly INPUT_ROOT="${INPUT_ROOT:-${RUN_ROOT}/inputs}" +readonly REFERENCE_ROOT="${REFERENCE_ROOT:-${RUN_ROOT}/reference}" +readonly LOG_ROOT="${LOG_ROOT:-${RUN_ROOT}/logs}" +readonly STATE_ROOT="${STATE_ROOT:-${RUN_ROOT}/state}" readonly RUN_ID="${RUN_ID:-$(date +%Y%m%dT%H%M%S)}" readonly RUN_LOG_ROOT="${LOG_ROOT}/runs/${RUN_ID}" readonly TIMINGS_FILE="${RUN_LOG_ROOT}/timings.tsv" @@ -25,7 +25,7 @@ readonly DAG_BUILDER="${BUILD_DIR}/src/dag_builder/dag-builder" # GT rasters. They are recreated from the raw datasets at the start of a run. readonly RAW_GS="/data/raw/raster_data/Oe_2020/OeRect_01m_gs_31287.img" readonly RAW_GT="/data/raw/raster_data/Oe_2020/OeRect_01m_gt_31287.img" -readonly ELEVATION_ROOT="${INPUT_ROOT}/grossglockner-elevation-4km" +readonly ELEVATION_ROOT="${ELEVATION_ROOT:-${INPUT_ROOT}/grossglockner-elevation-4km}" readonly GS_VRT="${ELEVATION_ROOT}/grossglockner-gs-4km.vrt" readonly GT_VRT="${ELEVATION_ROOT}/grossglockner-gt-4km.vrt" readonly BASEMAP_TILES="${INPUT_ROOT}/basemap" From b90a7a81a7e9aa13e038c44e317e18373e35c943 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:30:57 +0200 Subject: [PATCH 12/43] Fix stale SF tile ID test --- unittests/sf_builder/texture.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/sf_builder/texture.cpp b/unittests/sf_builder/texture.cpp index 06118437..3da052b4 100644 --- a/unittests/sf_builder/texture.cpp +++ b/unittests/sf_builder/texture.cpp @@ -351,7 +351,7 @@ TEST_CASE("texture assembler reports the content region", "[terrainbuilder]") { tile_image.row(row).setTo(uint8_t(row)); } - const radix::tile::Id root_tile(0, {0, 0}, radix::tile::Scheme::SlippyMap); + const radix::tile::Id root_tile(0, {0, 0}); const StaticTileProvider tile_provider({{root_tile, tile_image}}); const std::vector tiles_to_splatter = {root_tile}; From 008d47841767f211bdd6e2f104ab915e0f0af2ea Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:41:52 +0200 Subject: [PATCH 13/43] Add default Zstd envelope compression --- src/terrainlib/io/compression.cpp | 13 ++- src/terrainlib/io/compression.h | 1 + unittests/terrainlib/envelope.cpp | 171 ++++++++++++++++-------------- 3 files changed, 105 insertions(+), 80 deletions(-) diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index ba18e638..b3ee1e5f 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -63,6 +63,7 @@ std::expected validate_algorithms( switch (compression_algorithm) { case CompressionAlgorithm::None: case CompressionAlgorithm::ZstdBestCompressionWithChecksum: + case CompressionAlgorithm::ZstdDefaultCompressionWithChecksum: break; default: return std::unexpected(Error{ErrorCode::UnsupportedCompressionAlgorithm}); @@ -72,7 +73,8 @@ std::expected validate_algorithms( && (checksum_algorithm == ChecksumAlgorithm::None || checksum_algorithm == ChecksumAlgorithm::Crc32c); const bool zstd_with_checksum = - compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + (compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + || compression_algorithm == CompressionAlgorithm::ZstdDefaultCompressionWithChecksum) && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib || checksum_algorithm == ChecksumAlgorithm::Crc32c); if (!no_compression && !zstd_with_checksum) { @@ -109,7 +111,14 @@ std::expected compress_with_checksum( return std::unexpected(Error{ErrorCode::CompressionFailed}); } - if (ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_compressionLevel, ZSTD_maxCLevel())) + const int compression_level = + compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + ? ZSTD_maxCLevel() + : 0; + if (ZSTD_isError(ZSTD_CCtx_setParameter( + context.get(), + ZSTD_c_compressionLevel, + compression_level)) || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { return std::unexpected(Error{ErrorCode::CompressionFailed}); } diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h index 104f9edd..536fe694 100644 --- a/src/terrainlib/io/compression.h +++ b/src/terrainlib/io/compression.h @@ -24,6 +24,7 @@ enum class ChecksumAlgorithm : std::uint8_t { enum class CompressionAlgorithm : std::uint8_t { None, ZstdBestCompressionWithChecksum, + ZstdDefaultCompressionWithChecksum, }; enum class ErrorCode : std::uint8_t { diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index 109f2b1b..3cfbd8eb 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -72,6 +73,11 @@ using Schema = io::envelope::PayloadSchema< io::envelope::Version<2, v2::Payload>, io::envelope::Version<3, v3::Payload>>; +constexpr std::array zstd_compression_algorithms{ + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, +}; + static_assert(std::is_aggregate_v); static_assert(std::is_aggregate_v); static_assert(std::is_aggregate_v); @@ -288,30 +294,33 @@ TEST_CASE("CRC-32C protects independently compressed data") SECTION("with zstd compression") { - const auto compressed = io::envelope::compress_with_checksum( - original, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::Crc32c); - REQUIRE(compressed.has_value()); - REQUIRE(compressed->checksum.size() == 8); - - const auto result = io::envelope::checked_decompress( - compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::Crc32c, - compressed->checksum); - REQUIRE(result.has_value()); - CHECK(*result == original); + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto compressed = io::envelope::compress_with_checksum( + original, + compression_algorithm, + io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + REQUIRE(compressed->checksum.size() == 8); - auto wrong_checksum = compressed->checksum; - wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; - check_error( - io::envelope::checked_decompress( + const auto result = io::envelope::checked_decompress( compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c, - wrong_checksum), - io::envelope::ErrorCode::ChecksumMismatch); + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + auto wrong_checksum = compressed->checksum; + wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; + check_error( + io::envelope::checked_decompress( + compressed->compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::Crc32c, + wrong_checksum), + io::envelope::ErrorCode::ChecksumMismatch); + } } SECTION("malformed checksum") @@ -387,53 +396,56 @@ TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") TEST_CASE("Checked compression round trips and validates its checksum") { const io::envelope::Bytes original(4096, std::byte{0x2a}); - const auto compressed = io::envelope::compress_with_checksum( - original, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib); - REQUIRE(compressed.has_value()); - CHECK(compressed->compressed_data.size() < original.size()); - CHECK(compressed->checksum.empty()); - - const auto result = io::envelope::checked_decompress( - compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - compressed->checksum); - REQUIRE(result.has_value()); - CHECK(*result == original); - - const auto empty_compressed = io::envelope::compress_with_checksum( - {}, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib); - REQUIRE(empty_compressed.has_value()); - const auto empty_result = io::envelope::checked_decompress( - empty_compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - empty_compressed->checksum); - REQUIRE(empty_result.has_value()); - CHECK(empty_result->empty()); - - auto corrupted = compressed->compressed_data; - corrupted.back() ^= std::byte{0x01}; - check_error( - io::envelope::checked_decompress( - corrupted, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}), - io::envelope::ErrorCode::ChecksumMismatch); + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto compressed = io::envelope::compress_with_checksum( + original, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(compressed.has_value()); + CHECK(compressed->compressed_data.size() < original.size()); + CHECK(compressed->checksum.empty()); - check_error( - io::envelope::checked_decompress( + const auto result = io::envelope::checked_decompress( compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + const auto empty_compressed = io::envelope::compress_with_checksum( {}, - original.size() - 1), - io::envelope::ErrorCode::SizeLimitExceeded); + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(empty_compressed.has_value()); + const auto empty_result = io::envelope::checked_decompress( + empty_compressed->compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + empty_compressed->checksum); + REQUIRE(empty_result.has_value()); + CHECK(empty_result->empty()); + + auto corrupted = compressed->compressed_data; + corrupted.back() ^= std::byte{0x01}; + check_error( + io::envelope::checked_decompress( + corrupted, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}), + io::envelope::ErrorCode::ChecksumMismatch); + + check_error( + io::envelope::checked_decompress( + compressed->compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size() - 1), + io::envelope::ErrorCode::SizeLimitExceeded); + } } TEST_CASE("Checked decompression uses its maximum when the format omits the content size") @@ -441,23 +453,26 @@ TEST_CASE("Checked decompression uses its maximum when the format omits the cont const io::envelope::Bytes original(4096, std::byte{0x37}); const auto compressed_data = compress_without_content_size(original); - const auto result = io::envelope::checked_decompress( - compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}, - original.size()); - REQUIRE(result.has_value()); - CHECK(*result == original); - - check_error( - io::envelope::checked_decompress( + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto result = io::envelope::checked_decompress( compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, - original.size() - 1), - io::envelope::ErrorCode::SizeLimitExceeded); + original.size()); + REQUIRE(result.has_value()); + CHECK(*result == original); + + check_error( + io::envelope::checked_decompress( + compressed_data, + compression_algorithm, + io::envelope::ChecksumAlgorithm::HandledByCompressionLib, + {}, + original.size() - 1), + io::envelope::ErrorCode::SizeLimitExceeded); + } } TEST_CASE("Envelope rejects incompatible metadata") From f1813e9ac504ed8a3d964348c963763f21dc52f6 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:52:45 +0200 Subject: [PATCH 14/43] Add OpenCV raster conversion API --- src/terrainlib/CMakeLists.txt | 1 + src/terrainlib/io/conversion.h | 239 +++++++++++++++++++++++++ src/tile_builder/image_writer.cpp | 20 +-- unittests/CMakeLists.txt | 1 + unittests/terrainlib/io_conversion.cpp | 125 +++++++++++++ 5 files changed, 376 insertions(+), 10 deletions(-) create mode 100644 src/terrainlib/io/conversion.h create mode 100644 unittests/terrainlib/io_conversion.cpp diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 5ed36a37..74014a09 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -8,6 +8,7 @@ add_library(terrainlib io/bytes.cpp io/compression.cpp + io/conversion.h io/utils.cpp mesh/connectivity/adjacency.cpp diff --git a/src/terrainlib/io/conversion.h b/src/terrainlib/io/conversion.h new file mode 100644 index 00000000..246e87b0 --- /dev/null +++ b/src/terrainlib/io/conversion.h @@ -0,0 +1,239 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace io::conversion { + +enum class ErrorCode { + EmptyInput, + UnsupportedDimensions, + UnsupportedPixelType, + PixelTypeMismatch, + DimensionsOutOfRange, + AllocationFailed, + OpenCvFailure, +}; + +struct Error { + ErrorCode code; + int expected_cv_type{-1}; + int actual_cv_type{-1}; + + constexpr bool operator==(const Error &) const = default; +}; + +namespace detail { + +template +inline constexpr int cv_depth = -1; + +template <> +inline constexpr int cv_depth = CV_8U; +template <> +inline constexpr int cv_depth = CV_8S; +template <> +inline constexpr int cv_depth = CV_16U; +template <> +inline constexpr int cv_depth = CV_16S; +template <> +inline constexpr int cv_depth = CV_32S; +template <> +inline constexpr int cv_depth = CV_32F; +template <> +inline constexpr int cv_depth = CV_64F; + +template +struct PixelTraits { + using channel_type = std::remove_cv_t; + static constexpr int channels = 1; + static constexpr bool supported = cv_depth >= 0; + + static channel_type &channel(Pixel &pixel, const int) + { + return pixel; + } + + static const channel_type &channel(const Pixel &pixel, const int) + { + return pixel; + } +}; + +template +struct PixelTraits> { + using channel_type = Channel; + static constexpr int channels = static_cast(Length); + static constexpr bool supported = channels >= 2 && channels <= 4 + && cv_depth >= 0; + + static channel_type &channel(glm::vec &pixel, const int index) + { + return pixel[static_cast(index)]; + } + + static const channel_type &channel( + const glm::vec &pixel, + const int index) + { + return pixel[static_cast(index)]; + } +}; + +template +using Traits = PixelTraits>; + +template +inline constexpr int cv_type = Traits::supported + ? CV_MAKETYPE(cv_depth::channel_type>, Traits::channels) + : -1; + +template +inline constexpr bool has_packed_layout = Traits::supported + && std::is_trivially_copyable_v + && sizeof(Pixel) == sizeof(typename Traits::channel_type) * Traits::channels; + +template +void copy_mat_to_raster(const cv::Mat &source, radix::Raster &destination) +{ + using PixelInfo = Traits; + using Channel = typename PixelInfo::channel_type; + + const auto row_bytes = static_cast(source.cols) * sizeof(Pixel); + if constexpr (has_packed_layout) { + for (int row = 0; row < source.rows; ++row) { + std::memcpy( + destination.data() + static_cast(row) * source.cols, + source.ptr(row), + row_bytes); + } + } else { + for (int row = 0; row < source.rows; ++row) { + const Channel *source_row = source.ptr(row); + for (int column = 0; column < source.cols; ++column) { + Pixel &pixel = destination.pixel({ + static_cast(column), + static_cast(row), + }); + for (int channel = 0; channel < PixelInfo::channels; ++channel) { + PixelInfo::channel(pixel, channel) = + source_row[column * PixelInfo::channels + channel]; + } + } + } + } +} + +template +void copy_raster_to_mat(const radix::Raster &source, cv::Mat &destination) +{ + using PixelInfo = Traits; + using Channel = typename PixelInfo::channel_type; + + const auto row_bytes = static_cast(destination.cols) * sizeof(Pixel); + if constexpr (has_packed_layout) { + for (int row = 0; row < destination.rows; ++row) { + std::memcpy( + destination.ptr(row), + source.data() + static_cast(row) * destination.cols, + row_bytes); + } + } else { + for (int row = 0; row < destination.rows; ++row) { + Channel *destination_row = destination.ptr(row); + for (int column = 0; column < destination.cols; ++column) { + const Pixel &pixel = source.pixel({ + static_cast(column), + static_cast(row), + }); + for (int channel = 0; channel < PixelInfo::channels; ++channel) { + destination_row[column * PixelInfo::channels + channel] = + PixelInfo::channel(pixel, channel); + } + } + } + } +} + +} // namespace detail + +template +std::expected, Error> to_raster(const cv::Mat &source) +{ + if constexpr (!detail::Traits::supported) { + return std::unexpected(Error{ErrorCode::UnsupportedPixelType}); + } else { + if (source.empty()) { + return std::unexpected(Error{ErrorCode::EmptyInput}); + } + if (source.dims != 2) { + return std::unexpected(Error{ErrorCode::UnsupportedDimensions}); + } + if (source.type() != detail::cv_type) { + return std::unexpected(Error{ + ErrorCode::PixelTypeMismatch, + detail::cv_type, + source.type(), + }); + } + if (source.cols < 0 || source.rows < 0 + || static_cast(source.cols) + > std::numeric_limits::max() + || static_cast(source.rows) + > std::numeric_limits::max()) { + return std::unexpected(Error{ErrorCode::DimensionsOutOfRange}); + } + + try { + radix::Raster destination({ + static_cast(source.cols), + static_cast(source.rows), + }); + detail::copy_mat_to_raster(source, destination); + return destination; + } catch (const std::bad_alloc &) { + return std::unexpected(Error{ErrorCode::AllocationFailed}); + } catch (const cv::Exception &) { + return std::unexpected(Error{ErrorCode::OpenCvFailure}); + } + } +} + +template +std::expected to_mat(const radix::Raster &source) +{ + if constexpr (!detail::Traits::supported) { + return std::unexpected(Error{ErrorCode::UnsupportedPixelType}); + } else { + if (source.width() == 0 || source.height() == 0) { + return std::unexpected(Error{ErrorCode::EmptyInput}); + } + if (source.width() > static_cast(std::numeric_limits::max()) + || source.height() > static_cast(std::numeric_limits::max())) { + return std::unexpected(Error{ErrorCode::DimensionsOutOfRange}); + } + + try { + cv::Mat destination( + static_cast(source.height()), + static_cast(source.width()), + detail::cv_type); + detail::copy_raster_to_mat(source, destination); + return destination; + } catch (const std::bad_alloc &) { + return std::unexpected(Error{ErrorCode::AllocationFailed}); + } catch (const cv::Exception &) { + return std::unexpected(Error{ErrorCode::OpenCvFailure}); + } + } +} + +} // namespace io::conversion diff --git a/src/tile_builder/image_writer.cpp b/src/tile_builder/image_writer.cpp index 725cdfae..b3135f90 100644 --- a/src/tile_builder/image_writer.cpp +++ b/src/tile_builder/image_writer.cpp @@ -20,23 +20,23 @@ #include "image_writer.h" +#include "io/conversion.h" + #include #include void image::saveImageAsPng(const radix::Raster& input_image, const std::string& path) { - const int width = static_cast(input_image.width()); - const int height = static_cast(input_image.height()); - - cv::Mat image(height, width, CV_8UC3); - - for (int row = 0; row < height; ++row) { - for (int column = 0; column < width; ++column) { - const auto& pixel = input_image.pixel({ static_cast(column), static_cast(height - row - 1) }); - image.at(row, column) = cv::Vec3b(pixel.z, pixel.y, pixel.x); - } + auto converted = io::conversion::to_mat(input_image); + if (!converted) { + throw std::runtime_error("Failed to convert raster to OpenCV image"); } + cv::Mat flipped; + cv::flip(*converted, flipped, 0); + cv::Mat image; + cv::cvtColor(flipped, image, cv::COLOR_RGB2BGR); + try { if (!cv::imwrite(path, image)) throw std::runtime_error("Failed to write PNG image to " + path); diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 5a752900..d3338950 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -27,6 +27,7 @@ add_executable(unittests_terrainlib terrainlib/hash_utils.cpp terrainlib/hybrid_index_pair_map.cpp terrainlib/hybrid_vector.cpp + terrainlib/io_conversion.cpp terrainlib/offset_table.cpp terrainlib/segmented_buffer.cpp terrainlib/sf_validate_index.cpp diff --git a/unittests/terrainlib/io_conversion.cpp b/unittests/terrainlib/io_conversion.cpp new file mode 100644 index 00000000..2dd7c7e8 --- /dev/null +++ b/unittests/terrainlib/io_conversion.cpp @@ -0,0 +1,125 @@ +#include "../catch2_helpers.h" + +#include "io/conversion.h" + +#include +#include + +namespace { + +template +void check_round_trip(const int cv_type) +{ + cv::Mat source(2, 3, cv_type); + auto bytes = std::span(source.ptr(), source.total() * source.elemSize()); + for (std::size_t index = 0; index < bytes.size(); ++index) { + bytes[index] = static_cast(index + 1); + } + + const auto raster = io::conversion::to_raster(source); + REQUIRE(raster.has_value()); + CHECK(raster->size() == glm::uvec2(3, 2)); + + const auto result = io::conversion::to_mat(*raster); + REQUIRE(result.has_value()); + CHECK(result->type() == source.type()); + CHECK(result->rows == source.rows); + CHECK(result->cols == source.cols); + CHECK(std::equal(bytes.begin(), bytes.end(), result->template ptr())); +} + +} // namespace + +TEST_CASE("io::conversion round trips supported scalar and vector pixels") +{ + check_round_trip(CV_8UC1); + check_round_trip(CV_8SC1); + check_round_trip(CV_16UC1); + check_round_trip(CV_16SC1); + check_round_trip(CV_32SC1); + check_round_trip(CV_32FC1); + check_round_trip(CV_64FC1); + check_round_trip(CV_8UC2); + check_round_trip(CV_8UC3); + check_round_trip(CV_8UC4); + check_round_trip(CV_32FC3); + check_round_trip(CV_64FC4); +} + +TEST_CASE("io::conversion copies non-contiguous matrices row by row") +{ + cv::Mat backing(4, 6, CV_8UC3); + for (int row = 0; row < backing.rows; ++row) { + for (int column = 0; column < backing.cols; ++column) { + backing.at(row, column) = cv::Vec3b( + static_cast(row), + static_cast(column), + static_cast(row * backing.cols + column)); + } + } + const cv::Mat source = backing(cv::Rect(1, 1, 3, 2)); + REQUIRE_FALSE(source.isContinuous()); + + auto raster = io::conversion::to_raster(source); + REQUIRE(raster.has_value()); + CHECK(raster->pixel({0, 0}) == glm::u8vec3(1, 1, 7)); + CHECK(raster->pixel({2, 1}) == glm::u8vec3(2, 3, 15)); + + backing.setTo(cv::Scalar::all(0)); + CHECK(raster->pixel({0, 0}) == glm::u8vec3(1, 1, 7)); + + const auto round_trip = io::conversion::to_mat(*raster); + REQUIRE(round_trip.has_value()); + raster->fill(glm::u8vec3(0)); + CHECK(round_trip->at(0, 0) == cv::Vec3b(1, 1, 7)); +} + +TEST_CASE("io::conversion reports invalid inputs") +{ + SECTION("empty matrix") + { + const auto result = io::conversion::to_raster(cv::Mat{}); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == io::conversion::ErrorCode::EmptyInput); + } + + SECTION("multidimensional matrix") + { + const std::array sizes{2, 2, 2}; + const cv::Mat source(static_cast(sizes.size()), sizes.data(), CV_8UC1); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == io::conversion::ErrorCode::UnsupportedDimensions); + } + + SECTION("pixel type mismatch") + { + const cv::Mat source(2, 2, CV_8UC3); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + const io::conversion::Error expected{ + io::conversion::ErrorCode::PixelTypeMismatch, + CV_8UC1, + CV_8UC3, + }; + CHECK(result.error() == expected); + } + + SECTION("unsupported pixel") + { + struct Pixel { + std::uint64_t value; + }; + const cv::Mat source(2, 2, CV_8UC1); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == io::conversion::ErrorCode::UnsupportedPixelType); + } + + SECTION("empty raster") + { + const auto result = io::conversion::to_mat(radix::Raster{}); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code == io::conversion::ErrorCode::EmptyInput); + } +} From c7122abd1ab3f68032e4cb3b1e615057ccb6ccce Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:53:05 +0200 Subject: [PATCH 15/43] Migrate octree storage formats to envelopes --- docs/raster-store/envelope-migration-plan.md | 348 ++++++++++++++++ docs/raster-store/golden-e2e.sh | 37 +- docs/raster-store/storage-format.md | 18 +- src/dag_builder/build.cpp | 2 - src/dag_builder/codec/Dag.h | 129 ++++++ src/dag_builder/codec/DagFormat.h | 391 ++++++++++++++++++ src/dag_builder/codec/MetadataView.h | 34 +- src/dag_builder/codec/from_extension.h | 9 +- src/dag_builder/encoded.h | 21 - src/dag_builder/serialization.h | 212 ---------- src/dag_builder/storage.h | 17 +- src/dag_convert_debug/main.cpp | 16 +- src/sf_builder/main.cpp | 6 +- src/sf_builder/terrainbuilder.cpp | 1 - src/sf_merger/main.cpp | 4 +- src/terrainlib/CMakeLists.txt | 1 - src/terrainlib/io/Error.h | 13 - src/terrainlib/io/envelope.h | 8 +- src/terrainlib/io/envelope.inl | 12 + src/terrainlib/io/envelope_file.h | 80 ++++ src/terrainlib/io/serialize.h | 25 -- src/terrainlib/io/serialize.inl | 88 ---- src/terrainlib/mesh/EncodedMesh.h | 3 - src/terrainlib/mesh/codec/Gltf.h | 2 +- src/terrainlib/mesh/codec/SfMesh.h | 96 +++++ src/terrainlib/mesh/codec/SfMeshFormat.h | 96 +++++ src/terrainlib/mesh/codec/Terrain.h | 49 --- src/terrainlib/mesh/codec/from_extension.h | 6 +- src/terrainlib/mesh/io.cpp | 27 +- src/terrainlib/mesh/io/options.h | 2 +- src/terrainlib/mesh/io/terrain.cpp | 172 -------- src/terrainlib/mesh/io/terrain.h | 20 - src/terrainlib/mesh/io/texture.h | 31 -- src/terrainlib/mesh/storage.h | 11 +- src/terrainlib/octree/Id.h | 50 +-- src/terrainlib/octree/disk/IndexFile.h | 26 -- src/terrainlib/octree/storage/Format.h | 138 +++++++ src/terrainlib/octree/storage/IndexFile.h | 114 +++-- src/terrainlib/octree/storage/open.h | 3 +- src/terrainlib/octree/storage/open_runtime.h | 134 ++---- src/terrainlib/pch.h | 1 - src/terrainlib/store/Index.h | 5 - src/terrainlib/store/IndexFormat.h | 1 + src/terrainlib/store/NodeStatus.h | 6 - src/terrainlib/store/RawStorage.h | 2 +- src/terrainlib/store/Storage.h | 97 +---- src/terrainlib/store/codec/Path.h | 19 + src/terrainlib/store/codec/ZppBits.h | 57 --- src/terrainlib/store/open.h | 26 +- .../dag_builder/storage_compatibility.cpp | 61 ++- unittests/sf_builder/storage_finalization.cpp | 7 +- unittests/sf_merger/integration.cpp | 6 +- unittests/sf_merger/storage_boundaries.cpp | 4 +- unittests/terrainlib/envelope.cpp | 2 +- unittests/terrainlib/mesh_clip.cpp | 34 -- unittests/terrainlib/mesh_io.cpp | 6 +- unittests/terrainlib/store_codec.cpp | 48 +-- unittests/terrainlib/store_compatibility.cpp | 102 +++-- unittests/terrainlib/store_layout.cpp | 22 +- unittests/terrainlib/store_storage.cpp | 90 +++- unittests/terrainlib/string_utils.cpp | 2 +- 61 files changed, 1816 insertions(+), 1234 deletions(-) create mode 100644 docs/raster-store/envelope-migration-plan.md create mode 100644 src/dag_builder/codec/Dag.h create mode 100644 src/dag_builder/codec/DagFormat.h delete mode 100644 src/dag_builder/encoded.h delete mode 100644 src/dag_builder/serialization.h create mode 100644 src/terrainlib/io/envelope_file.h delete mode 100644 src/terrainlib/io/serialize.h delete mode 100644 src/terrainlib/io/serialize.inl create mode 100644 src/terrainlib/mesh/codec/SfMesh.h create mode 100644 src/terrainlib/mesh/codec/SfMeshFormat.h delete mode 100644 src/terrainlib/mesh/codec/Terrain.h delete mode 100644 src/terrainlib/mesh/io/terrain.cpp delete mode 100644 src/terrainlib/mesh/io/terrain.h delete mode 100644 src/terrainlib/octree/disk/IndexFile.h create mode 100644 src/terrainlib/octree/storage/Format.h create mode 100644 src/terrainlib/store/codec/Path.h delete mode 100644 src/terrainlib/store/codec/ZppBits.h diff --git a/docs/raster-store/envelope-migration-plan.md b/docs/raster-store/envelope-migration-plan.md new file mode 100644 index 00000000..0767b6be --- /dev/null +++ b/docs/raster-store/envelope-migration-plan.md @@ -0,0 +1,348 @@ +# Versioned storage-envelope migration plan + +Status: implemented in the `raster-store` worktree; awaiting commit. + +## Purpose + +This plan replaces the repository's remaining private raw ZPP Bits disk +formats with the generic versioned envelope in `terrainlib/io`. It also moves +dataset-wide format selection out of the octree index and into an independent +metadata file, splits DAG data from its cheaply readable metadata, and removes +domain transformations and validation from ZPP serialization callbacks. + +The migration intentionally breaks compatibility with the pre-envelope +`terrain.index`, `.terrain`, and `.bin` formats. Every changed format receives +a new filename ending. Once an enveloped format is introduced, later readers +must continue to read its older envelope payload versions and upgrade them to +the latest in-memory type. + +## Decisions already made + +- The generic `io::envelope` magic, class name, class version, bounded + decompression, checksum, and compression fields are used for every new + private binary format. +- Default zstd compression with its embedded checksum is used for indexes, + metadata, SF meshes, DAG data, and DAG metadata, including already encoded + meshoptimizer and JPEG byte streams. +- The new persistent names are: + - `octree.storemeta` for dataset-wide metadata; + - `octree.storeindex` for the sparse hierarchy index; + - `.sfmesh` for enveloped SF mesh nodes; + - `.dag` for enveloped DAG clustering data; and + - `.dagmeta` for enveloped DAG node metadata. +- Standard `.glb` and `.gltf` files remain unchanged. They are debugging + formats and are not wrapped in the private envelope. +- Filename endings continue to identify formats. A format change receives a + new ending; versions that remain readable by the same schema retain the + ending. +- The dataset metadata file, not the index, is authoritative for the layout + ID, payload class, and codec selector. It remains readable when the index is + missing or malformed. +- The index continues to persist the complete `store::Index` state, including + `Leaf`, `Inner`, and `Virtual`. Deterministic byte order is not required. +- A DAG logical node consists of two files. The writable batch codec writes + both `.dag` and `.dagmeta`; the read-only metadata codec reads only + `.dagmeta`. +- Hard-link behavior remains path-list based. Identical new formats use the + same endings and can be linked; changing format or compression policy uses + a different ending and therefore decodes and rewrites. +- Domain-specific ZPP serialization callbacks are removed. ZPP serializes + versioned aggregate wire types mechanically. Explicit free functions encode, + validate, and decode those wire types. +- The old `io/serialize.h` and `io/serialize.inl` API is removed after all + production and test callers migrate. +- Avoidable copies are removed with spans and moves. Compression necessarily + requires owned uncompressed and compressed buffers; zero-copy decoding is + not a goal. + +## Goals + +1. Give every private binary file envelope identity, versioning, checksum, + compression, bounded allocation, and complete-input validation. +2. Keep persistence schemas explicit and versioned without putting disk + concerns into processing types. +3. Make validation independently callable and testable. +4. Preserve cheap DAG metadata reads without reading or decompressing `.dag`. +5. Remove layout/codec guessing from directory contents. +6. Keep standard glTF behavior unchanged. +7. Preserve current application semantics, topology rules, hard-link reuse, + and error propagation apart from the deliberate format break. + +## Non-goals + +- Reading pre-envelope `terrain.index`, `.terrain`, or `.bin` files. +- Changing GLB or JSON glTF serialization. +- Defining raster-fundamentalis `.amort`, source-attribution, or raster index + payloads. +- Changing hierarchy algorithms, SF merge policy, DAG construction, mesh + processing, or snapshot publication. +- Making index serialization deterministic. +- Adding a hard-link compatibility registry or implicit transcoding between + formats with the same ending. +- Refactoring unrelated image, tile-builder, or downloader formats. + +## Format boundary + +```text +octree dataset root +├── octree.storemeta envelope: octree.StoreMetadata +├── octree.storeindex envelope: octree.StoreIndex +└── + ├── .sfmesh envelope: mesh.SfMesh + ├── .dag envelope: dag.Clustering + ├── .dagmeta envelope: dag.NodeMetadata + ├── .glb unchanged standard debug output + └── .gltf unchanged standard debug output +``` + +The files shown under one node are alternatives except that `.dag` and +`.dagmeta` form one logical DAG node. + +## Versioned payloads + +Each format defines `v1` aggregate payloads, a schema alias, a latest-type +alias, and small read/write wrappers. Future `v2` types add +`from_previous(v1::Type)` as demonstrated by the envelope tests. + +### Dataset metadata + +`octree::storage::v1::StoreMetadata` contains: + +- layout ID; +- payload class ID; and +- codec selector/format ending. + +Its validator rejects empty identifiers and unknown layouts at the adapter +boundary. Payload-specific open functions additionally require the expected +payload class and codec. + +### Sparse index + +The versioned index payload contains the complete runtime +`store::Index`. To remove the custom `octree::Id` ZPP +callback, the disk representation uses aggregate key/status entries and +explicit conversion to and from the runtime index. Conversion validates IDs, +status values, duplicates, and topology consistency. + +The conversion is allowed to retain the runtime index's semantic state and +non-deterministic iteration order. It must not silently normalize malformed +serialized status combinations. + +### SF mesh + +`mesh::sf::v1::Payload` contains fixed-width counts and encoded byte buffers +for triangles, positions, UVs, and the optional texture. The envelope version +replaces `mesh::Encoded::Header::version`. Dimension and component-type fields +that are invariant for `mesh::Simple3d` are not used for runtime dispatch. + +Free functions perform: + +```text +mesh::Simple + -> encode meshoptimizer buffers and image + -> validate v1 payload + -> envelope write + +envelope read + -> validate v1 payload + -> decode meshoptimizer buffers and image + -> mesh::Simple +``` + +### DAG clustering and metadata + +The `.dag` payload contains the encoded clustering only. Its aggregate DTOs +use fixed-width counts, primitive coordinate aggregates, encoded +meshoptimizer buffers, encoded JPEG byte vectors, IDs, texture references, +and errors. The `.dagmeta` payload independently contains group assignment, +groups, child IDs, bounds, and errors. + +The writable `Codec` returns both paths, encodes the two +payloads, and writes both envelopes. Its read path reads both envelopes and +combines their decoded latest types. `Codec` returns and +reads only `.dagmeta`, with writes unsupported. + +Validation rejects malformed encoded buffers, invalid octree IDs, invalid +cluster/group/texture references, inconsistent counts, and dimensions that +would overflow allocations before constructing processing objects. + +## Serialization and I/O cleanup + +One bounded ZPP byte API supports the envelope implementation. Public path +helpers read and write `io::envelope::Bytes`; path wrappers compose byte I/O +with `io::envelope::serialize` and `deserialize`. All error conversion occurs +at the format/codec boundary. + +The migration removes: + +- raw `io::write_to_bytes`, `read_from_bytes`, `write_to_path`, and + `read_from_path` serialization; +- the `octree::Id` custom serializer and its embedded `try_make` validation; +- `store::Index` and `NodeStatus` persistence hooks; +- `mesh::Encoded` and `ImageAndExt` persistence hooks; +- DAG GLM, ID, metadata, texture, cluster, clustering, and batch serializers; + and +- unused standalone clustering persistence helpers. + +`io::bytes` remains the filesystem byte layer. Envelope path helpers expose +its `uint8_t` buffers as `std::byte` spans without copying. + +## Opening and creation lifecycle + +Opening an existing private dataset requires both fixed root files: + +1. Read and validate `octree.storemeta`. +2. Resolve the layout and payload codec from metadata. +3. Read and validate `octree.storeindex`. +4. Construct indexed storage. + +An unreadable metadata or index file returns a typed open error. There is no +directory scan fallback. + +Creating a dataset receives an explicit layout and codec, creates an indexed +in-memory store immediately, and writes node payloads while updating that +index. Finalization writes dataset metadata and the index. The implementation +does not infer layout or codec from existing node filenames. + +## Implementation phases + +### Phase 0 — Plan and semantic baseline + +- Record this approved design. +- Retain semantic round-trip, store, merge, and builder tests rather than raw + legacy bytes. +- Confirm the clean raster-store worktree and current local commits. + +Exit criterion: format names, compatibility boundary, split DAG layout, and +metadata authority are explicit. + +### Phase 1 — Consolidate envelope I/O + +- Add envelope path read/write helpers and zero-copy byte-span bridging to the + filesystem I/O layer. +- Provide a latest-version serialization overload so ordinary writers do not + repeat the latest version number. +- Remove avoidable intermediate copies. +- Add path I/O and latest-version tests. + +Exit criterion: production codecs can use the envelope without the legacy raw +serialization API. + +### Phase 2 — Version dataset metadata and index + +- Add versioned dataset metadata and index DTOs and schemas. +- Add free validation/conversion functions. +- Write `octree.storemeta` and `octree.storeindex` through the envelope. +- Separate dataset metadata persistence from index persistence in the shared + storage plumbing. +- Remove directory discovery and index reconstruction by extension scanning. +- Start new outputs indexed in memory. + +Exit criterion: a new empty or populated store opens only through its metadata +and index, and corrupt/mismatched files return typed failures. + +### Phase 3 — Migrate SF mesh payloads + +- Add the versioned `.sfmesh` wire payload. +- Move meshoptimizer and texture work into explicit encode/decode functions. +- Add free validation for counts, buffers, texture data, and mesh semantics. +- Update the SF mesh codec resolver and CLI defaults to `.sfmesh`. +- Retain `.glb`/`.gltf` unchanged. + +Exit criterion: SF builder/merger/store tests round-trip `.sfmesh`; standard +debug codecs still behave as before; no `.terrain` production path remains. + +### Phase 4 — Split and migrate DAG payloads + +- Add versioned `.dag` clustering and `.dagmeta` metadata payloads. +- Move all DAG encoding/decoding out of ZPP callbacks. +- Add a two-file writable/readable batch codec. +- Add a separate read-only `.dagmeta` codec. +- Update DAG builder and debug converter callers. + +Exit criterion: full batches round-trip through two files, metadata reads touch +only `.dagmeta`, and malformed encoded data returns errors rather than +asserting. + +### Phase 5 — Remove legacy serialization and inference + +- Delete domain-specific serializers and `io/serialize.h/.inl`. +- Delete unused standalone clustering persistence. +- Remove `.terrain`, `.bin`, and `terrain.index` private-format selectors and + tests. +- Remove unindexed directory discovery and suffix-probe index scans. +- Update documentation, golden scripts, test expectations, and error text. + +Exit criterion: repository search finds no production raw ZPP persistence or +legacy private extension and no format inference from directory contents. + +### Phase 6 — Verification + +- Run focused envelope, index, codec, storage, SF, and DAG tests after their + phases. +- Run the complete terrainlib, DAG builder, SF builder finalization, and SF + merger suites. +- Build `sf-builder`, `sf-merger`, `sf-index-browser`, `dag-builder`, and + `dag-convert-debug` in the repository build configuration. +- Run the golden end-to-end workflow with new expected endings where its data + prerequisites are available. +- Inspect `git diff --check` and the final worktree. + +## Required tests + +- Every production schema round-trips v1 through default zstd. +- Each schema rejects the wrong envelope class and unsupported versions. +- Corrupt compressed data and malformed aggregate payloads return typed + errors. +- Dataset metadata remains readable when `octree.storeindex` is corrupt. +- Payload class, codec, and layout mismatches fail before node processing. +- Invalid octree IDs, node statuses, duplicates, and inconsistent topology are + rejected by free validation/conversion functions. +- SF mesh encode/decode preserves current semantic mesh equality and rejects + malformed meshoptimizer/image data. +- DAG full-batch reads require `.dag` and `.dagmeta`. +- DAG metadata reads succeed with only `.dagmeta` present and do not access + `.dag`. +- Same-format node copies hard-link both DAG files; differing formats + decode/re-encode. +- GLB/GLTF tests remain unchanged. +- New dataset construction never scans extensions to decide its format. + +## Risks and controls + +| Risk | Control | +|---|---| +| A future reader cannot load v1 | Versioned namespaces, schema conversion trail, and retained v1 tests | +| Dataset metadata and index disagree | Metadata is authoritative; index contains topology only | +| Broken index hides format information | Independent `octree.storemeta` is read and tested separately | +| DAG metadata reads regress to full data reads | Separate `.dagmeta` file and read-only codec tests | +| Custom ZPP callbacks continue to hide failures | Aggregate DTOs plus explicit expected-returning validation/decode | +| Meshoptimizer corruption aborts through assertions | Checked decode wrappers return typed errors | +| New compression adds excessive copies | Shared byte type, spans, moves, and copy-focused review | +| New format is accidentally linked to an old one | Every changed format uses a new ending | +| Standard debug formats are accidentally wrapped | Dedicated unchanged GLB/GLTF codecs and existing tests | +| Incomplete output becomes authoritative | Keep explicit finalization boundaries and write root metadata/index last | + +## Expected migration map + +| Current | Target | +|---|---| +| `terrain.index` raw ZPP DTO | `octree.storeindex` envelope schema | +| layout and preferred extension inside index | `octree.storemeta` envelope schema | +| `.terrain` raw encoded mesh | `.sfmesh` envelope schema | +| `.bin` DAG batch | `.dag` clustering plus `.dagmeta` metadata envelopes | +| `.bin` metadata-prefix view | read-only `.dagmeta` codec | +| extension/layout directory discovery | explicit metadata-driven open | +| unindexed save then recursive scan | indexed in-memory creation and final write | +| domain `serialize(Archive&, T&)` callbacks | aggregate DTO plus free encode/validate/decode | +| `io/serialize.h/.inl` | envelope path helpers | +| `.glb` / `.gltf` | unchanged | + +## Verification result + +The implementation builds the SF builder/merger, DAG builder/debug converter, +and SF index browser targets. The terrainlib, DAG builder, SF finalization, SF +merger, and SF builder/merger integration tests pass with the new formats. The +golden end-to-end script has been migrated to the new filenames and passes +shell syntax validation; its external-data workflow was not run as part of +this implementation. diff --git a/docs/raster-store/golden-e2e.sh b/docs/raster-store/golden-e2e.sh index 894622cf..df44f53e 100755 --- a/docs/raster-store/golden-e2e.sh +++ b/docs/raster-store/golden-e2e.sh @@ -109,11 +109,13 @@ verify_snapshot() { local snapshot="$1" local extension="$2" + local companion_extension="${3:-}" local payload_count - require_file "${snapshot}/terrain.index" - if [[ ! -s "${snapshot}/terrain.index" ]]; then - printf 'Index is empty: %s\n' "${snapshot}/terrain.index" >&2 + require_file "${snapshot}/octree.storemeta" + require_file "${snapshot}/octree.storeindex" + if [[ ! -s "${snapshot}/octree.storemeta" ]] || [[ ! -s "${snapshot}/octree.storeindex" ]]; then + printf 'Octree metadata or index is empty: %s\n' "${snapshot}" >&2 return 1 fi @@ -122,6 +124,15 @@ verify_snapshot() printf 'No %s payloads found in %s\n' "${extension}" "${snapshot}" >&2 return 1 fi + if [[ -n "${companion_extension}" ]]; then + local companion_count + companion_count="$(find "${snapshot}" -type f -name "*${companion_extension}" | wc -l)" + if [[ "${companion_count}" != "${payload_count}" ]]; then + printf '%s payload count (%s) does not match %s count (%s) in %s\n' \ + "${extension}" "${payload_count}" "${companion_extension}" "${companion_count}" "${snapshot}" >&2 + return 1 + fi + fi printf 'Verified %s: %s payloads\n' "${snapshot}" "${payload_count}" } @@ -135,7 +146,7 @@ build_sf() if [[ -f "${marker}" ]]; then printf '[%s] SKIP %s: completion marker exists\n' "$(timestamp)" "${name}" - verify_snapshot "${output}" ".terrain" + verify_snapshot "${output}" ".sfmesh" return fi @@ -151,10 +162,10 @@ build_sf() batch \ --target-level "${SF_TARGET_LEVEL}" \ --output "${output}" \ - --format .terrain \ + --format .sfmesh \ --threads "${SF_THREADS}" - verify_snapshot "${output}" ".terrain" + verify_snapshot "${output}" ".sfmesh" touch "${marker}" } @@ -164,7 +175,7 @@ merge_sf() if [[ -f "${marker}" ]]; then printf '[%s] SKIP merge_sf: completion marker exists\n' "$(timestamp)" - verify_snapshot "${SF_MERGED}" ".terrain" + verify_snapshot "${SF_MERGED}" ".sfmesh" return fi @@ -183,7 +194,7 @@ merge_sf() --output "${SF_MERGED}" \ --verbosity info - verify_snapshot "${SF_MERGED}" ".terrain" + verify_snapshot "${SF_MERGED}" ".sfmesh" touch "${marker}" } @@ -194,12 +205,12 @@ build_dag() if [[ -f "${marker}" ]]; then printf '[%s] SKIP build_dag: completion marker exists\n' "$(timestamp)" - verify_snapshot "${DAG_MERGED}" ".bin" + verify_snapshot "${DAG_MERGED}" ".dag" ".dagmeta" return fi mkdir -p "${DAG_MERGED}" - if [[ -s "${DAG_MERGED}/terrain.index" ]]; then + if [[ -s "${DAG_MERGED}/octree.storeindex" ]]; then continuation=(--resume) fi @@ -210,7 +221,7 @@ build_dag() "${continuation[@]}" \ --verbosity info - verify_snapshot "${DAG_MERGED}" ".bin" + verify_snapshot "${DAG_MERGED}" ".dag" ".dagmeta" touch "${marker}" } @@ -222,7 +233,7 @@ write_manifest() ( cd "${snapshot}" - find . -type f ! -name terrain.index -print0 \ + find . -type f ! -name octree.storeindex ! -name octree.storemeta -print0 \ | sort -z \ | xargs -0 -r sha256sum ) > "${output}" @@ -258,7 +269,7 @@ record_hard_links() fi ((newly_written += 1)) - done < <(find "${SF_MERGED}" -type f -name '*.terrain' -print0) + done < <(find "${SF_MERGED}" -type f -name '*.sfmesh' -print0) printf 'merged_payloads=%s\nlinked_to_gataki=%s\nlinked_to_basemap=%s\nnewly_written=%s\n' \ "${merged_count}" "${linked_to_gataki}" "${linked_to_basemap}" "${newly_written}" \ diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 0332e8e5..18497b2d 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -5,12 +5,12 @@ > requirements were not acceptance criteria for that refactor. See > [refactor-status.md](refactor-status.md) for the implemented shared boundary. -The refactor preserves the existing 3D `terrain.index` and payload contracts -through the adapters under `octree` and `mesh::storage`. It supplies only an -in-memory `raster_store::StoreTraits` adapter for `radix::tile::Id`; there is -currently no persistent RF index, `.amort` codec, snapshot publisher, or RF -opening API. Names and formats in this document remain proposals until their -own implementation plans approve them. +The 3D octree stores now use the versioned envelope formats described in +[envelope-migration-plan.md](envelope-migration-plan.md). The shared store +still supplies only an in-memory `raster_store::StoreTraits` adapter for +`radix::tile::Id`; there is currently no persistent RF index, `.amort` codec, +snapshot publisher, or RF opening API. Names and formats below remain +proposals until their own implementation plans approve them. This document specifies the logical format and required invariants for the following data stores: - raster-fundamentalis (our authoritative raster store) @@ -74,7 +74,7 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - `uint32 class_version`, identifying the versioned payload type; - `ChecksumAlgorithm checksum_algorithm`, default `HandledByCompressionLib`, alternatively `Crc32c` or `None`; - `string checksum`, empty when no external checksum is used, otherwise the CRC-32C value; - - `CompressionAlgorithm compression_algorithm`, default `ZstdBestCompressionWithChecksum`, alternatively `None`; + - `CompressionAlgorithm compression_algorithm`, default `ZstdDefaultCompressionWithChecksum`, alternatively `ZstdBestCompressionWithChecksum` or `None`; - `uint64 uncompressed_size`, the exact size of the uncompressed second level; - `Bytes compressed_data`, containing the second level. - The second level is a compressed byte vector. It is deserialised directly into the selected versioned payload class. @@ -93,12 +93,12 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - if the compression algorithm is missing or unsupported. - deserialization fails - we fail by returning an unexpected in these cases -- Compression uses libzstd at its best compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. `ZstdBestCompressionWithChecksum` writes an embedded zstd frame checksum, which libzstd verifies while decompressing. +- Compression uses libzstd at its default compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. Both zstd modes write an embedded frame checksum, which libzstd verifies while decompressing; `ZstdBestCompressionWithChecksum` remains available for explicit callers. - `compress_with_checksum` accepts a `vector` and returns the compressed bytes plus the external checksum string. `checked_decompress` accepts both and returns the decompressed bytes. Both use `std::expected` and dispatch with a switch. - `Crc32c` is an external CRC-32C (Castagnoli) checksum of the complete uncompressed serialised payload. It uses the reflected polynomial `82F63B78`, an initial value of `FFFFFFFF`, and a final XOR of `FFFFFFFF`. The checksum string contains exactly eight lowercase hexadecimal characters. The empty payload has checksum `00000000`; the ASCII test vector `123456789` has checksum `e3069283`. CRC-32C detects accidental corruption but is not cryptographic. - `checked_decompress` accepts a maximum decompressed size, defaulting to 1 GiB. It uses a size reported by the compression format when available, otherwise it uses the maximum as its allocation bound and shrinks the result to the produced size. - Envelope deserialisation passes `uncompressed_size` as that maximum and requires the produced size to match it exactly. A mismatch is a decompression failure. A declared size above the caller's limit or the hard 1 GiB limit is a size-limit failure. -- `None` compression may be paired with `None` or `Crc32c`. `ZstdBestCompressionWithChecksum` may be paired with `HandledByCompressionLib` or `Crc32c`; with `Crc32c`, both the embedded zstd checksum and the external CRC-32C are checked. `HandledByCompressionLib` is invalid with `None` compression, and `None` checksum is invalid with `ZstdBestCompressionWithChecksum`. +- `None` compression may be paired with `None` or `Crc32c`. Either zstd mode may be paired with `HandledByCompressionLib` or `Crc32c`; with `Crc32c`, both the embedded zstd checksum and the external CRC-32C are checked. `HandledByCompressionLib` is invalid with `None` compression, and `None` checksum is invalid with a zstd mode. - `None` and `HandledByCompressionLib` require an empty checksum string. `Crc32c` requires its canonical eight-character checksum. Deserialisation verifies it after bounded decompression and before parsing the payload with `zpp::bits`; envelope deserialisation also requires the resulting size to exactly match `uncompressed_size`. - Decompression never allocates or produces output larger than 1 GiB. diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index d8e265a6..dad2fdbb 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -15,7 +15,6 @@ #include "cluster.h" #include "clusterize.h" #include "compact.h" -#include "encoded.h" #include "geometry/geometry.h" #include "numeric/int_math.h" #include "log.h" @@ -524,7 +523,6 @@ std::unordered_set build_level( options.preferred_extension = ".glb"; auto debug_result = octree::open_folder( ctx.output_storage.base_path().string() + "-debug", - false, std::move(options)); if (!debug_result.has_value()) { LOG_ERROR_AND_EXIT( diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h new file mode 100644 index 00000000..3209f4bb --- /dev/null +++ b/src/dag_builder/codec/Dag.h @@ -0,0 +1,129 @@ +#pragma once + +#include +#include + +#include "codec/DagFormat.h" +#include "io/envelope_file.h" +#include "store/Codec.h" +#include "store/codec/Path.h" + +namespace dag::codec { + +inline store::CodecError file_error( + const store::CodecOperation operation, + const io::envelope::FileError &error) +{ + const bool missing = std::holds_alternative(error) + && std::get(error) == io::Error(io::Error::OpenFile); + return { + operation, + missing ? store::CodecErrorCategory::FileNotFound + : operation == store::CodecOperation::Read + ? store::CodecErrorCategory::InvalidData + : store::CodecErrorCategory::Io, + io::envelope::describe_error(error), + }; +} + +class Dag final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override + { + return { + store::codec::append_extension(node_path, ".dag"), + store::codec::append_extension(node_path, ".dagmeta"), + }; + } + + std::expected read( + const store::NodePath &node_path) const override + { + const auto node_paths = paths(node_path); + auto clustering_payload = + io::envelope::read_from_path(node_paths[0]); + if (!clustering_payload) { + return std::unexpected(file_error( + store::CodecOperation::Read, + clustering_payload.error())); + } + auto metadata_payload = + io::envelope::read_from_path(node_paths[1]); + if (!metadata_payload) { + return std::unexpected(file_error( + store::CodecOperation::Read, + metadata_payload.error())); + } + auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); + if (!clustering) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + clustering.error(), + }); + } + auto metadata = dag::format::decode_metadata(std::move(*metadata_payload)); + if (!metadata) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + metadata.error(), + }); + } + return dag::ClusterBatch{ + .metadata = std::move(*metadata), + .clustering = std::move(*clustering), + }; + } + + std::expected write( + const store::NodePath &node_path, + const dag::ClusterBatch &batch) const override + { + auto clustering = dag::format::encode_clustering(batch.clustering); + if (!clustering) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + clustering.error(), + }); + } + const dag::format::NodeMetadata metadata = + dag::format::encode_metadata(batch.metadata); + if (auto valid = dag::format::validate(*clustering); !valid) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + valid.error(), + }); + } + if (auto valid = dag::format::validate(metadata); !valid) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + valid.error(), + }); + } + const auto node_paths = paths(node_path); + auto data_written = io::envelope::write_to_path( + *clustering, + node_paths[0]); + if (!data_written) { + return std::unexpected(file_error( + store::CodecOperation::Write, + data_written.error())); + } + auto metadata_written = io::envelope::write_to_path( + metadata, + node_paths[1]); + if (!metadata_written) { + return std::unexpected(file_error( + store::CodecOperation::Write, + metadata_written.error())); + } + return {}; + } +}; + +} // namespace dag::codec diff --git a/src/dag_builder/codec/DagFormat.h b/src/dag_builder/codec/DagFormat.h new file mode 100644 index 00000000..41bb19e8 --- /dev/null +++ b/src/dag_builder/codec/DagFormat.h @@ -0,0 +1,391 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "cluster.h" +#include "dag_node.h" +#include "io/envelope.h" +#include "mesh/io/texture.h" +#include "meshopt.h" + +namespace dag::format { + +namespace v1 { + +struct OctreeId { + std::uint8_t level; + std::uint64_t index; +}; + +struct Id { + OctreeId source_batch; + std::uint32_t cluster_index; +}; + +struct Vec3d { + double x; + double y; + double z; +}; + +struct Bounds { + Vec3d min; + Vec3d max; +}; + +struct Group { + std::vector children; + double error; + Bounds bounds; +}; + +struct NodeMetadata { + std::vector group_assignment; + std::vector groups; +}; + +struct Texture { + std::vector jpeg; +}; + +struct Cluster { + std::uint64_t triangle_count; + std::uint64_t vertex_count; + std::uint64_t uv_count; + std::vector encoded_triangles; + std::vector encoded_vertex_indices; + std::vector encoded_uvs; + std::uint32_t id; + std::uint32_t texture_id; + double absolute_error; +}; + +struct Clustering { + std::uint64_t vertex_count; + std::vector encoded_positions; + std::vector clusters; + std::vector textures; +}; + +} // namespace v1 + +using MetadataSchema = io::envelope::PayloadSchema< + "dag.NodeMetadata", + io::envelope::Version<1, v1::NodeMetadata>>; +using ClusteringSchema = io::envelope::PayloadSchema< + "dag.Clustering", + io::envelope::Version<1, v1::Clustering>>; +using NodeMetadata = MetadataSchema::latest_type; +using Clustering = ClusteringSchema::latest_type; + +inline v1::OctreeId encode_id(const octree::Id &id) +{ + return {.level = id.level(), .index = id.index_on_level()}; +} + +inline std::expected decode_id(const v1::OctreeId &id) +{ + auto result = octree::Id::try_make( + static_cast(id.level), + static_cast(id.index)); + if (!result) { + return std::unexpected("DAG payload contains an invalid octree ID"); + } + return *result; +} + +inline NodeMetadata encode_metadata(const dag::NodeMetadata &metadata) +{ + NodeMetadata result; + result.group_assignment = metadata.group_assignment; + result.groups.reserve(metadata.groups.size()); + for (const dag::Group &group : metadata.groups) { + v1::Group encoded_group{ + .children = {}, + .error = group.error, + .bounds = { + .min = {group.bounds.min.x, group.bounds.min.y, group.bounds.min.z}, + .max = {group.bounds.max.x, group.bounds.max.y, group.bounds.max.z}, + }, + }; + encoded_group.children.reserve(group.children.size()); + for (const dag::Id &child : group.children) { + encoded_group.children.push_back({ + .source_batch = encode_id(child.source_batch), + .cluster_index = child.cluster_index, + }); + } + result.groups.push_back(std::move(encoded_group)); + } + return result; +} + +inline std::expected validate(const NodeMetadata &metadata) +{ + for (const std::uint32_t group : metadata.group_assignment) { + if (group >= metadata.groups.size()) { + return std::unexpected("DAG metadata contains an invalid group assignment"); + } + } + for (const v1::Group &group : metadata.groups) { + if (!std::isfinite(group.error)) { + return std::unexpected("DAG metadata contains a non-finite error"); + } + for (const v1::Id &child : group.children) { + if (!decode_id(child.source_batch)) { + return std::unexpected("DAG metadata contains an invalid child ID"); + } + } + } + return {}; +} + +inline std::expected decode_metadata(NodeMetadata metadata) +{ + if (auto valid = validate(metadata); !valid) { + return std::unexpected(valid.error()); + } + dag::NodeMetadata result; + result.group_assignment = std::move(metadata.group_assignment); + result.groups.reserve(metadata.groups.size()); + for (v1::Group &group : metadata.groups) { + dag::Group decoded_group{ + .children = {}, + .error = group.error, + .bounds = { + {group.bounds.min.x, group.bounds.min.y, group.bounds.min.z}, + {group.bounds.max.x, group.bounds.max.y, group.bounds.max.z}, + }, + }; + decoded_group.children.reserve(group.children.size()); + for (const v1::Id &child : group.children) { + auto source_batch = decode_id(child.source_batch); + if (!source_batch) { + return std::unexpected(source_batch.error()); + } + decoded_group.children.push_back({*source_batch, child.cluster_index}); + } + result.groups.push_back(std::move(decoded_group)); + } + return result; +} + +inline std::expected encode_clustering( + const ::Clustering &clustering) +{ + Clustering result{ + .vertex_count = clustering.positions.size(), + .encoded_positions = meshopt::encode_vertex_buffer(clustering.positions), + .clusters = {}, + .textures = {}, + }; + result.clusters.reserve(clustering.clusters.size()); + for (const ::Cluster &cluster : clustering.clusters) { + if (cluster.texture_id + == (std::numeric_limits::max)()) { + return std::unexpected("DAG texture ID collides with the on-disk null sentinel"); + } + result.clusters.push_back({ + .triangle_count = cluster.local_triangles.size(), + .vertex_count = cluster.vertex_indices.size(), + .uv_count = cluster.uvs.size(), + .encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles), + .encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices), + .encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs), + .id = cluster.id, + .texture_id = cluster.texture_id.value_or((std::numeric_limits::max)()), + .absolute_error = cluster.absolute_error, + }); + } + result.textures.reserve(clustering.textures.size()); + try { + for (const cv::Mat &texture : clustering.textures) { + auto jpeg = mesh::io::write_texture_to_encoded_buffer(texture, ".jpeg"); + if (!texture.empty() && jpeg.empty()) { + return std::unexpected("could not encode DAG texture"); + } + result.textures.push_back({std::move(jpeg)}); + } + } catch (const cv::Exception &) { + return std::unexpected("could not encode DAG texture"); + } + return result; +} + +inline bool count_fits(const std::uint64_t count, const std::size_t element_size) +{ + return count <= std::numeric_limits::max() / element_size + && count * element_size <= io::envelope::default_max_decompressed_size; +} + +inline std::expected validate(const Clustering &clustering) +{ + if (!count_fits(clustering.vertex_count, sizeof(glm::dvec3))) { + return std::unexpected("DAG position count exceeds the allocation limit"); + } + if (clustering.vertex_count != 0 && clustering.encoded_positions.empty()) { + return std::unexpected("DAG position count and encoded data disagree"); + } + std::size_t decoded_size = + static_cast(clustering.vertex_count) * sizeof(glm::dvec3); + for (const v1::Cluster &cluster : clustering.clusters) { + if (!count_fits(cluster.triangle_count, sizeof(glm::uvec3)) + || !count_fits(cluster.vertex_count, sizeof(std::uint32_t)) + || !count_fits(cluster.uv_count, sizeof(glm::dvec2))) { + return std::unexpected("DAG cluster count exceeds the allocation limit"); + } + const std::size_t cluster_size = + static_cast(cluster.triangle_count) * sizeof(glm::uvec3) + + static_cast(cluster.vertex_count) * sizeof(std::uint32_t) + + static_cast(cluster.uv_count) * sizeof(glm::dvec2); + if (cluster_size > io::envelope::default_max_decompressed_size - decoded_size) { + return std::unexpected("DAG decoded data exceeds the allocation limit"); + } + decoded_size += cluster_size; + if (cluster.uv_count != 0 && cluster.uv_count != cluster.vertex_count) { + return std::unexpected("DAG cluster UV count does not match its vertex count"); + } + if ((cluster.triangle_count != 0 && cluster.encoded_triangles.empty()) + || (cluster.vertex_count != 0 && cluster.encoded_vertex_indices.empty()) + || (cluster.uv_count != 0 && cluster.encoded_uvs.empty())) { + return std::unexpected("DAG cluster count and encoded data disagree"); + } + if (cluster.texture_id != (std::numeric_limits::max)() + && cluster.texture_id >= clustering.textures.size()) { + return std::unexpected("DAG cluster contains an invalid texture reference"); + } + if (!std::isfinite(cluster.absolute_error)) { + return std::unexpected("DAG cluster contains a non-finite error"); + } + } + return {}; +} + +template +inline std::expected, std::string> decode_vertex_buffer( + const std::uint64_t count, + const std::vector &encoded, + const std::string_view label) +{ + if (!count_fits(count, sizeof(T))) { + return std::unexpected(std::string(label) + " count exceeds the allocation limit"); + } + std::vector result(static_cast(count)); + if (meshopt_decodeVertexBuffer( + result.data(), + result.size(), + sizeof(T), + encoded.data(), + encoded.size()) + != 0) { + return std::unexpected("could not decode " + std::string(label)); + } + return result; +} + +inline std::expected, std::string> decode_triangles( + const std::uint64_t count, + const std::vector &encoded) +{ + if (!count_fits(count, sizeof(glm::uvec3))) { + return std::unexpected("DAG triangle count exceeds the allocation limit"); + } + std::vector result(static_cast(count)); + if (meshopt_decodeIndexBuffer( + result.data(), + result.size() * 3, + sizeof(std::uint32_t), + encoded.data(), + encoded.size()) + != 0) { + return std::unexpected("could not decode DAG triangles"); + } + return result; +} + +inline std::expected<::Clustering, std::string> decode_clustering(Clustering clustering) +{ + if (auto valid = validate(clustering); !valid) { + return std::unexpected(valid.error()); + } + auto positions = decode_vertex_buffer( + clustering.vertex_count, + clustering.encoded_positions, + "DAG positions"); + if (!positions) { + return std::unexpected(positions.error()); + } + + ::Clustering result; + result.positions = std::move(*positions); + result.clusters.reserve(clustering.clusters.size()); + for (const v1::Cluster &encoded : clustering.clusters) { + auto triangles = decode_triangles(encoded.triangle_count, encoded.encoded_triangles); + auto vertex_indices = decode_vertex_buffer( + encoded.vertex_count, + encoded.encoded_vertex_indices, + "DAG vertex indices"); + auto uvs = decode_vertex_buffer( + encoded.uv_count, + encoded.encoded_uvs, + "DAG UVs"); + if (!triangles) { + return std::unexpected(triangles.error()); + } + if (!vertex_indices) { + return std::unexpected(vertex_indices.error()); + } + if (!uvs) { + return std::unexpected(uvs.error()); + } + for (const std::uint32_t vertex : *vertex_indices) { + if (vertex >= result.positions.size()) { + return std::unexpected("DAG cluster contains an invalid global vertex index"); + } + } + for (const glm::uvec3 triangle : *triangles) { + if (glm::any(glm::greaterThanEqual( + triangle, + glm::uvec3(static_cast(vertex_indices->size()))))) { + return std::unexpected("DAG cluster contains an invalid local triangle index"); + } + } + result.clusters.push_back({ + .id = encoded.id, + .vertex_indices = std::move(*vertex_indices), + .local_triangles = std::move(*triangles), + .uvs = std::move(*uvs), + .texture_id = encoded.texture_id == (std::numeric_limits::max)() + ? std::nullopt + : std::optional{encoded.texture_id}, + .absolute_error = encoded.absolute_error, + }); + } + + std::vector textures; + textures.reserve(clustering.textures.size()); + try { + for (const v1::Texture &texture : clustering.textures) { + cv::Mat image = mesh::io::read_texture_from_encoded_bytes(texture.jpeg); + if (image.empty()) { + return std::unexpected("could not decode DAG texture"); + } + textures.push_back(std::move(image)); + } + } catch (const cv::Exception &) { + return std::unexpected("could not decode DAG texture"); + } + result.textures = TextureSet(std::move(textures)); + return result; +} + +} // namespace dag::format diff --git a/src/dag_builder/codec/MetadataView.h b/src/dag_builder/codec/MetadataView.h index 9df6b856..c27dfa62 100644 --- a/src/dag_builder/codec/MetadataView.h +++ b/src/dag_builder/codec/MetadataView.h @@ -3,35 +3,39 @@ #include #include -#include - -#include "io/serialize.h" -#include "serialization.h" +#include "codec/Dag.h" +#include "codec/DagFormat.h" +#include "io/envelope_file.h" #include "store/Codec.h" -#include "store/codec/ZppBits.h" +#include "store/codec/Path.h" namespace dag::codec { class MetadataView final : public store::Codec { public: std::vector paths( - const store::NodePath &node_path) const override { - return {store::codec::append_extension(node_path, ".bin")}; + const store::NodePath &node_path) const override + { + return {store::codec::append_extension(node_path, ".dagmeta")}; } std::expected read( - const store::NodePath &node_path) const override { - const auto result = io::read_from_path(paths(node_path).front()); - if (!result.has_value()) { + const store::NodePath &node_path) const override + { + auto payload = io::envelope::read_from_path( + paths(node_path).front()); + if (!payload) { + return std::unexpected(file_error(store::CodecOperation::Read, payload.error())); + } + auto metadata = dag::format::decode_metadata(std::move(*payload)); + if (!metadata) { return std::unexpected(store::CodecError{ store::CodecOperation::Read, - result.error() == io::Error(io::Error::OpenFile) - ? store::CodecErrorCategory::FileNotFound - : store::CodecErrorCategory::Io, - fmt::format("{}", result.error()), + store::CodecErrorCategory::InvalidData, + metadata.error(), }); } - return result.value(); + return std::move(*metadata); } }; diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h index 3233f5af..53d558d1 100644 --- a/src/dag_builder/codec/from_extension.h +++ b/src/dag_builder/codec/from_extension.h @@ -7,16 +7,15 @@ #include #include "codec/MetadataView.h" +#include "codec/Dag.h" #include "dag_node.h" -#include "serialization.h" -#include "store/codec/ZppBits.h" namespace dag::codec { inline std::expected>, store::CodecError> from_extension(const std::string_view extension) { - if (extension == ".bin") { - return std::make_unique>(); + if (extension == ".dag") { + return std::make_unique(); } return std::unexpected(store::CodecError{ store::CodecOperation::Resolve, @@ -27,7 +26,7 @@ from_extension(const std::string_view extension) { inline std::expected>, store::CodecError> metadata_from_extension(const std::string_view extension) { - if (extension == ".bin") { + if (extension == ".dag") { return std::make_unique(); } return std::unexpected(store::CodecError{ diff --git a/src/dag_builder/encoded.h b/src/dag_builder/encoded.h deleted file mode 100644 index e9b3510a..00000000 --- a/src/dag_builder/encoded.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include - -#include - -#include "io/Error.h" -#include "io/serialize.h" -#include "serialization.h" - -inline std::expected save_clustering( - const Clustering &clustering, - const std::filesystem::path &path, - const bool make_dirs = true) { - return ::io::write_to_path(clustering, path, make_dirs); -} - -inline std::expected load_clustering( - const std::filesystem::path &path) { - return ::io::read_from_path(path); -} diff --git a/src/dag_builder/serialization.h b/src/dag_builder/serialization.h deleted file mode 100644 index b5c58481..00000000 --- a/src/dag_builder/serialization.h +++ /dev/null @@ -1,212 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - -#include "dag_node.h" -#include "glm_utils.h" -#include "io/bytes.h" -#include "mesh/io/texture.h" -#include "meshopt.h" - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const glm::vec &vector) { - return archive(as_span(vector)); -} - -template -auto serialize(Archive &archive, glm::vec &vector) { - return archive(as_span(vector)); -} - -template -auto serialize(Archive &archive, const dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -template -auto serialize(Archive &archive, dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -template -auto serialize(Archive &archive, const radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} - -template -auto serialize(Archive &archive, radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} - -template -auto serialize(Archive &archive, const dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} - -template -auto serialize(Archive &archive, dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} - -template -auto serialize(Archive &archive, const dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} - -template -auto serialize(Archive &archive, dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} - -template -auto serialize(Archive &archive, const dag::ClusterBatch &batch) { - return archive(batch.metadata, batch.clustering); -} - -template -auto serialize(Archive &archive, dag::ClusterBatch &batch) { - return archive(batch.metadata, batch.clustering); -} - -} // namespace zpp::bits - -template -auto serialize(Archive &archive, const TextureSet &textures) { - size_t size = textures.size(); - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v - && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - for (const auto &texture : textures) { - const mesh::io::ImageAndExt item{texture, ".jpeg"}; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - } - return result; - } -} - -template -auto serialize(Archive &archive, TextureSet &textures) { - size_t size; - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v - && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - std::vector images; - images.reserve(size); - for (size_t index = 0; index < size; ++index) { - mesh::io::ImageAndExt item; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - images.push_back(item.image); - } - textures = TextureSet(images); - return result; - } -} - -template -auto serialize(Archive &archive, const Cluster &cluster) { - const size_t triangle_count = cluster.triangle_count(); - const size_t vertex_count = cluster.vertex_count(); - const size_t uv_count = cluster.uvs.size(); - auto encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles); - auto encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices); - auto encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs); - return archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); -} - -template -auto serialize(Archive &archive, Cluster &cluster) { - size_t triangle_count; - size_t vertex_count; - size_t uv_count; - std::vector encoded_triangles; - std::vector encoded_vertex_indices; - std::vector encoded_uvs; - auto result = archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v - && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - meshopt::decode_index_buffer(cluster.local_triangles, triangle_count, encoded_triangles); - meshopt::decode_vertex_buffer(cluster.vertex_indices, vertex_count, encoded_vertex_indices); - meshopt::decode_vertex_buffer(cluster.uvs, uv_count, encoded_uvs); - return result; - } -} - -template -auto serialize(Archive &archive, const Clustering &clustering) { - const size_t vertex_count = clustering.vertex_count(); - auto encoded_positions = meshopt::encode_vertex_buffer(clustering.positions); - return archive(vertex_count, encoded_positions, clustering.clusters, clustering.textures); -} - -template -auto serialize(Archive &archive, Clustering &clustering) { - size_t vertex_count; - std::vector encoded_positions; - auto result = archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v - && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - meshopt::decode_vertex_buffer(clustering.positions, vertex_count, encoded_positions); - return result; - } -} diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index b3638172..77250463 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -7,11 +7,12 @@ #include "octree/StoreTraits.h" #include "octree/storage/IndexFile.h" #include "octree/storage/open_runtime.h" -#include "serialization.h" #include "store/IndexedStorage.h" namespace dag::storage { +inline constexpr std::string_view payload_class = "dag.ClusterBatch"; + using OpenOptions = octree::storage::OpenOptions; using Storage = store::Storage; using IndexedStorage = store::IndexedStorage; @@ -24,17 +25,17 @@ inline std::expected> open_index( return store::open_index( path, octree::storage::index_format(), + payload_class, dag::codec::from_extension); } inline std::expected> open_folder( const std::filesystem::path &path, - const bool create_index = false, OpenOptions options = {}) { return octree::storage::open_folder( path, - create_index, - ".bin", + std::string(payload_class), + ".dag", dag::codec::from_extension, std::move(options)); } @@ -43,7 +44,8 @@ inline std::expected> open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { return octree::storage::open_folder_indexed( path, - ".bin", + std::string(payload_class), + ".dag", dag::codec::from_extension, std::move(options)); } @@ -51,12 +53,13 @@ open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) inline std::expected> open_metadata_indexed(const std::filesystem::path &path) { const std::filesystem::path index_path = - path.filename() == octree::disk::v1::index_file_name() + path.filename() == octree::storage::index_file_name ? path - : path / octree::disk::v1::index_file_name(); + : path / octree::storage::index_file_name; return store::open_index( index_path, octree::storage::index_format(), + payload_class, dag::codec::metadata_from_extension); } diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index f4901974..6f21e90a 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -2,9 +2,8 @@ #include #include "cli.h" +#include "codec/Dag.h" #include "dag_node.h" -#include "encoded.h" -#include "io/serialize.h" #include "log.h" #include "mesh/io.h" #include "mesh/storage.h" @@ -12,15 +11,21 @@ #include "ProgressIndicator.h" #include "storage.h" #include "utils.h" -#include "serialization.h" #include "store/describe_error.h" namespace { void export_node(const cli::Args &args) { - const auto load_result = io::read_from_path(args.input_path); + if (args.input_path.extension() != ".dag") { + LOG_ERROR("Expected a .dag input file, got {}", args.input_path); + return; + } + std::filesystem::path node_path = args.input_path; + node_path.replace_extension(); + const dag::codec::Dag codec; + const auto load_result = codec.read(store::NodePath(node_path)); if (!load_result.has_value()) { - LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error()); + LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error().message); return; } @@ -47,7 +52,6 @@ void export_storage(const cli::Args &args) { options.preferred_extension = ".glb"; auto output_result = octree::open_folder( args.output_path, - false, std::move(options)); if (!output_result.has_value()) { LOG_ERROR( diff --git a/src/sf_builder/main.cpp b/src/sf_builder/main.cpp index 6bef9d68..e95c3464 100644 --- a/src/sf_builder/main.cpp +++ b/src/sf_builder/main.cpp @@ -232,7 +232,7 @@ int run(std::span args) { ->expected(2, 4); target->require_option(1); - single->add_option("--output", output_path, "Output path were the mesh is written to (.terrain, .gltf or .glb)") + single->add_option("--output", output_path, "Output path were the mesh is written to (.sfmesh, .gltf or .glb)") ->required(); single->add_option("--srs", target_srs_input, "EPSG code of the srs of the target bounds or id"); @@ -260,8 +260,8 @@ int run(std::span args) { ->required(); std::string output_format; batch->add_option("--format", output_format, "Output mesh format") - ->check(CLI::IsMember({".glb", ".gltf", ".terrain"})) - ->default_val(".terrain"); + ->check(CLI::IsMember({".glb", ".gltf", ".sfmesh"})) + ->default_val(".sfmesh"); uint32_t num_threads = 0; batch->add_option("--threads", num_threads, "Number of threads to use") ->check(CLI::PositiveNumber); diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 310e384d..22a4d392 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -198,7 +198,6 @@ std::expected build_all_patches( open_options.preferred_extension = output_format; auto storage_result = octree::open_folder( output_base_path, - false, std::move(open_options)); if (!storage_result.has_value()) { LOG_ERROR_AND_EXIT( diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 75518b5b..9fa72497 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -58,8 +58,8 @@ void run(const cli::MergeArgs& args) { std::filesystem::create_directories(args.output_path); octree::OpenOptions options; options.default_mapping = base_dataset.layout().mapping(); - options.preferred_extension = std::string(base_dataset.codec_selector().value_or(".terrain")); - auto output_result = octree::open_folder(args.output_path, false, std::move(options)); + options.preferred_extension = std::string(base_dataset.codec_selector().value_or(".sfmesh")); + auto output_result = octree::open_folder(args.output_path, std::move(options)); if (!output_result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to open output dataset {}: {}", diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 74014a09..773d223a 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -20,7 +20,6 @@ add_library(terrainlib mesh/igl/igl.cpp mesh/io/gltf.cpp - mesh/io/terrain.cpp mesh/io/texture.cpp mesh/io/utils.cpp diff --git a/src/terrainlib/io/Error.h b/src/terrainlib/io/Error.h index 831c9842..b4e7aece 100644 --- a/src/terrainlib/io/Error.h +++ b/src/terrainlib/io/Error.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include "log.h" @@ -14,8 +13,6 @@ class Error { WriteBytes, DetermineSize, ReadBytes, - Serialize, - Deserialize, OutOfMemory }; @@ -37,10 +34,6 @@ class Error { private: Value _value; - -public: - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; }; } // namespace io @@ -70,12 +63,6 @@ struct fmt::formatter { case io::Error::Value::ReadBytes: name = "ReadBytes"; break; - case io::Error::Value::Serialize: - name = "Serialize"; - break; - case io::Error::Value::Deserialize: - name = "Deserialize"; - break; case io::Error::Value::OutOfMemory: name = "OutOfMemory"; break; diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h index 4b36065b..d643e383 100644 --- a/src/terrainlib/io/envelope.h +++ b/src/terrainlib/io/envelope.h @@ -128,7 +128,13 @@ struct PayloadSchema { template std::expected serialize( const typename Schema::template payload_type &payload, - CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdBestCompressionWithChecksum, + CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, + ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); + +template +std::expected serialize( + const typename Schema::latest_type &payload, + CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template diff --git a/src/terrainlib/io/envelope.inl b/src/terrainlib/io/envelope.inl index c9055894..4fc57314 100644 --- a/src/terrainlib/io/envelope.inl +++ b/src/terrainlib/io/envelope.inl @@ -100,6 +100,18 @@ std::expected serialize( return detail::serialize_to_bytes(envelope); } +template +std::expected serialize( + const typename Schema::latest_type &payload, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + return serialize( + payload, + compression_algorithm, + checksum_algorithm); +} + template std::expected deserialize( const std::span bytes, diff --git a/src/terrainlib/io/envelope_file.h b/src/terrainlib/io/envelope_file.h new file mode 100644 index 00000000..f5e3cd6a --- /dev/null +++ b/src/terrainlib/io/envelope_file.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "io/bytes.h" +#include "io/envelope.h" + +namespace io::envelope { + +using FileError = std::variant<::io::Error, Error>; + +inline std::string describe_error(const Error &error) +{ + return "envelope error " + std::to_string(static_cast(error.code)); +} + +inline std::string describe_error(const FileError &error) +{ + return std::visit( + [](const auto &value) -> std::string { + using Value = std::remove_cvref_t; + if constexpr (std::same_as) { + return value.to_string(); + } else { + return describe_error(value); + } + }, + error); +} + +template +std::expected read_from_path( + const std::filesystem::path &path, + const std::size_t max_decompressed_size = default_max_decompressed_size) +{ + auto file_bytes = ::io::read_bytes_from_path(path); + if (!file_bytes) { + return std::unexpected(FileError{file_bytes.error()}); + } + const auto bytes = std::as_bytes(std::span{*file_bytes}); + auto result = deserialize(bytes, max_decompressed_size); + if (!result) { + return std::unexpected(FileError{result.error()}); + } + return std::move(*result); +} + +template +std::expected write_to_path( + const typename Schema::latest_type &payload, + const std::filesystem::path &path, + const bool make_dirs = true, + const CompressionAlgorithm compression_algorithm = + CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, + const ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib) +{ + auto serialized = serialize(payload, compression_algorithm, checksum_algorithm); + if (!serialized) { + return std::unexpected(FileError{serialized.error()}); + } + const auto bytes = std::span{ + reinterpret_cast(serialized->data()), + serialized->size(), + }; + auto result = ::io::write_bytes_to_path(bytes, path, make_dirs); + if (!result) { + return std::unexpected(FileError{result.error()}); + } + return {}; +} + +} // namespace io::envelope diff --git a/src/terrainlib/io/serialize.h b/src/terrainlib/io/serialize.h deleted file mode 100644 index ffda2d1f..00000000 --- a/src/terrainlib/io/serialize.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "io/Error.h" - -namespace io { - -template -std::expected, Error> write_to_bytes(const T &value); -template -std::expected read_from_bytes(const std::span bytes); - -template -std::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs = true); -template -std::expected read_from_path(const std::filesystem::path &path); - -} - -#include "io/serialize.inl" diff --git a/src/terrainlib/io/serialize.inl b/src/terrainlib/io/serialize.inl deleted file mode 100644 index 80324d03..00000000 --- a/src/terrainlib/io/serialize.inl +++ /dev/null @@ -1,88 +0,0 @@ -#pragma once - -#include - -#include "io/bytes.h" -#include "io/utils.h" -#include "type_utils.h" - -namespace io { - -template -std::expected, Error> write_to_bytes(const T &value) { - std::vector data; - zpp::bits::out out(data); - const auto result = out(value); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while writing {}: {}", type_name(), error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. - case std::errc::message_size: // message size is beyond the user defined allocation limits. - case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return std::unexpected(Error::OutOfMemory); - case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. - case std::errc::bad_message: // attempt to read a variant of unrecognized type. - case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return std::unexpected(Error::Serialize); - case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. - case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. - UNREACHABLE(); - default: - UNREACHABLE(); - } - } - return data; -} - -template -std::expected read_from_bytes(const std::span bytes) { - zpp::bits::in in(bytes); - T value; - const auto result = in(value); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while reading {}: {}", type_name(), error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. - case std::errc::message_size: // message size is beyond the user defined allocation limits. - case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return std::unexpected(Error::OutOfMemory); - case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. - case std::errc::bad_message: // attempt to read a variant of unrecognized type. - case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return std::unexpected(Error::Deserialize); - case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. - case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. - UNREACHABLE(); - default: - UNREACHABLE(); - } - } - return value; -} - -template -std::expected read_from_path(const std::filesystem::path &path) { - const auto result = read_bytes_from_path(path); - if (!result.has_value()) { - return std::unexpected(result.error()); - } - const std::vector bytes = result.value(); - return read_from_bytes(bytes); -} - -template -std::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs) { - const auto result = write_to_bytes(value); - if (!result.has_value()) { - return std::unexpected(result.error()); - } - - const std::vector bytes = result.value(); - return write_bytes_to_path(bytes, path, make_dirs); -} - -} // namespace io diff --git a/src/terrainlib/mesh/EncodedMesh.h b/src/terrainlib/mesh/EncodedMesh.h index 4e9e4610..fd0927a6 100644 --- a/src/terrainlib/mesh/EncodedMesh.h +++ b/src/terrainlib/mesh/EncodedMesh.h @@ -3,8 +3,6 @@ #include #include -#include - namespace mesh { class Encoded { @@ -20,7 +18,6 @@ class Encoded { bool operator!=(const Header &) const = default; }; - using serialize = zpp::bits::members<5>; Header header; std::vector triangles; std::vector positions; diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index 9db26fe9..f2ea337a 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -8,7 +8,7 @@ #include "mesh/SimpleMesh.h" #include "mesh/io/gltf.h" #include "store/Codec.h" -#include "store/codec/ZppBits.h" +#include "store/codec/Path.h" namespace mesh::codec { diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h new file mode 100644 index 00000000..33fb2c80 --- /dev/null +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include + +#include "io/envelope_file.h" +#include "mesh/SimpleMesh.h" +#include "mesh/codec/SfMeshFormat.h" +#include "store/Codec.h" +#include "store/codec/Path.h" + +namespace mesh::codec { + +class SfMesh final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override + { + return {store::codec::append_extension(node_path, ".sfmesh")}; + } + + std::expected read( + const store::NodePath &node_path) const override + { + auto payload = ::io::envelope::read_from_path( + paths(node_path).front()); + if (!payload) { + const bool missing = std::holds_alternative<::io::Error>(payload.error()) + && std::get<::io::Error>(payload.error()) == ::io::Error(::io::Error::OpenFile); + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + missing ? store::CodecErrorCategory::FileNotFound + : store::CodecErrorCategory::InvalidData, + ::io::envelope::describe_error(payload.error()), + }); + } + if (auto valid = mesh::sf::validate(*payload); !valid) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + valid.error(), + }); + } + auto decoded = mesh::sf::decode_payload(std::move(*payload)); + if (!decoded) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + "could not decode SF mesh payload", + }); + } + return std::move(*decoded); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh) const override + { + return write(node_path, mesh, {}); + } + + std::expected write( + const store::NodePath &node_path, + const mesh::Simple &mesh, + const mesh::EncodeOptions options) const + { + auto payload = mesh::sf::encode_payload(mesh, options); + if (!payload) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + "could not encode SF mesh payload", + }); + } + if (auto valid = mesh::sf::validate(*payload); !valid) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Domain, + valid.error(), + }); + } + auto result = ::io::envelope::write_to_path( + *payload, + paths(node_path).front()); + if (!result) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + ::io::envelope::describe_error(result.error()), + }); + } + return {}; + } +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/SfMeshFormat.h b/src/terrainlib/mesh/codec/SfMeshFormat.h new file mode 100644 index 00000000..b1461505 --- /dev/null +++ b/src/terrainlib/mesh/codec/SfMeshFormat.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include + +#include + +#include "io/envelope.h" +#include "mesh/EncodedMesh.h" +#include "mesh/SimpleMesh.h" +#include "mesh/encode.h" + +namespace mesh::sf { + +namespace v1 { + +struct Payload { + std::uint32_t vertex_count; + std::uint32_t face_count; + std::vector triangles; + std::vector positions; + std::vector uvs; + std::vector texture; +}; + +} // namespace v1 + +using Schema = ::io::envelope::PayloadSchema< + "mesh.SfMesh", + ::io::envelope::Version<1, v1::Payload>>; +using Payload = Schema::latest_type; + +inline std::expected validate(const Payload &payload) +{ + if (payload.vertex_count + > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Position)) { + return std::unexpected("SF mesh vertex count exceeds the allocation limit"); + } + if (payload.face_count + > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Triangle)) { + return std::unexpected("SF mesh face count exceeds the allocation limit"); + } + if (payload.vertex_count == 0 + && (!payload.positions.empty() || !payload.uvs.empty())) { + return std::unexpected("empty mesh contains vertex data"); + } + if (payload.vertex_count != 0 && payload.positions.empty()) { + return std::unexpected("non-empty mesh contains no position data"); + } + if (payload.face_count == 0 && !payload.triangles.empty()) { + return std::unexpected("empty mesh contains triangle data"); + } + if (payload.face_count != 0 && payload.triangles.empty()) { + return std::unexpected("non-empty mesh contains no triangle data"); + } + return {}; +} + +inline std::expected encode_payload( + const mesh::Simple &mesh, + const mesh::EncodeOptions options = {}) +{ + auto encoded = mesh::encode(mesh, options); + if (!encoded) { + return std::unexpected(encoded.error()); + } + return Payload{ + .vertex_count = encoded->header.vertex_count, + .face_count = encoded->header.face_count, + .triangles = std::move(encoded->triangles), + .positions = std::move(encoded->positions), + .uvs = std::move(encoded->uvs), + .texture = std::move(encoded->texture), + }; +} + +inline std::expected decode_payload(Payload payload) +{ + const mesh::Encoded encoded{ + .header = { + .version = 1, + .n_dims = mesh::Simple::n_dims, + .component_type = mesh::component_type_id(), + .vertex_count = payload.vertex_count, + .face_count = payload.face_count, + }, + .triangles = std::move(payload.triangles), + .positions = std::move(payload.positions), + .uvs = std::move(payload.uvs), + .texture = std::move(payload.texture), + }; + return mesh::decode(encoded); +} + +} // namespace mesh::sf diff --git a/src/terrainlib/mesh/codec/Terrain.h b/src/terrainlib/mesh/codec/Terrain.h deleted file mode 100644 index d1356e76..00000000 --- a/src/terrainlib/mesh/codec/Terrain.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include - -#include "mesh/SimpleMesh.h" -#include "mesh/io/terrain.h" -#include "store/Codec.h" -#include "store/codec/ZppBits.h" - -namespace mesh::codec { - -class Terrain final : public store::Codec { -public: - std::vector paths(const store::NodePath &node_path) const override { - return {store::codec::append_extension(node_path, ".terrain")}; - } - - std::expected read( - const store::NodePath &node_path) const override { - const auto result = mesh::io::terrain::load_from_path(paths(node_path).front()); - if (!result.has_value()) { - return std::unexpected(store::CodecError{ - store::CodecOperation::Read, - result.error() == mesh::io::LoadMeshErrorKind::FileNotFound - ? store::CodecErrorCategory::FileNotFound - : store::CodecErrorCategory::InvalidData, - result.error().description(), - }); - } - return result.value(); - } - - std::expected write( - const store::NodePath &node_path, - const mesh::Simple &mesh) const override { - const auto result = mesh::io::terrain::save_to_path(mesh, paths(node_path).front()); - if (!result.has_value()) { - return std::unexpected(store::CodecError{ - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - result.error().description(), - }); - } - return {}; - } -}; - -} // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/from_extension.h b/src/terrainlib/mesh/codec/from_extension.h index abc18813..21209dbc 100644 --- a/src/terrainlib/mesh/codec/from_extension.h +++ b/src/terrainlib/mesh/codec/from_extension.h @@ -7,15 +7,15 @@ #include #include "mesh/codec/Gltf.h" -#include "mesh/codec/Terrain.h" +#include "mesh/codec/SfMesh.h" #include "store/Codec.h" namespace mesh::codec { inline std::expected>, store::CodecError> from_extension(const std::string_view extension) { - if (extension == ".terrain") { - return std::make_unique(); + if (extension == ".sfmesh") { + return std::make_unique(); } if (extension == ".glb") { return std::make_unique(GltfContainer::Binary); diff --git a/src/terrainlib/mesh/io.cpp b/src/terrainlib/mesh/io.cpp index 467e0537..d1a9be1d 100644 --- a/src/terrainlib/mesh/io.cpp +++ b/src/terrainlib/mesh/io.cpp @@ -1,7 +1,7 @@ #include "mesh/io.h" #include "log.h" #include "mesh/io/gltf.h" -#include "mesh/io/terrain.h" +#include "mesh/codec/SfMesh.h" #include "mesh/validate.h" namespace mesh::io { @@ -12,8 +12,15 @@ std::expected load_from_path( const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { return gltf::load_from_path(path, options); - } else if (extension == ".terrain") { - return terrain::load_from_path(path, options); + } else if (extension == ".sfmesh") { + std::filesystem::path node_path = path; + node_path.replace_extension(); + const mesh::codec::SfMesh codec; + auto result = codec.read(store::NodePath(node_path)); + if (!result) { + return std::unexpected(LoadMeshErrorKind::InvalidFormat); + } + return std::move(*result); } else { return std::unexpected(LoadMeshErrorKind::UnsupportedFormat); } @@ -30,8 +37,18 @@ std::expected save_to_path( const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { return gltf::save_to_path(mesh, path, options); - } else if (extension == ".terrain") { - return terrain::save_to_path(mesh, path, options); + } else if (extension == ".sfmesh") { + std::filesystem::path node_path = path; + node_path.replace_extension(); + const mesh::codec::SfMesh codec; + auto result = codec.write( + store::NodePath(node_path), + mesh, + mesh::EncodeOptions{.texture_format = options.texture_format}); + if (!result) { + return std::unexpected(SaveMeshErrorKind::WriteFile); + } + return {}; } else { return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); } diff --git a/src/terrainlib/mesh/io/options.h b/src/terrainlib/mesh/io/options.h index e7c2ec27..332ade5e 100644 --- a/src/terrainlib/mesh/io/options.h +++ b/src/terrainlib/mesh/io/options.h @@ -11,7 +11,7 @@ struct LoadOptions { struct SaveOptions { std::string texture_format = ".jpeg"; - std::string name = "Terrain"; + std::string name = "Octree"; std::unordered_map metadata = {}; }; diff --git a/src/terrainlib/mesh/io/terrain.cpp b/src/terrainlib/mesh/io/terrain.cpp deleted file mode 100644 index d443d6f8..00000000 --- a/src/terrainlib/mesh/io/terrain.cpp +++ /dev/null @@ -1,172 +0,0 @@ -#include - -#include - -#include "io/bytes.h" -#include "log.h" -#include "mesh/io/terrain.h" -#include "mesh/EncodedMesh.h" -#include "mesh/encode.h" - -namespace mesh::io::terrain { - -namespace { -LoadMeshError load_error_from_io_error(::io::Error error) { - switch (error) { - case ::io::Error::Value::OpenFile: - return LoadMeshErrorKind::FileNotFound; - case ::io::Error::Value::DetermineSize: - case ::io::Error::Value::ReadBytes: - return LoadMeshErrorKind::InvalidFormat; - default: - return LoadMeshErrorKind::InvalidFormat; - } -} - -SaveMeshError save_error_from_io_error(::io::Error error) { - switch (error) { - case ::io::Error::Value::OpenFile: - return SaveMeshErrorKind::OpenFile; - case ::io::Error::Value::WriteBytes: - return SaveMeshErrorKind::WriteFile; - default: - return SaveMeshErrorKind::WriteFile; - } -} - -std::expected write_bytes_to_path( - const std::span bytes, const std::filesystem::path &path) { - const auto result = ::io::write_bytes_to_path(bytes, path); - if (!result.has_value()) { - return std::unexpected(save_error_from_io_error(result.error())); - } - return {}; -} - -std::expected, LoadMeshError> read_bytes_from_path(const std::filesystem::path &path) { - const auto result = ::io::read_bytes_from_path(path); - if (!result.has_value()) { - return std::unexpected(load_error_from_io_error(result.error())); - } - return result.value(); -} - -std::expected, SaveMeshError> save_encoded_to_buffer(const mesh::Encoded &mesh) { - LOG_TRACE("Serializing mesh to buffer"); - - // TODO: this ignores the texture format in SaveOptions - std::vector data; - zpp::bits::out out(data); - auto result = out(mesh); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("Error while writing mesh: {}", error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: - case std::errc::message_size: - case std::errc::result_out_of_range: - return std::unexpected(SaveMeshErrorKind::OutOfMemory); - break; - default: - UNREACHABLE(); - break; - } - } - - return data; -} - -std::expected load_encoded_from_buffer(const std::span bytes) { - LOG_TRACE("Deserializing mesh from buffer"); - - zpp::bits::in in(bytes); - mesh::Encoded mesh; - auto result = in(mesh); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while reading mesh: {}", error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: - case std::errc::message_size: - return std::unexpected(LoadMeshErrorKind::OutOfMemory); - case std::errc::value_too_large: - case std::errc::bad_message: - case std::errc::protocol_error: - case std::errc::result_out_of_range: - return std::unexpected(LoadMeshErrorKind::InvalidFormat); - case std::errc::not_supported: - case std::errc::invalid_argument: - UNREACHABLE(); - break; - default: - throw std::runtime_error("unexpected error"); - } - } - - return mesh; -} -} - -std::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions& options) { - const auto encode_result = mesh::encode(mesh, mesh::EncodeOptions{ - .texture_format = options.texture_format}); - if (!encode_result.has_value()) { - return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); - } - const mesh::Encoded encoded = encode_result.value(); - - const auto deser_result = save_encoded_to_buffer(encoded); - if (!deser_result.has_value()) { - return std::unexpected(deser_result.error()); - } - const std::vector buffer = deser_result.value(); - - return buffer; -} - -std::expected load_from_buffer(const std::span bytes, const LoadOptions & /* options */) { - const auto deser_result = load_encoded_from_buffer(bytes); - if (!deser_result.has_value()) { - return std::unexpected(deser_result.error()); - } - const mesh::Encoded encoded = deser_result.value(); - - const auto decode_result = mesh::decode(encoded); - if (!decode_result.has_value()) { - return std::unexpected(LoadMeshErrorKind::InvalidFormat); - } - const mesh::Simple mesh = decode_result.value(); - - return mesh; -} - -std::expected load_from_path(const std::filesystem::path &path, const LoadOptions & /* options */) { - const auto bytes_result = read_bytes_from_path(path); - if (!bytes_result.has_value()) { - return std::unexpected(bytes_result.error()); - } - const std::vector bytes = bytes_result.value(); - - return load_from_buffer(bytes); -} - -std::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options) { - LOG_TRACE("Saving mesh as high precision terrain"); - - const auto result = save_to_buffer(mesh, options); - if (!result.has_value()) { - return std::unexpected(result.error()); - } - const std::vector bytes = result.value(); - - const auto write_result = write_bytes_to_path(bytes, path); - if (!write_result.has_value()) { - return std::unexpected(write_result.error()); - } - - return {}; -} - -} // namespace mesh::io::terrain diff --git a/src/terrainlib/mesh/io/terrain.h b/src/terrainlib/mesh/io/terrain.h deleted file mode 100644 index fd097a24..00000000 --- a/src/terrainlib/mesh/io/terrain.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "mesh/SimpleMesh.h" -#include "mesh/io/error.h" -#include "mesh/io/options.h" - -namespace mesh::io::terrain { - -std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options = {}); -std::expected load_from_buffer(const std::span buffer, const LoadOptions &options = {}); - -std::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -std::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions &options = {}); - -} // namespace mesh::io::terrain diff --git a/src/terrainlib/mesh/io/texture.h b/src/terrainlib/mesh/io/texture.h index c6849014..e110878e 100644 --- a/src/terrainlib/mesh/io/texture.h +++ b/src/terrainlib/mesh/io/texture.h @@ -3,7 +3,6 @@ #include #include #include -#include #include @@ -21,33 +20,3 @@ std::vector write_texture_to_encoded_buffer(const cv::Mat &image, const std::vector write_texture_to_encoded_buffer(const ImageAndExt &item); } - -#include - -namespace mesh::io { - -template -auto serialize(Archive &archive, const mesh::io::ImageAndExt &item) { - const std::vector encoded = mesh::io::write_texture_to_encoded_buffer(item.image, item.ext); - return archive(item.ext, encoded); -} - -template -auto serialize(Archive &archive, mesh::io::ImageAndExt &item) { - std::vector encoded; - - auto result = archive(item.ext, encoded); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - item.image = mesh::io::read_texture_from_encoded_bytes(encoded); - return result; - } -} - -} // namespace mesh::io diff --git a/src/terrainlib/mesh/storage.h b/src/terrainlib/mesh/storage.h index 102f6afc..a26ac905 100644 --- a/src/terrainlib/mesh/storage.h +++ b/src/terrainlib/mesh/storage.h @@ -12,6 +12,8 @@ namespace mesh::storage { +inline constexpr std::string_view payload_class = "mesh.Simple3d"; + using Storage = store::Storage; using IndexedStorage = store::IndexedStorage; using OpenOptions = octree::storage::OpenOptions; @@ -21,17 +23,17 @@ inline std::expected> open_index( return store::open_index( path, octree::storage::index_format(), + payload_class, mesh::codec::from_extension); } inline std::expected> open_folder( const std::filesystem::path &path, - const bool create_index = false, OpenOptions options = {}) { return octree::storage::open_folder( path, - create_index, - ".terrain", + std::string(payload_class), + ".sfmesh", mesh::codec::from_extension, std::move(options)); } @@ -41,7 +43,8 @@ inline std::expected> open_folder_i OpenOptions options = {}) { return octree::storage::open_folder_indexed( path, - ".terrain", + std::string(payload_class), + ".sfmesh", mesh::codec::from_extension, std::move(options)); } diff --git a/src/terrainlib/octree/Id.h b/src/terrainlib/octree/Id.h index 44518c71..f1619cb3 100644 --- a/src/terrainlib/octree/Id.h +++ b/src/terrainlib/octree/Id.h @@ -98,7 +98,7 @@ class Id_ { return Coords(max_coord_on_level(level)); } - constexpr Id_() = default; // zpp::bits requires a default constructor + constexpr Id_() = default; template requires(sizeof...(Coordinates) == Dimensions && (std::convertible_to && ...)) constexpr Id_(const Level level, Coordinates... coordinates) : Id_(level, Coords{coordinates...}) {} @@ -399,51 +399,3 @@ struct hash> { } }; } // namespace std - -#include -namespace octree { -namespace { -constexpr zpp::bits::errc success() { - return zpp::bits::errc(std::errc()); -} -} // namespace - -template < - size_t Dimensions, - size_t MaxLevel, - typename LevelT, - typename IndexT, - typename CoordT> -constexpr auto serialize(auto &archive, octree::Id_ &id) { - using Id = octree::Id_; - typename Id::Level level; - typename Id::Index index; - auto result = archive(level, index); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - auto maybe_id = Id::try_make(level, index); - if (!maybe_id) { - return zpp::bits::errc(std::errc::bad_message); - } - - id = *maybe_id; - return success(); - } -} - -template < - size_t Dimensions, - size_t MaxLevel, - typename LevelT, - typename IndexT, - typename CoordT> -constexpr auto serialize(auto &archive, const octree::Id_ &id) { - return archive(id.level(), id.index_on_level()); -} -} // namespace octree diff --git a/src/terrainlib/octree/disk/IndexFile.h b/src/terrainlib/octree/disk/IndexFile.h deleted file mode 100644 index 92baacd9..00000000 --- a/src/terrainlib/octree/disk/IndexFile.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/StoreTraits.h" -#include "store/Index.h" - -namespace octree::disk::v1 { - -inline constexpr std::string_view index_file_name() { - return "terrain.index"; -} -inline constexpr std::string_view index_extension() { - return ".index"; -} - -struct IndexFile { - using serialize = zpp::bits::members<3>; - - std::string layout_strategy_id; - std::string preferred_extension; - store::Index map; -}; -} diff --git a/src/terrainlib/octree/storage/Format.h b/src/terrainlib/octree/storage/Format.h new file mode 100644 index 00000000..e7e48121 --- /dev/null +++ b/src/terrainlib/octree/storage/Format.h @@ -0,0 +1,138 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "io/envelope.h" +#include "octree/StoreTraits.h" +#include "store/Index.h" + +namespace octree::storage { + +inline constexpr std::string_view metadata_file_name = "octree.storemeta"; +inline constexpr std::string_view index_file_name = "octree.storeindex"; + +namespace v1 { + +struct StoreMetadata { + std::string layout_id; + std::string payload_class; + std::string codec_selector; +}; + +struct IndexEntry { + std::uint8_t level; + std::uint64_t index; + std::uint8_t status; +}; + +struct StoreIndex { + std::vector entries; +}; + +} // namespace v1 + +using StoreMetadataSchema = io::envelope::PayloadSchema< + "octree.StoreMetadata", + io::envelope::Version<1, v1::StoreMetadata>>; +using StoreIndexSchema = io::envelope::PayloadSchema< + "octree.StoreIndex", + io::envelope::Version<1, v1::StoreIndex>>; + +using StoreMetadata = StoreMetadataSchema::latest_type; +using StoreIndex = StoreIndexSchema::latest_type; + +inline std::expected validate(const StoreMetadata &metadata) +{ + if (metadata.layout_id.empty()) { + return std::unexpected("layout ID is empty"); + } + if (metadata.payload_class.empty()) { + return std::unexpected("payload class is empty"); + } + if (metadata.codec_selector.empty()) { + return std::unexpected("codec selector is empty"); + } + return {}; +} + +inline StoreIndex encode_index(const store::Index &index) +{ + StoreIndex result; + result.entries.reserve(index.size()); + for (const auto &[id, status] : index) { + result.entries.push_back({ + .level = id.level(), + .index = id.index_on_level(), + .status = static_cast(static_cast(status)), + }); + } + return result; +} + +inline std::expected, std::string> decode_index( + const StoreIndex &encoded) +{ + store::Index result; + std::unordered_set seen; + for (const v1::IndexEntry &entry : encoded.entries) { + const auto id = Id::try_make( + static_cast(entry.level), + static_cast(entry.index)); + if (!id) { + return std::unexpected("index contains an invalid octree ID"); + } + if (!seen.insert(*id).second) { + return std::unexpected("index contains a duplicate octree ID"); + } + if (entry.status > static_cast(store::NodeStatus::Virtual)) { + return std::unexpected("index contains an invalid node status"); + } + const auto set = result.set_raw( + *id, + store::NodeStatus{static_cast(entry.status)}); + if (!set) { + return std::unexpected("index contains an invalid octree ID"); + } + } + + for (const auto &[id, status] : result) { + const auto children = StoreTraits::children(id); + bool has_child = false; + if (children) { + for (const Id &child : *children) { + const auto child_status = result.get(child); + if (!child_status) { + return std::unexpected("index child lookup failed"); + } + has_child = has_child || child_status->has_value(); + } + } + if ((status == store::NodeStatus::Leaf && has_child) + || ((status == store::NodeStatus::Inner + || status == store::NodeStatus::Virtual) + && !has_child)) { + return std::unexpected("index contains inconsistent node topology"); + } + + const auto parent = StoreTraits::parent(id); + if (!parent) { + if (id != StoreTraits::root()) { + return std::unexpected("index contains a non-root node without a parent"); + } + continue; + } + const auto parent_status = result.get(*parent); + if (!parent_status || !parent_status->has_value() + || parent_status->value() == store::NodeStatus::Leaf) { + return std::unexpected("index contains a node without a valid indexed parent"); + } + } + return result; +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index b296a755..1ef8f9b5 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -2,61 +2,123 @@ #include -#include - -#include "io/serialize.h" +#include "io/envelope_file.h" #include "octree/StoreTraits.h" -#include "octree/disk/IndexFile.h" +#include "octree/storage/Format.h" #include "octree/store_layout/Mappings.h" #include "store/IndexFormat.h" namespace octree::storage { +inline store::IndexFormatError read_error( + const std::filesystem::path &path, + const io::envelope::FileError &error) +{ + const bool missing = std::holds_alternative(error) + && std::get(error) == io::Error(io::Error::OpenFile); + return { + missing ? store::IndexFormatErrorCategory::Open + : store::IndexFormatErrorCategory::Malformed, + path, + io::envelope::describe_error(error), + }; +} + +inline std::expected read_store_metadata( + const std::filesystem::path &base_path) +{ + const std::filesystem::path path = base_path / metadata_file_name; + auto metadata = io::envelope::read_from_path(path); + if (!metadata) { + return std::unexpected(read_error(path, metadata.error())); + } + if (auto valid = validate(*metadata); !valid) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Malformed, + path, + valid.error(), + }); + } + return std::move(*metadata); +} + inline std::expected, store::IndexFormatError> -read_index_file(const std::filesystem::path &path) { - auto result = io::read_from_path(path); - if (!result.has_value()) { +read_index_file(const std::filesystem::path &path) +{ + auto metadata = read_store_metadata(path.parent_path()); + if (!metadata) { + return std::unexpected(metadata.error()); + } + + auto encoded_index = io::envelope::read_from_path(path); + if (!encoded_index) { + return std::unexpected(read_error(path, encoded_index.error())); + } + auto index = decode_index(*encoded_index); + if (!index) { return std::unexpected(store::IndexFormatError{ - result.error() == io::Error(io::Error::OpenFile) - ? store::IndexFormatErrorCategory::Open - : store::IndexFormatErrorCategory::Malformed, + store::IndexFormatErrorCategory::Malformed, path, - fmt::format("{}", result.error()), + index.error(), }); } - disk::v1::IndexFile file = std::move(result.value()); return store::IndexMetadata{ - std::move(file.map), - std::move(file.layout_strategy_id), - std::move(file.preferred_extension), + std::move(*index), + std::move(metadata->layout_id), + std::move(metadata->payload_class), + std::move(metadata->codec_selector), }; } inline std::expected write_index_file( const std::filesystem::path &path, - const store::IndexMetadata &metadata) { - disk::v1::IndexFile file; - file.layout_strategy_id = metadata.layout_id; - file.preferred_extension = metadata.codec_selector; - file.map = metadata.index; - const auto result = io::write_to_path(file, path); - if (!result.has_value()) { + const store::IndexMetadata &metadata) +{ + const StoreMetadata store_metadata{ + .layout_id = metadata.layout_id, + .payload_class = metadata.payload_class, + .codec_selector = metadata.codec_selector, + }; + if (auto valid = validate(store_metadata); !valid) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Write, + path.parent_path() / metadata_file_name, + valid.error(), + }); + } + const std::filesystem::path metadata_path = path.parent_path() / metadata_file_name; + if (auto written = io::envelope::write_to_path( + store_metadata, + metadata_path); + !written) { + return std::unexpected(store::IndexFormatError{ + store::IndexFormatErrorCategory::Write, + metadata_path, + io::envelope::describe_error(written.error()), + }); + } + + const StoreIndex encoded_index = encode_index(metadata.index); + if (auto written = io::envelope::write_to_path(encoded_index, path); + !written) { return std::unexpected(store::IndexFormatError{ store::IndexFormatErrorCategory::Write, path, - fmt::format("{}", result.error()), + io::envelope::describe_error(written.error()), }); } return {}; } -inline store::PathMapping default_mapping() { +inline store::PathMapping default_mapping() +{ return store_layout::level_and_coordinate_directories(); } -inline store::IndexFormat index_format() { +inline store::IndexFormat index_format() +{ return { - disk::v1::index_file_name(), + index_file_name, read_index_file, write_index_file, store_layout::from_id, diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h index c15b1a0b..cfd10db0 100644 --- a/src/terrainlib/octree/storage/open.h +++ b/src/terrainlib/octree/storage/open.h @@ -16,9 +16,8 @@ inline std::expected> open_i inline std::expected> open_folder( const std::filesystem::path &path, - const bool create_index = false, OpenOptions options = {}) { - return mesh::storage::open_folder(path, create_index, std::move(options)); + return mesh::storage::open_folder(path, std::move(options)); } inline std::expected> open_folder_indexed( diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index 64070609..1994bfd9 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -1,13 +1,10 @@ #pragma once -#include #include #include #include #include -#include #include -#include #include @@ -22,78 +19,14 @@ struct OpenOptions { std::optional preferred_extension; }; -struct DiscoveredLayout { - store::PathMapping mapping; - std::string codec_selector; -}; - -template -std::optional discover_layout( - const std::filesystem::path &base_path, - CodecResolver &resolve_codec) { - struct Candidate { - std::filesystem::path path; - std::string extension; - }; - std::vector candidates; - std::unordered_map extension_counts; - std::error_code error; - for (std::filesystem::recursive_directory_iterator iterator(base_path, error), end; - !error && iterator != end; - iterator.increment(error)) { - if (!iterator->is_regular_file(error) && !iterator->is_symlink(error)) { - if (error) { - break; - } - continue; - } - if (iterator->path().filename() == disk::v1::index_file_name()) { - continue; - } - const std::string extension = iterator->path().extension().string(); - if (extension.empty() || !resolve_codec(extension).has_value()) { - continue; - } - candidates.push_back({iterator->path(), extension}); - ++extension_counts[extension]; - } - if (error || candidates.empty()) { - return std::nullopt; - } - - std::optional> best_mapping; - size_t best_match_count = 0; - for (const auto mapping : store_layout::all()) { - const store::Layout layout(base_path, mapping); - size_t matches = 0; - for (const Candidate &candidate : candidates) { - std::filesystem::path node_path = candidate.path; - node_path.replace_extension(); - matches += layout.key_from_node_path(store::NodePath(node_path)).has_value(); - } - if (matches > best_match_count) { - best_mapping = mapping; - best_match_count = matches; - } - } - if (!best_mapping.has_value()) { - return std::nullopt; - } - - const auto extension = std::max_element( - extension_counts.begin(), - extension_counts.end(), - [](const auto &left, const auto &right) { return left.second < right.second; }); - return DiscoveredLayout{best_mapping.value(), extension->first}; -} - template std::expected, store::OpenError> open_folder( const std::filesystem::path &base_path, - const bool create_index, + std::string payload_class, std::string default_codec_selector, CodecResolver resolve_codec, - OpenOptions options = {}) { + OpenOptions options = {}) +{ std::error_code error; const bool base_exists = std::filesystem::exists(base_path, error); if (error) { @@ -123,6 +56,7 @@ std::expected, store::OpenError> open_ const auto format = index_format(); const std::filesystem::path index_path = base_path / format.index_filename; + const std::filesystem::path metadata_path = base_path / metadata_file_name; const bool index_exists = std::filesystem::exists(index_path, error); if (error) { return std::unexpected(store::OpenError(store::FilesystemError{ @@ -131,66 +65,70 @@ std::expected, store::OpenError> open_ error, })); } + const bool metadata_exists = std::filesystem::exists(metadata_path, error); + if (error) { + return std::unexpected(store::OpenError(store::FilesystemError{ + metadata_path, + "exists", + error, + })); + } + if (index_exists != metadata_exists) { + return std::unexpected(store::OpenError(store::IndexFormatError{ + store::IndexFormatErrorCategory::Open, + index_exists ? metadata_path : index_path, + "octree dataset metadata and index must either both exist or both be absent", + })); + } if (index_exists) { auto indexed = store::open_index( index_path, format, + payload_class, resolve_codec); - if (!indexed.has_value()) { + if (!indexed) { return std::unexpected(indexed.error()); } - return store::Storage(std::move(indexed.value())); + return store::Storage(std::move(*indexed)); } - const auto discovered = discover_layout(base_path, resolve_codec); - const store::PathMapping mapping = discovered.has_value() - ? discovered->mapping - : options.default_mapping.value_or(format.default_mapping()); - std::string codec_selector = discovered.has_value() - ? discovered->codec_selector - : options.preferred_extension.value_or(std::move(default_codec_selector)); + const store::PathMapping mapping = + options.default_mapping.value_or(format.default_mapping()); + std::string codec_selector = + options.preferred_extension.value_or(std::move(default_codec_selector)); auto codec = resolve_codec(codec_selector); - if (!codec.has_value()) { + if (!codec) { return std::unexpected(store::OpenError(codec.error())); } - auto storage_result = store::make_storage( + return store::make_storage( base_path, format, mapping, + std::move(payload_class), std::move(codec_selector), - std::move(codec.value()), - false); - if (!storage_result.has_value()) { - return std::unexpected(storage_result.error()); - } - auto storage = std::move(storage_result.value()); - if (create_index) { - const auto index_result = storage.save_or_create_index(); - if (!index_result.has_value()) { - return std::unexpected(store::OpenError(index_result.error())); - } - } - return storage; + std::move(*codec)); } template std::expected, store::OpenError> open_folder_indexed( const std::filesystem::path &base_path, + std::string payload_class, std::string default_codec_selector, CodecResolver resolve_codec, - OpenOptions options = {}) { + OpenOptions options = {}) +{ auto result = open_folder( base_path, - true, + std::move(payload_class), std::move(default_codec_selector), std::move(resolve_codec), std::move(options)); - if (!result.has_value()) { + if (!result) { return std::unexpected(result.error()); } - return store::IndexedStorage(std::move(result.value())); + return store::IndexedStorage(std::move(*result)); } } // namespace octree::storage diff --git a/src/terrainlib/pch.h b/src/terrainlib/pch.h index dfbd471c..a936589c 100644 --- a/src/terrainlib/pch.h +++ b/src/terrainlib/pch.h @@ -35,7 +35,6 @@ #include #include #include -#include // Internal headers #include "log.h" diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h index 22664de1..024e3b2b 100644 --- a/src/terrainlib/store/Index.h +++ b/src/terrainlib/store/Index.h @@ -6,8 +6,6 @@ #include #include -#include - #include "log.h" #include "store/InvalidKey.h" #include "store/NodeStatus.h" @@ -182,9 +180,6 @@ class Index { return _index.cend(); } - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; - private: NodeStatus *get_raw_unchecked(const Key &key) { const auto iterator = _index.find(key); diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h index 7e0c3b8c..d307f14e 100644 --- a/src/terrainlib/store/IndexFormat.h +++ b/src/terrainlib/store/IndexFormat.h @@ -30,6 +30,7 @@ template struct IndexMetadata { Index index; std::string layout_id; + std::string payload_class; std::string codec_selector; }; diff --git a/src/terrainlib/store/NodeStatus.h b/src/terrainlib/store/NodeStatus.h index 775cf246..5a012f3f 100644 --- a/src/terrainlib/store/NodeStatus.h +++ b/src/terrainlib/store/NodeStatus.h @@ -6,8 +6,6 @@ #include #include -#include - #include "log.h" namespace store { @@ -45,10 +43,6 @@ class NodeStatus { private: Value _value; - -public: - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; }; } // namespace store diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index c4bb1abc..9245c0dc 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -38,7 +38,7 @@ class RawStorage { if (!result.has_value()) { return std::unexpected(LoadError(result.error())); } - return result.value(); + return std::move(*result); } std::expected> save(const Key &key, const NodeData &data) const { diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index e6a4f211..e043d9a5 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -29,21 +28,21 @@ class Storage { IndexFormat format; std::filesystem::path index_path; std::string layout_id; + std::string payload_class; std::string codec_selector; }; explicit Storage(RawStorage raw) : _raw(std::move(raw)) {} - Storage(RawStorage raw, Persistence persistence) - : _raw(std::move(raw)), _persistence(std::move(persistence)) {} - Storage( RawStorage raw, Index index, - Persistence persistence) + Persistence persistence, + const bool dirty = false) : _raw(std::move(raw)), _index(std::move(index)), - _persistence(std::move(persistence)) {} + _persistence(std::move(persistence)), + _dirty(dirty) {} virtual ~Storage() { finalize_displaced_state(); @@ -265,6 +264,7 @@ class Storage { const IndexMetadata metadata{ _index.value(), _persistence->layout_id, + _persistence->payload_class, _persistence->codec_selector, }; const auto result = _persistence->format.write( @@ -278,11 +278,7 @@ class Storage { std::expected save_or_create_index() { if (!_index.has_value()) { - const auto scan_result = scan_index(); - if (!scan_result.has_value()) { - return std::unexpected(scan_result.error()); - } - _index = std::move(scan_result.value()); + _index.emplace(); _dirty = true; } return save_index(); @@ -310,85 +306,6 @@ class Storage { } private: - std::expected, IndexFormatError> scan_index() const { - std::error_code error; - std::filesystem::create_directories(base_path(), error); - if (error) { - return std::unexpected(IndexFormatError{ - IndexFormatErrorCategory::Open, - base_path(), - error.message(), - }); - } - - const NodePath probe("__codec_probe__/node"); - const std::string probe_text = probe.path().generic_string(); - std::vector suffixes; - for (const auto &codec_path : _raw.codec().paths(probe)) { - const std::string text = codec_path.generic_string(); - if (text.starts_with(probe_text)) { - suffixes.push_back(text.substr(probe_text.size())); - } - } - - std::unordered_set candidates; - for (std::filesystem::recursive_directory_iterator iterator(base_path(), error), end; - !error && iterator != end; - iterator.increment(error)) { - if (!iterator->is_regular_file(error) && !iterator->is_symlink(error)) { - if (error) { - break; - } - continue; - } - const std::string path_text = iterator->path().generic_string(); - for (const std::string &suffix : suffixes) { - if (path_text.size() < suffix.size() - || !path_text.ends_with(suffix)) { - continue; - } - const NodePath node_path( - std::filesystem::path(path_text.substr(0, path_text.size() - suffix.size()))); - const auto key = _raw.layout().key_from_node_path(node_path); - if (key.has_value() && Traits::is_valid(key.value())) { - candidates.insert(key.value()); - } - } - } - if (error) { - return std::unexpected(IndexFormatError{ - IndexFormatErrorCategory::Open, - base_path(), - error.message(), - }); - } - - Index result; - for (const Key &key : candidates) { - const auto present = _raw.has(key); - if (!present.has_value()) { - const FilesystemError &filesystem_error = - std::get(present.error()); - return std::unexpected(IndexFormatError{ - IndexFormatErrorCategory::Open, - filesystem_error.path, - filesystem_error.error.message(), - }); - } - if (present.value()) { - const auto added = result.add(key); - if (!added.has_value()) { - return std::unexpected(IndexFormatError{ - IndexFormatErrorCategory::Malformed, - base_path(), - "codec path resolved to an invalid hierarchy key", - }); - } - } - } - return result; - } - void finalize_displaced_state() noexcept { if (!_dirty || !_index.has_value()) { return; diff --git a/src/terrainlib/store/codec/Path.h b/src/terrainlib/store/codec/Path.h new file mode 100644 index 00000000..4715291b --- /dev/null +++ b/src/terrainlib/store/codec/Path.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include "store/NodePath.h" + +namespace store::codec { + +inline std::filesystem::path append_extension( + const NodePath &node_path, + const std::string_view extension) +{ + std::filesystem::path result = node_path.path(); + result += extension; + return result; +} + +} // namespace store::codec diff --git a/src/terrainlib/store/codec/ZppBits.h b/src/terrainlib/store/codec/ZppBits.h deleted file mode 100644 index 1a37153f..00000000 --- a/src/terrainlib/store/codec/ZppBits.h +++ /dev/null @@ -1,57 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "io/serialize.h" -#include "store/Codec.h" - -namespace store::codec { - -inline std::filesystem::path append_extension( - const NodePath &node_path, - const std::string_view extension) { - std::filesystem::path result = node_path.path(); - result += extension; - return result; -} - -template -class ZppBits final : public Codec { -public: - std::vector paths(const NodePath &node_path) const override { - return {append_extension(node_path, ".bin")}; - } - - std::expected read(const NodePath &node_path) const override { - const auto result = io::read_from_path(paths(node_path).front()); - if (!result.has_value()) { - return std::unexpected(CodecError{ - CodecOperation::Read, - result.error() == io::Error(io::Error::OpenFile) - ? CodecErrorCategory::FileNotFound - : CodecErrorCategory::Io, - fmt::format("{}", result.error()), - }); - } - return result.value(); - } - - std::expected write( - const NodePath &node_path, - const NodeData &data) const override { - const auto result = io::write_to_path(data, paths(node_path).front()); - if (!result.has_value()) { - return std::unexpected(CodecError{ - CodecOperation::Write, - CodecErrorCategory::Io, - fmt::format("{}", result.error()), - }); - } - return {}; - } -}; - -} // namespace store::codec diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h index 70bf3647..5bf36352 100644 --- a/src/terrainlib/store/open.h +++ b/src/terrainlib/store/open.h @@ -15,6 +15,7 @@ template std::expected, OpenError> open_index( const std::filesystem::path &index_path, const IndexFormat format, + const std::string_view expected_payload_class, CodecResolver &&resolve_codec) { using Key = typename Traits::Key; const auto metadata_result = format.read(index_path); @@ -22,6 +23,13 @@ std::expected, OpenError> return std::unexpected(OpenError(metadata_result.error())); } IndexMetadata metadata = std::move(metadata_result.value()); + if (metadata.payload_class != expected_payload_class) { + return std::unexpected(OpenError(CodecError{ + CodecOperation::Resolve, + CodecErrorCategory::UnsupportedCodec, + "unexpected payload class: " + metadata.payload_class, + })); + } for (const auto &[key, status] : metadata.index) { static_cast(status); if (!Traits::is_valid(key)) { @@ -49,6 +57,7 @@ std::expected, OpenError> format, index_path, std::move(metadata.layout_id), + std::move(metadata.payload_class), std::move(metadata.codec_selector), }); } @@ -58,9 +67,9 @@ std::expected, OpenError> make_s const std::filesystem::path &base_path, const IndexFormat format, const PathMapping mapping, + std::string payload_class, std::string codec_selector, - std::unique_ptr> codec, - const bool indexed) { + std::unique_ptr> codec) { using Key = typename Traits::Key; std::error_code error; std::filesystem::create_directories(base_path, error); @@ -76,18 +85,17 @@ std::expected, OpenError> make_s format, base_path / format.index_filename, std::string(mapping.id), + std::move(payload_class), std::move(codec_selector), }; RawStorage raw( Layout(base_path, mapping), std::move(codec)); - if (indexed) { - return Storage( - std::move(raw), - Index{}, - std::move(persistence)); - } - return Storage(std::move(raw), std::move(persistence)); + return Storage( + std::move(raw), + Index{}, + std::move(persistence), + true); } } // namespace store diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index bde98acb..22d6a704 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -2,15 +2,16 @@ #include #include #include +#include #include #include "build.h" +#include "codec/Dag.h" #include "dag_node.h" #include "mesh/SimpleMesh.h" #include "octree/storage/open.h" #include "storage.h" -#include "store/codec/ZppBits.h" #include "thread_safe_storage.h" namespace { @@ -69,11 +70,11 @@ mesh::Simple sample_mesh() { } // namespace -TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { +TEST_CASE("runtime DAG envelope codec is reentrant") { const dag::ClusterBatch batch = sample_batch(); TemporaryDirectory output; - store::codec::ZppBits codec; + dag::codec::Dag codec; std::vector> operations; for (int index = 0; index < 8; ++index) { operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { @@ -91,14 +92,32 @@ TEST_CASE("runtime DAG ZPP Bits codec is reentrant") { } } +TEST_CASE("versioned DAG payload validation is free-standing") { + dag::format::NodeMetadata invalid_metadata{ + .group_assignment = {0}, + .groups = {}, + }; + CHECK_FALSE(dag::format::validate(invalid_metadata).has_value()); + + dag::format::Clustering oversized_clustering{ + .vertex_count = std::numeric_limits::max(), + .encoded_positions = {}, + .clusters = {}, + .textures = {}, + }; + CHECK_FALSE(dag::format::validate(oversized_clustering).has_value()); +} + TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") { - const auto batch_codec = dag::codec::from_extension(".bin"); + const auto batch_codec = dag::codec::from_extension(".dag"); REQUIRE(batch_codec.has_value()); CHECK(batch_codec.value()->paths(store::NodePath("node")) - == std::vector{"node.bin"}); + == std::vector{"node.dag", "node.dagmeta"}); - const auto metadata_codec = dag::codec::metadata_from_extension(".bin"); + const auto metadata_codec = dag::codec::metadata_from_extension(".dag"); REQUIRE(metadata_codec.has_value()); + CHECK(metadata_codec.value()->paths(store::NodePath("node")) + == std::vector{"node.dagmeta"}); const auto write_result = metadata_codec.value()->write( store::NodePath("node"), dag::NodeMetadata{}); @@ -122,21 +141,49 @@ TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[s auto released = std::move(synchronized).release(); REQUIRE(released.save_index().has_value()); - CHECK(std::filesystem::is_regular_file(output.path() / "terrain.index")); + CHECK(std::filesystem::is_regular_file(output.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(output.path() / "octree.storemeta")); auto reopened_result = dag::storage::open_folder_indexed(output.path()); REQUIRE(reopened_result.has_value()); CHECK(reopened_result->has(octree::Id::root()).value()); CHECK(reopened_result->load(octree::Id::root()).has_value()); + const auto full_paths = reopened_result->paths(octree::Id::root()); + REQUIRE(full_paths.has_value()); + REQUIRE(full_paths->size() == 2); auto metadata_result = dag::storage::open_metadata_indexed(output.path()); REQUIRE(metadata_result.has_value()); + REQUIRE(std::filesystem::remove(full_paths->front())); const auto metadata = metadata_result->load(octree::Id::root()); REQUIRE(metadata.has_value()); CHECK(metadata->group_assignment == batch.metadata.group_assignment); CHECK(metadata->groups.front().error == batch.metadata.groups.front().error); } +TEST_CASE("DAG storage hard-links both files for the same format", "[store][copy]") { + TemporaryDirectory source_directory; + TemporaryDirectory target_directory; + auto source_result = dag::storage::open_folder_indexed(source_directory.path()); + auto target_result = dag::storage::open_folder_indexed(target_directory.path()); + REQUIRE(source_result.has_value()); + REQUIRE(target_result.has_value()); + auto source = std::move(*source_result); + auto target = std::move(*target_result); + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, sample_batch()).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + + const auto source_paths = source.paths(root); + const auto target_paths = target.paths(root); + REQUIRE(source_paths.has_value()); + REQUIRE(target_paths.has_value()); + REQUIRE(source_paths->size() == 2); + REQUIRE(target_paths->size() == 2); + CHECK(std::filesystem::equivalent((*source_paths)[0], (*target_paths)[0])); + CHECK(std::filesystem::equivalent((*source_paths)[1], (*target_paths)[1])); +} + TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") { TemporaryDirectory input_directory; TemporaryDirectory output_directory; diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp index 229b2e3b..d864949a 100644 --- a/unittests/sf_builder/storage_finalization.cpp +++ b/unittests/sf_builder/storage_finalization.cpp @@ -49,7 +49,8 @@ TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuil REQUIRE(storage.save(octree::Id::root(), sample_mesh()).has_value()); CHECK(terrainbuilder::finalize_storage(storage).has_value()); - CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storemeta")); } TEST_CASE( @@ -68,9 +69,9 @@ TEST_CASE( REQUIRE_FALSE(finalized.has_value()); REQUIRE(std::holds_alternative(finalized.error())); CHECK(std::get(finalized.error()).key == root); - CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); - auto reopened = octree::open_index(directory.path() / "terrain.index"); + auto reopened = octree::open_index(directory.path() / "octree.storeindex"); REQUIRE(reopened.has_value()); CHECK(reopened->index().is(store::NodeStatus::Inner, root).value()); } diff --git a/unittests/sf_merger/integration.cpp b/unittests/sf_merger/integration.cpp index 608d4d91..faa25e0d 100644 --- a/unittests/sf_merger/integration.cpp +++ b/unittests/sf_merger/integration.cpp @@ -58,7 +58,7 @@ int build_sf( " --min-texture-level 12 --max-texture-level 17" " --mesh-srs EPSG:4978 --verbosity warn" " batch --target-level 15 --output " + shell_quote(output) + - " --format .terrain --threads 1"; + " --format .sfmesh --threads 1"; return std::system(command.c_str()); } @@ -90,8 +90,8 @@ TEST_CASE("SF builders preserve geometry at regular and point-touching mask bord " --verbosity warn"; REQUIRE(std::system(merge_command.c_str()) == 0); - const std::filesystem::path working_relative = "15/26291/18610/27235.terrain"; - const std::filesystem::path regression_relative = "15/26290/18610/27235.terrain"; + const std::filesystem::path working_relative = "15/26291/18610/27235.sfmesh"; + const std::filesystem::path regression_relative = "15/26290/18610/27235.sfmesh"; const std::filesystem::path working_path = merged_output / working_relative; const std::filesystem::path regression_path = merged_output / regression_relative; diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp index dc8fc567..05de6686 100644 --- a/unittests/sf_merger/storage_boundaries.cpp +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -95,7 +95,7 @@ TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][valida REQUIRE_FALSE(merged.has_value()); REQUIRE(std::holds_alternative(merged.error())); CHECK(std::get(merged.error()).key == root); - CHECK_FALSE(std::filesystem::exists(output_directory.path() / "terrain.index")); + CHECK_FALSE(std::filesystem::exists(output_directory.path() / "octree.storeindex")); } TEST_CASE( @@ -115,7 +115,7 @@ TEST_CASE( REQUIRE_FALSE(merged.has_value()); REQUIRE(std::holds_alternative(merged.error())); CHECK(std::get(merged.error()).key == root); - CHECK(std::filesystem::is_regular_file(output_directory.path() / "terrain.index")); + CHECK(std::filesystem::is_regular_file(output_directory.path() / "octree.storeindex")); } TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfmerger][sf][copy]") { diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index 3cfbd8eb..ab461b7a 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -170,7 +170,7 @@ TEST_CASE("Envelope round trips the latest payload version") CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::HandledByCompressionLib); CHECK(envelope.checksum.empty()); CHECK(envelope.compression_algorithm - == io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum); + == io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum); CHECK(envelope.uncompressed_size == encode_value(expected).size()); const auto result = io::envelope::deserialize(*bytes); diff --git a/unittests/terrainlib/mesh_clip.cpp b/unittests/terrainlib/mesh_clip.cpp index f3b2d369..a5646c0e 100644 --- a/unittests/terrainlib/mesh_clip.cpp +++ b/unittests/terrainlib/mesh_clip.cpp @@ -354,37 +354,3 @@ TEST_CASE("clip_on_bounds produces manifold mesh") { CHECK(mesh::is_manifold(clipped)); CHECK(clipped.positions.size() == 6); } - -#ifdef NDEBUG -TEST_CASE("mesh::clip_on_bounds benchmark") { - BENCHMARK_ADVANCED("clip based on octree")(Catch::Benchmark::Chronometer meter) { - const std::filesystem::path mesh_path = ALP_TEST_DATA_DIR "/meshes/6857.terrain"; - auto mesh_result = mesh::io::load_from_path(mesh_path); - REQUIRE(mesh_result.has_value()); - SimpleMesh &mesh = mesh_result.value(); - mesh.uvs.clear(); - mesh.texture = std::nullopt; - - const octree::Space space = octree::Space::earth(); - const auto mesh_bounds = calculate_bounds(mesh); - const octree::Id id = space.find_smallest_node_encompassing_bounds(mesh_bounds).value(); - const auto child_ids = id.children().value(); - std::vector child_bounds; - child_bounds.reserve(child_ids.size()); - for (const auto &child_id : child_ids) { - const auto bounds = space.get_node_bounds(child_id); - child_bounds.push_back(bounds); - } - - meter.measure([mesh, child_bounds] { - std::vector child_meshes; - child_meshes.reserve(child_bounds.size()); - for (const auto &bounds : child_bounds) { - const auto clipped_mesh = mesh::clip_on_bounds(mesh, bounds); - child_meshes.push_back(clipped_mesh); - } - return child_meshes; - }); - }; -} -#endif diff --git a/unittests/terrainlib/mesh_io.cpp b/unittests/terrainlib/mesh_io.cpp index fd886bac..3100bbd3 100644 --- a/unittests/terrainlib/mesh_io.cpp +++ b/unittests/terrainlib/mesh_io.cpp @@ -109,7 +109,7 @@ TEST_CASE("io roundtrip") { } TEST_CASE("io roundtrip high precision") { - for (const auto& format : {"terrain"}) { + for (const auto& format : {"sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; @@ -152,7 +152,7 @@ TEST_CASE("io roundtrip high precision") { } TEST_CASE("io roundtrip no texture") { - for (const auto &format : {"gltf", "glb", "terrain"}) { + for (const auto &format : {"gltf", "glb", "sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; @@ -194,7 +194,7 @@ TEST_CASE("io roundtrip no texture") { } TEST_CASE("io roundtrip no texture and uvs") { - for (const auto &format : {"gltf", "glb", "terrain"}) { + for (const auto &format : {"gltf", "glb", "sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp index 5b4c2fca..47dfdb46 100644 --- a/unittests/terrainlib/store_codec.cpp +++ b/unittests/terrainlib/store_codec.cpp @@ -9,9 +9,8 @@ #include "io/bytes.h" #include "mesh/codec/Gltf.h" -#include "mesh/codec/Terrain.h" +#include "mesh/codec/SfMesh.h" #include "store/Codec.h" -#include "store/codec/ZppBits.h" namespace { @@ -130,52 +129,11 @@ TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][cod CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); } -TEST_CASE("ZPP Bits runtime codec creates directories and converts errors", "[store][codec]") { - TemporaryDirectory directory("zpp"); - const store::NodePath path(directory.path() / "nested/deeper/node"); - store::codec::ZppBits codec; - - REQUIRE(codec.write(path, 12345).has_value()); - CHECK(codec.paths(path) == std::vector{directory.path() / "nested/deeper/node.bin"}); - REQUIRE(std::filesystem::is_regular_file(codec.paths(path).front())); - const auto value = codec.read(path); - REQUIRE(value.has_value()); - CHECK(value.value() == 12345); - - REQUIRE(io::write_bytes_to_path( - std::vector{0xff}, - directory.path() / "invalid.bin") - .has_value()); - const auto invalid = codec.read(store::NodePath(directory.path() / "invalid")); - REQUIRE_FALSE(invalid.has_value()); - CHECK(invalid.error().operation == store::CodecOperation::Read); - CHECK(invalid.error().category == store::CodecErrorCategory::Io); -} - -TEST_CASE("ZPP Bits runtime codec is reentrant", "[store][codec]") { - TemporaryDirectory directory("zpp-reentrant"); - store::codec::ZppBits codec; - std::vector> operations; - for (int index = 0; index < 16; ++index) { - operations.push_back(std::async(std::launch::async, [&codec, &directory, index] { - const store::NodePath path(directory.path() / std::to_string(index) / "node"); - if (!codec.write(path, index).has_value()) { - return false; - } - const auto value = codec.read(path); - return value.has_value() && value.value() == index; - })); - } - for (auto &operation : operations) { - REQUIRE(operation.get()); - } -} - TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") { TemporaryDirectory directory("mesh-reentrant"); - SECTION("terrain") { - check_mesh_codec_reentrancy(mesh::codec::Terrain{}, directory.path() / "terrain"); + SECTION("SF mesh") { + check_mesh_codec_reentrancy(mesh::codec::SfMesh{}, directory.path() / "sfmesh"); } SECTION("binary glTF") { check_mesh_codec_reentrancy( diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 79a1d3ad..9b5c8fad 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -2,16 +2,16 @@ #include #include #include +#include #include #include #include #include -#include "io/serialize.h" #include "io/bytes.h" #include "mesh/SimpleMesh.h" -#include "octree/disk/IndexFile.h" +#include "mesh/codec/SfMeshFormat.h" #include "mesh/codec/from_extension.h" #include "mesh/storage.h" #include "octree/store_layout/Mappings.h" @@ -63,8 +63,9 @@ mesh::storage::IndexedStorage make_storage( store::Index{}, mesh::storage::Storage::Persistence{ octree::storage::index_format(), - path / "terrain.index", + path / "octree.storeindex", std::string(mapping.id), + std::string(mesh::storage::payload_class), extension, }); } @@ -89,17 +90,36 @@ TEST_CASE("octree layouts round-trip boundary IDs") { } } +TEST_CASE("versioned octree and SF payload validation is free-standing") { + const octree::storage::StoreIndex invalid_index{{{ + .level = std::numeric_limits::max(), + .index = 0, + .status = static_cast(store::NodeStatus::Leaf), + }}}; + CHECK_FALSE(octree::storage::decode_index(invalid_index).has_value()); + + const mesh::sf::Payload oversized_mesh{ + .vertex_count = std::numeric_limits::max(), + .face_count = 0, + .triangles = {}, + .positions = {0}, + .uvs = {}, + .texture = {}, + }; + CHECK_FALSE(mesh::sf::validate(oversized_mesh).has_value()); +} + TEST_CASE("storage hard-links matching payload formats") { TemporaryDirectory source_directory("hard-link-source"); TemporaryDirectory target_directory("hard-link-target"); auto source = make_storage( source_directory.path(), octree::store_layout::flat(), - ".terrain"); + ".sfmesh"); auto target = make_storage( target_directory.path(), octree::store_layout::flat(), - ".terrain"); + ".sfmesh"); const octree::Id root = octree::Id::root(); REQUIRE(source.save(root, triangle_mesh(0.0)).has_value()); @@ -115,7 +135,7 @@ TEST_CASE("storage re-encodes differing payload formats") { auto source = make_storage( source_directory.path(), octree::store_layout::flat(), - ".terrain"); + ".sfmesh"); auto target = make_storage( target_directory.path(), octree::store_layout::flat(), @@ -139,7 +159,7 @@ TEST_CASE("storage supports enabled overwrites") { auto storage = make_storage( directory.path(), octree::store_layout::flat(), - ".terrain"); + ".sfmesh"); storage.settings().allow_overwrite = true; const octree::Id root = octree::Id::root(); @@ -152,19 +172,21 @@ TEST_CASE("storage supports enabled overwrites") { REQUIRE(storage.save_index().has_value()); } -TEST_CASE("folder opening scans payloads and creates an index") { +TEST_CASE("folder opening persists and reopens metadata-driven storage") { TemporaryDirectory directory("folder-open"); octree::OpenOptions options; options.default_mapping = octree::store_layout::flat(); - options.preferred_extension = ".terrain"; + options.preferred_extension = ".sfmesh"; { - auto storage_result = octree::open_folder(directory.path(), false, std::move(options)); + auto storage_result = octree::open_folder(directory.path(), std::move(options)); REQUIRE(storage_result.has_value()); auto storage = std::move(storage_result.value()); - CHECK_FALSE(storage.is_indexed()); + CHECK(storage.is_indexed()); REQUIRE(storage.save(octree::Id::root(), triangle_mesh(2.0)).has_value()); - CHECK_FALSE(std::filesystem::exists(directory.path() / "terrain.index")); + REQUIRE(storage.save_index().has_value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storemeta")); } { @@ -173,16 +195,22 @@ TEST_CASE("folder opening scans payloads and creates an index") { const auto storage = std::move(storage_result.value()); CHECK(storage.is_indexed()); CHECK(storage.has(octree::Id::root()).value()); - CHECK(std::filesystem::is_regular_file(directory.path() / "terrain.index")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); } - const auto reopened = octree::open_index(directory.path() / "terrain.index"); + const auto reopened = octree::open_index(directory.path() / "octree.storeindex"); REQUIRE(reopened.has_value()); CHECK(reopened->has(octree::Id::root()).value()); + + const auto metadata = octree::storage::read_store_metadata(directory.path()); + REQUIRE(metadata.has_value()); + CHECK(metadata->layout_id == "flat"); + CHECK(metadata->payload_class == mesh::storage::payload_class); + CHECK(metadata->codec_selector == ".sfmesh"); } -TEST_CASE("mesh codec resolver dispatches every supported legacy selector", "[store][open]") { - for (const std::string extension : {".terrain", ".glb", ".gltf"}) { +TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][open]") { + for (const std::string extension : {".sfmesh", ".glb", ".gltf"}) { const auto codec = mesh::codec::from_extension(extension); REQUIRE(codec.has_value()); const auto paths = codec.value()->paths(store::NodePath("node")); @@ -196,23 +224,44 @@ TEST_CASE("mesh codec resolver dispatches every supported legacy selector", "[st CHECK(unknown.error().operation == store::CodecOperation::Resolve); } +TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store][open]") { + TemporaryDirectory directory("payload-class"); + const std::filesystem::path index_path = directory.path() / "octree.storeindex"; + const store::IndexMetadata index_file{ + .index = {}, + .layout_id = "flat", + .payload_class = "dag.ClusterBatch", + .codec_selector = ".sfmesh", + }; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); + + const auto result = octree::open_index(index_path); + REQUIRE_FALSE(result.has_value()); + REQUIRE(std::holds_alternative(result.error())); + CHECK(std::get(result.error()).category + == store::CodecErrorCategory::UnsupportedCodec); +} + TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") { TemporaryDirectory directory("open-errors"); - const std::filesystem::path index_path = directory.path() / "terrain.index"; + const std::filesystem::path index_path = directory.path() / "octree.storeindex"; - octree::disk::v1::IndexFile index_file; - index_file.layout_strategy_id = "unknown-layout"; - index_file.preferred_extension = ".terrain"; - REQUIRE(io::write_to_path(index_file, index_path).has_value()); + store::IndexMetadata index_file{ + .index = {}, + .layout_id = "unknown-layout", + .payload_class = std::string(mesh::storage::payload_class), + .codec_selector = ".sfmesh", + }; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); const auto unknown_layout = octree::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_layout.has_value()); REQUIRE(std::holds_alternative(unknown_layout.error())); CHECK(std::get(unknown_layout.error()).id == "unknown-layout"); - index_file.layout_strategy_id = "flat"; - index_file.preferred_extension = ".unknown"; - REQUIRE(io::write_to_path(index_file, index_path).has_value()); + index_file.layout_id = "flat"; + index_file.codec_selector = ".unknown"; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); const auto unknown_codec = octree::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_codec.has_value()); REQUIRE(std::holds_alternative(unknown_codec.error())); @@ -226,4 +275,9 @@ TEST_CASE("octree opening retains index failures without directory fallback", "[ REQUIRE(std::holds_alternative(malformed.error())); CHECK(std::get(malformed.error()).category == store::IndexFormatErrorCategory::Malformed); + + const auto independent_metadata = + octree::storage::read_store_metadata(directory.path()); + REQUIRE(independent_metadata.has_value()); + CHECK(independent_metadata->codec_selector == ".unknown"); } diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp index 736df506..4d0db771 100644 --- a/unittests/terrainlib/store_layout.cpp +++ b/unittests/terrainlib/store_layout.cpp @@ -3,19 +3,19 @@ #include #include "mesh/codec/Gltf.h" -#include "mesh/codec/Terrain.h" +#include "mesh/codec/SfMesh.h" #include "octree/store_layout/Mappings.h" #include "store/Layout.h" TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layout]") { const std::filesystem::path dataset = "dataset"; - mesh::codec::Terrain terrain; + mesh::codec::SfMesh sfmesh; SECTION("flat") { const store::Layout layout(dataset / "sf-flat", octree::store_layout::flat()); const store::NodePath node_path = layout.node_path(octree::Id::root()); CHECK(node_path.path() == dataset / "sf-flat/0-0"); - CHECK(terrain.paths(node_path) == std::vector{dataset / "sf-flat/0-0.terrain"}); + CHECK(sfmesh.paths(node_path) == std::vector{dataset / "sf-flat/0-0.sfmesh"}); CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); } @@ -25,10 +25,10 @@ TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layo octree::store_layout::level_and_coordinate_directories()); const octree::Id child = octree::Id::root().child(2).value(); const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); - CHECK(terrain.paths(layout.node_path(child)) - == std::vector{dataset / "sf-coordinates/1/0/1/0.terrain"}); - CHECK(terrain.paths(layout.node_path(deep)) - == std::vector{dataset / "sf-coordinates/2/3/1/3.terrain"}); + CHECK(sfmesh.paths(layout.node_path(child)) + == std::vector{dataset / "sf-coordinates/1/0/1/0.sfmesh"}); + CHECK(sfmesh.paths(layout.node_path(deep)) + == std::vector{dataset / "sf-coordinates/2/3/1/3.sfmesh"}); CHECK(layout.key_from_node_path(layout.node_path(child)) == child); CHECK(layout.key_from_node_path(layout.node_path(deep)) == deep); } @@ -46,12 +46,12 @@ TEST_CASE("octree mapping lookup is explicit", "[store][layout]") { TEST_CASE("extensionless octree mappings validate complete paths", "[store][layout]") { const auto flat = octree::store_layout::flat(); - CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2.terrain")).has_value()); + CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2.sfmesh")).has_value()); CHECK_FALSE(flat.node_path_to_key(store::NodePath("nested/1-2")).has_value()); CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2-extra")).has_value()); const auto coordinates = octree::store_layout::level_and_coordinate_directories(); - CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0.terrain")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0.sfmesh")).has_value()); CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0")).has_value()); CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0/extra")).has_value()); CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("22/0/0/0")).has_value()); @@ -59,11 +59,11 @@ TEST_CASE("extensionless octree mappings validate complete paths", "[store][layo TEST_CASE("runtime mesh codecs own their filename endings", "[store][layout][codec]") { const store::NodePath path("12/34/56/78"); - mesh::codec::Terrain terrain; + mesh::codec::SfMesh sfmesh; mesh::codec::Gltf binary(mesh::codec::GltfContainer::Binary); mesh::codec::Gltf json(mesh::codec::GltfContainer::Json); - CHECK(terrain.paths(path) == std::vector{"12/34/56/78.terrain"}); + CHECK(sfmesh.paths(path) == std::vector{"12/34/56/78.sfmesh"}); CHECK(binary.paths(path) == std::vector{"12/34/56/78.glb"}); CHECK(json.paths(path) == std::vector{"12/34/56/78.gltf"}); } diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index 3065a6d2..6a06b075 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -9,12 +10,11 @@ #include #include -#include "io/serialize.h" +#include "io/bytes.h" #include "octree/StoreTraits.h" #include "octree/store_layout/Mappings.h" #include "raster_store/StoreTraits.h" #include "store/IndexedStorage.h" -#include "store/codec/ZppBits.h" #include "string_utils.h" namespace { @@ -80,11 +80,66 @@ store::PathMapping test_mapping() { return {"test_tiles", tile_to_path, path_to_tile}; } +std::expected read_int(const std::filesystem::path &path) { + auto bytes = io::read_bytes_from_path(path); + if (!bytes || bytes->size() != sizeof(int)) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Read, + store::CodecErrorCategory::Io, + "test integer read failed", + }); + } + int value; + std::memcpy(&value, bytes->data(), sizeof(value)); + return value; +} + +std::expected write_int( + const int value, + const std::filesystem::path &path) +{ + const auto bytes = std::span( + reinterpret_cast(&value), + sizeof(value)); + if (!io::write_bytes_to_path(bytes, path)) { + return std::unexpected(store::CodecError{ + store::CodecOperation::Write, + store::CodecErrorCategory::Io, + "test integer write failed", + }); + } + return {}; +} + +class IntCodec final : public store::Codec { +public: + std::vector paths( + const store::NodePath &node_path) const override + { + std::filesystem::path path = node_path.path(); + path += ".data"; + return {path}; + } + + std::expected read( + const store::NodePath &node_path) const override + { + return read_int(paths(node_path).front()); + } + + std::expected write( + const store::NodePath &node_path, + const int &value) const override + { + return write_int(value, paths(node_path).front()); + } +}; + template store::Storage make_storage(const std::filesystem::path &path) { return store::Storage(store::RawStorage( store::Layout(path, test_mapping()), - std::make_unique>())); + std::make_unique())); } class MultiFileCodec final : public store::Codec { @@ -99,8 +154,8 @@ class MultiFileCodec final : public store::Codec { std::expected read( const store::NodePath &node_path) const override { - const auto value = io::read_from_path(paths(node_path).front()); - if (!value.has_value()) { + const auto value = read_int(paths(node_path).front()); + if (!value) { return std::unexpected(store::CodecError{ store::CodecOperation::Read, store::CodecErrorCategory::Io, @@ -114,8 +169,8 @@ class MultiFileCodec final : public store::Codec { const store::NodePath &node_path, const int &value) const override { const auto node_paths = paths(node_path); - if (!io::write_to_path(value, node_paths[0]).has_value() - || !io::write_to_path(value + 1, node_paths[1]).has_value()) { + if (!write_int(value, node_paths[0]) + || !write_int(value + 1, node_paths[1])) { return std::unexpected(store::CodecError{ store::CodecOperation::Write, store::CodecErrorCategory::Io, @@ -147,8 +202,8 @@ class ConfigurableCodec final : public store::Codec { "injected decode failure", }); } - const auto value = io::read_from_path(paths(node_path).front()); - if (!value.has_value()) { + const auto value = read_int(paths(node_path).front()); + if (!value) { return std::unexpected(store::CodecError{ store::CodecOperation::Read, store::CodecErrorCategory::Io, @@ -168,7 +223,7 @@ class ConfigurableCodec final : public store::Codec { "injected encode failure", }); } - if (!io::write_to_path(value, paths(node_path).front()).has_value()) { + if (!write_int(value, paths(node_path).front())) { return std::unexpected(store::CodecError{ store::CodecOperation::Write, store::CodecErrorCategory::Io, @@ -214,7 +269,7 @@ store::PathMapping fake_default_mapping() { store::IndexFormat counting_index_format() { return { - "terrain.index", + "octree.storeindex", unused_index_read, count_index_write, fake_mapping_from_id, @@ -225,7 +280,7 @@ store::IndexFormat counting_index_format() { store::IndexedStorage make_indexed_storage( const std::filesystem::path &path, std::unique_ptr> codec = - std::make_unique>()) { + std::make_unique()) { using Storage = store::Storage; return store::IndexedStorage( store::RawStorage( @@ -234,9 +289,10 @@ store::IndexedStorage make_indexed_storage( store::Index{}, Storage::Persistence{ counting_index_format(), - path / "terrain.index", + path / "octree.storeindex", "flat", - ".bin", + "test.Int", + ".data", }); } @@ -301,9 +357,9 @@ TEST_CASE("raw storage requires and removes every codec path", "[store][storage] const auto paths = storage.paths(root).value(); REQUIRE(paths.size() == 2); - REQUIRE(io::write_to_path(1, paths[0]).has_value()); + REQUIRE(write_int(1, paths[0]).has_value()); CHECK_FALSE(storage.has(root).value()); - REQUIRE(io::write_to_path(2, paths[1]).has_value()); + REQUIRE(write_int(2, paths[1]).has_value()); CHECK(storage.has(root).value()); REQUIRE(storage.remove(root).value()); CHECK_FALSE(std::filesystem::exists(paths[0])); @@ -431,7 +487,7 @@ TEST_CASE( const auto target_paths = target.paths(root).value(); REQUIRE(std::filesystem::remove(target_paths[1])); REQUIRE(std::filesystem::create_directory(target_paths[1])); - REQUIRE(io::write_to_path(9, target_paths[1] / "blocker.bin").has_value()); + REQUIRE(write_int(9, target_paths[1] / "blocker.data").has_value()); const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); diff --git a/unittests/terrainlib/string_utils.cpp b/unittests/terrainlib/string_utils.cpp index c54490a4..9ef3699a 100644 --- a/unittests/terrainlib/string_utils.cpp +++ b/unittests/terrainlib/string_utils.cpp @@ -21,7 +21,7 @@ TEST_CASE("from_chars rejects non-numeric input", "[string_utils]") { } TEST_CASE("from_chars rejects trailing garbage after a valid number", "[string_utils]") { - CHECK(from_chars(std::string_view("123.terrain")) == std::nullopt); + CHECK(from_chars(std::string_view("123.sfmesh")) == std::nullopt); CHECK(from_chars(std::string_view("123abc")) == std::nullopt); } From e310cf50f814983aa55399bc1c23b3069d29d07b Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 22 Aug 2026 21:33:12 +0200 Subject: [PATCH 16/43] Fix unity build and sanitizer symbolization --- .github/workflows/ci.yml | 5 ++++- src/dag_builder/build.cpp | 12 +++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4516ef8..4e0fd500 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null echo "deb https://apt.llvm.org/noble/ llvm-toolchain-noble-23 main" | sudo tee /etc/apt/sources.list.d/llvm-23.list sudo apt-get update - sudo apt-get install -y --no-install-recommends clang-23 lld-23 libclang-rt-23-dev + sudo apt-get install -y --no-install-recommends clang-23 lld-23 llvm-23 libclang-rt-23-dev - name: Install GCC ${{ matrix.cxx }} if: matrix.cxx == 'g++-16' @@ -186,6 +186,9 @@ jobs: export PROJ_DATA="${{ github.workspace }}/build/alp_external/proj/share/proj" export PROJ_LIB="${PROJ_DATA}" export GDAL_DATA="${{ github.workspace }}/build/alp_external/gdal/share/gdal" + if [[ "${{ matrix.sanitizer }}" == "asan" ]]; then + export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-23 + fi if [[ "${{ matrix.sanitizer }}" == "tsan" ]]; then export TSAN_OPTIONS="suppressions=${{ github.workspace }}/extern/tbb/cmake/suppressions/tsan.suppressions" fi diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index dad2fdbb..e5bda9cf 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -452,7 +452,7 @@ LevelWorkplan build_level_workplan( } // Pre-filter input nodes to only those that intersect the target root bounds. -std::vector> gather_relevant_input_leaves( +std::expected>, sf::InvalidTopology> gather_relevant_input_leaves( const store::Index &index, const octree::Space &space, const radix::geometry::Aabb3d &root_bounds) @@ -471,7 +471,9 @@ std::vector> gather_relevant_input_leaves( return radix::geometry::intersect(root_bounds, space.get_node_bounds(id)); }, start); - DEBUG_ASSERT(traversal.has_value()); + if (!traversal.has_value()) { + return std::unexpected(sf::InvalidTopology{traversal.error().key}); + } return result; } @@ -596,7 +598,11 @@ std::expected build_levels( const octree::Space space = octree::Space::earth(); const octree::Id root_node = options.root_node; const auto root_bounds = shifted_space.get_node_bounds_with_children(root_node); - auto input_by_level = gather_relevant_input_leaves(input_storage.index(), space, root_bounds); + auto input_by_level_result = gather_relevant_input_leaves(input_storage.index(), space, root_bounds); + if (!input_by_level_result.has_value()) { + return std::unexpected(input_by_level_result.error()); + } + auto input_by_level = std::move(input_by_level_result.value()); // Find the maximum input level that has any nodes auto max_input_level_opt = find_max_input_level(input_by_level); From 5a6e9538c73b6a73723a4a0a9ad659249056557b Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 23 Aug 2026 07:35:12 +0200 Subject: [PATCH 17/43] Update cgltf for writer leak fix --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4a7150f8..aeea6394 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -32,7 +32,7 @@ find_package(CGAL REQUIRED) include(../cmake/SetupOpenCV.cmake) alp_setup_opencv(4.11.0) -alp_add_git_repository(cgltf URL https://github.com/AlpineMapsOrgDependencies/cgltf.git COMMITISH e035b820ccde6ab20451b612d8033c9b76ea0ce4 DO_NOT_ADD_SUBPROJECT) +alp_add_git_repository(cgltf URL https://github.com/AlpineMapsOrgDependencies/cgltf.git COMMITISH 1549b07fe059e1917bbdda1f61ec36b5f4d6718a DO_NOT_ADD_SUBPROJECT) add_library(cgltf INTERFACE) target_include_directories(cgltf SYSTEM INTERFACE ${cgltf_SOURCE_DIR}) From 1a1a8d7b398e91df6f163af8adab75167be499d6 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:11:49 +0200 Subject: [PATCH 18/43] Apply renderer code style to raster store changes --- cmake/SetupGDAL.cmake | 2 +- docs/raster-store/storage-format.md | 4 +- src/dag_builder/codec/Dag.h | 69 ++-- src/dag_builder/codec/DagFormat.h | 224 ++++------ src/dag_builder/codec/MetadataView.h | 13 +- src/dag_builder/codec/from_extension.h | 14 +- src/terrainlib/io/compression.cpp | 228 +++++----- src/terrainlib/io/compression.h | 13 +- src/terrainlib/io/conversion.h | 259 ++++++------ src/terrainlib/io/envelope.h | 94 ++--- src/terrainlib/io/envelope_file.h | 34 +- src/terrainlib/mesh/codec/Gltf.h | 41 +- src/terrainlib/mesh/codec/SfMesh.h | 43 +- src/terrainlib/mesh/codec/SfMeshFormat.h | 37 +- src/terrainlib/mesh/codec/from_extension.h | 6 +- src/terrainlib/mesh/storage.h | 34 +- src/terrainlib/octree/StoreTraits.h | 17 +- src/terrainlib/octree/storage/Format.h | 69 ++-- src/terrainlib/octree/storage/IndexFile.h | 47 +-- src/terrainlib/octree/storage/open_runtime.h | 50 +-- src/terrainlib/octree/store_layout/Flat.h | 9 +- .../LevelAndCoordinateDirectories.h | 18 +- src/terrainlib/octree/store_layout/Mappings.h | 14 +- src/terrainlib/raster_store/StoreTraits.h | 15 +- src/terrainlib/sf/Error.h | 15 +- src/terrainlib/sf/InvalidTopology.h | 2 +- src/terrainlib/sf/finalize_storage.h | 3 +- src/terrainlib/sf/validate_index.h | 8 +- src/terrainlib/store/Codec.h | 16 +- src/terrainlib/store/CodecError.h | 9 +- src/terrainlib/store/Index.h | 123 +++--- src/terrainlib/store/IndexFormat.h | 13 +- src/terrainlib/store/IndexedStorage.h | 26 +- src/terrainlib/store/InvalidKey.h | 4 +- src/terrainlib/store/Layout.h | 31 +- src/terrainlib/store/NodePath.h | 13 +- src/terrainlib/store/NodeStatus.h | 48 +-- src/terrainlib/store/NodeStatusOrMissing.h | 113 +++-- src/terrainlib/store/OpenError.h | 11 +- src/terrainlib/store/PathMapping.h | 8 +- src/terrainlib/store/RawStorage.h | 106 +++-- src/terrainlib/store/Storage.h | 291 ++++++------- src/terrainlib/store/StorageError.h | 29 +- src/terrainlib/store/Traits.h | 2 +- src/terrainlib/store/cache/Dummy.h | 18 +- src/terrainlib/store/cache/Interface.h | 10 +- src/terrainlib/store/cache/Lru.h | 69 ++-- src/terrainlib/store/codec/Path.h | 4 +- src/terrainlib/store/describe_error.h | 41 +- src/terrainlib/store/open.h | 45 +- src/terrainlib/store/traverse.h | 29 +- src/terrainlib/tile_path.h | 4 +- src/tile_builder/alpine_raster.cpp | 2 +- src/tile_builder/image_writer.cpp | 2 +- src/tile_builder/image_writer.h | 13 +- src/tile_downloader/write_file.h | 12 +- .../dag_builder/storage_compatibility.cpp | 92 ++--- unittests/sf_builder/storage_finalization.cpp | 34 +- unittests/sf_merger/storage_boundaries.cpp | 126 +++--- unittests/terrainlib/envelope.cpp | 388 +++++++----------- unittests/terrainlib/io_conversion.cpp | 20 +- unittests/terrainlib/sf_validate_index.cpp | 6 +- unittests/terrainlib/store_codec.cpp | 99 ++--- unittests/terrainlib/store_compatibility.cpp | 132 +++--- unittests/terrainlib/store_index.cpp | 53 ++- unittests/terrainlib/store_layout.cpp | 39 +- unittests/terrainlib/store_storage.cpp | 308 ++++++-------- unittests/terrainlib/store_traverse.cpp | 50 +-- unittests/tile_downloader/downloader.cpp | 45 +- unittests/tile_downloader/http_client.cpp | 32 +- unittests/tile_downloader/logger.cpp | 2 +- unittests/tile_downloader/write_file.cpp | 36 +- unittests/tilebuilder/dataset_reading.cpp | 26 +- unittests/tilebuilder/image_writer.cpp | 11 +- 74 files changed, 1702 insertions(+), 2271 deletions(-) diff --git a/cmake/SetupGDAL.cmake b/cmake/SetupGDAL.cmake index fdc85178..e08315b9 100644 --- a/cmake/SetupGDAL.cmake +++ b/cmake/SetupGDAL.cmake @@ -39,7 +39,7 @@ function(alp_setup_gdal) -DBUILD_TESTING=OFF -DGEOS_BUILD_DEVELOPER=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON - ) + ) alp_setup_cmake_project(gdal URL https://github.com/OSGeo/gdal.git diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 18497b2d..5bfaf28e 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -18,7 +18,7 @@ This document specifies the logical format and required invariants for the follo - the source-attribution table (for correct copyright attribution and data selection) ## Source-attribution table -The source attribution table is used in raster-fundamentalis, tile-base, and in abbreviated format in the delivered tiles. +The source attribution table is used in raster-fundamentalis, tile-base, and in abbreviated format in the delivered tiles. - stores a vector of structs (called Table), each struct describing one source - the struct (called Entity) contains: - the spatial resolution in pixel-width at the equator (EPSG:3857) @@ -62,7 +62,7 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - should generate tiles of requested resolution and pixel type (vertex|area) on the fly - requests by url, e.g.: layer/vertex|area/resolution/z/x/y.ending - live in src/tile-server/* -- the delivery tile format is not yet defined +- the delivery tile format is not yet defined ## serialization / deserialization envelope and versioning - `zpp::bits` serialises C++ objects to byte streams and deserialises them again. diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index 3209f4bb..5894ada6 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -10,26 +10,21 @@ namespace dag::codec { -inline store::CodecError file_error( - const store::CodecOperation operation, - const io::envelope::FileError &error) +inline store::CodecError file_error(const store::CodecOperation operation, const io::envelope::FileError& error) { - const bool missing = std::holds_alternative(error) - && std::get(error) == io::Error(io::Error::OpenFile); + const bool missing = std::holds_alternative(error) && std::get(error) == io::Error(io::Error::OpenFile); return { operation, - missing ? store::CodecErrorCategory::FileNotFound - : operation == store::CodecOperation::Read - ? store::CodecErrorCategory::InvalidData - : store::CodecErrorCategory::Io, + missing ? store::CodecErrorCategory::FileNotFound + : operation == store::CodecOperation::Read ? store::CodecErrorCategory::InvalidData + : store::CodecErrorCategory::Io, io::envelope::describe_error(error), }; } class Dag final : public store::Codec { public: - std::vector paths( - const store::NodePath &node_path) const override + std::vector paths(const store::NodePath& node_path) const override { return { store::codec::append_extension(node_path, ".dag"), @@ -37,27 +32,20 @@ class Dag final : public store::Codec { }; } - std::expected read( - const store::NodePath &node_path) const override + std::expected read(const store::NodePath& node_path) const override { const auto node_paths = paths(node_path); - auto clustering_payload = - io::envelope::read_from_path(node_paths[0]); + auto clustering_payload = io::envelope::read_from_path(node_paths[0]); if (!clustering_payload) { - return std::unexpected(file_error( - store::CodecOperation::Read, - clustering_payload.error())); + return std::unexpected(file_error(store::CodecOperation::Read, clustering_payload.error())); } - auto metadata_payload = - io::envelope::read_from_path(node_paths[1]); + auto metadata_payload = io::envelope::read_from_path(node_paths[1]); if (!metadata_payload) { - return std::unexpected(file_error( - store::CodecOperation::Read, - metadata_payload.error())); + return std::unexpected(file_error(store::CodecOperation::Read, metadata_payload.error())); } auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); if (!clustering) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::InvalidData, clustering.error(), @@ -65,62 +53,51 @@ class Dag final : public store::Codec { } auto metadata = dag::format::decode_metadata(std::move(*metadata_payload)); if (!metadata) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::InvalidData, metadata.error(), }); } - return dag::ClusterBatch{ + return dag::ClusterBatch { .metadata = std::move(*metadata), .clustering = std::move(*clustering), }; } - std::expected write( - const store::NodePath &node_path, - const dag::ClusterBatch &batch) const override + std::expected write(const store::NodePath& node_path, const dag::ClusterBatch& batch) const override { auto clustering = dag::format::encode_clustering(batch.clustering); if (!clustering) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, clustering.error(), }); } - const dag::format::NodeMetadata metadata = - dag::format::encode_metadata(batch.metadata); + const dag::format::NodeMetadata metadata = dag::format::encode_metadata(batch.metadata); if (auto valid = dag::format::validate(*clustering); !valid) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, valid.error(), }); } if (auto valid = dag::format::validate(metadata); !valid) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, valid.error(), }); } const auto node_paths = paths(node_path); - auto data_written = io::envelope::write_to_path( - *clustering, - node_paths[0]); + auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); if (!data_written) { - return std::unexpected(file_error( - store::CodecOperation::Write, - data_written.error())); + return std::unexpected(file_error(store::CodecOperation::Write, data_written.error())); } - auto metadata_written = io::envelope::write_to_path( - metadata, - node_paths[1]); + auto metadata_written = io::envelope::write_to_path(metadata, node_paths[1]); if (!metadata_written) { - return std::unexpected(file_error( - store::CodecOperation::Write, - metadata_written.error())); + return std::unexpected(file_error(store::CodecOperation::Write, metadata_written.error())); } return {}; } diff --git a/src/dag_builder/codec/DagFormat.h b/src/dag_builder/codec/DagFormat.h index 41bb19e8..a5c73f11 100644 --- a/src/dag_builder/codec/DagFormat.h +++ b/src/dag_builder/codec/DagFormat.h @@ -21,94 +21,85 @@ namespace dag::format { namespace v1 { -struct OctreeId { - std::uint8_t level; - std::uint64_t index; -}; + struct OctreeId { + std::uint8_t level; + std::uint64_t index; + }; -struct Id { - OctreeId source_batch; - std::uint32_t cluster_index; -}; + struct Id { + OctreeId source_batch; + std::uint32_t cluster_index; + }; -struct Vec3d { - double x; - double y; - double z; -}; + struct Vec3d { + double x; + double y; + double z; + }; -struct Bounds { - Vec3d min; - Vec3d max; -}; + struct Bounds { + Vec3d min; + Vec3d max; + }; -struct Group { - std::vector children; - double error; - Bounds bounds; -}; + struct Group { + std::vector children; + double error; + Bounds bounds; + }; -struct NodeMetadata { - std::vector group_assignment; - std::vector groups; -}; + struct NodeMetadata { + std::vector group_assignment; + std::vector groups; + }; -struct Texture { - std::vector jpeg; -}; + struct Texture { + std::vector jpeg; + }; -struct Cluster { - std::uint64_t triangle_count; - std::uint64_t vertex_count; - std::uint64_t uv_count; - std::vector encoded_triangles; - std::vector encoded_vertex_indices; - std::vector encoded_uvs; - std::uint32_t id; - std::uint32_t texture_id; - double absolute_error; -}; + struct Cluster { + std::uint64_t triangle_count; + std::uint64_t vertex_count; + std::uint64_t uv_count; + std::vector encoded_triangles; + std::vector encoded_vertex_indices; + std::vector encoded_uvs; + std::uint32_t id; + std::uint32_t texture_id; + double absolute_error; + }; -struct Clustering { - std::uint64_t vertex_count; - std::vector encoded_positions; - std::vector clusters; - std::vector textures; -}; + struct Clustering { + std::uint64_t vertex_count; + std::vector encoded_positions; + std::vector clusters; + std::vector textures; + }; } // namespace v1 -using MetadataSchema = io::envelope::PayloadSchema< - "dag.NodeMetadata", - io::envelope::Version<1, v1::NodeMetadata>>; -using ClusteringSchema = io::envelope::PayloadSchema< - "dag.Clustering", - io::envelope::Version<1, v1::Clustering>>; +using MetadataSchema = io::envelope::PayloadSchema<"dag.NodeMetadata", io::envelope::Version<1, v1::NodeMetadata>>; +using ClusteringSchema = io::envelope::PayloadSchema<"dag.Clustering", io::envelope::Version<1, v1::Clustering>>; using NodeMetadata = MetadataSchema::latest_type; using Clustering = ClusteringSchema::latest_type; -inline v1::OctreeId encode_id(const octree::Id &id) -{ - return {.level = id.level(), .index = id.index_on_level()}; -} +inline v1::OctreeId encode_id(const octree::Id& id) { return { .level = id.level(), .index = id.index_on_level() }; } -inline std::expected decode_id(const v1::OctreeId &id) +inline std::expected decode_id(const v1::OctreeId& id) { - auto result = octree::Id::try_make( - static_cast(id.level), - static_cast(id.index)); + auto result = octree::Id::try_make(static_cast(id.level), static_cast(id.index)); if (!result) { return std::unexpected("DAG payload contains an invalid octree ID"); } return *result; } -inline NodeMetadata encode_metadata(const dag::NodeMetadata &metadata) +inline NodeMetadata encode_metadata(const dag::NodeMetadata& metadata) { NodeMetadata result; result.group_assignment = metadata.group_assignment; result.groups.reserve(metadata.groups.size()); - for (const dag::Group &group : metadata.groups) { + for (const dag::Group& group : metadata.groups) { v1::Group encoded_group{ .children = {}, .error = group.error, @@ -118,7 +109,7 @@ inline NodeMetadata encode_metadata(const dag::NodeMetadata &metadata) }, }; encoded_group.children.reserve(group.children.size()); - for (const dag::Id &child : group.children) { + for (const dag::Id& child : group.children) { encoded_group.children.push_back({ .source_batch = encode_id(child.source_batch), .cluster_index = child.cluster_index, @@ -129,18 +120,18 @@ inline NodeMetadata encode_metadata(const dag::NodeMetadata &metadata) return result; } -inline std::expected validate(const NodeMetadata &metadata) +inline std::expected validate(const NodeMetadata& metadata) { for (const std::uint32_t group : metadata.group_assignment) { if (group >= metadata.groups.size()) { return std::unexpected("DAG metadata contains an invalid group assignment"); } } - for (const v1::Group &group : metadata.groups) { + for (const v1::Group& group : metadata.groups) { if (!std::isfinite(group.error)) { return std::unexpected("DAG metadata contains a non-finite error"); } - for (const v1::Id &child : group.children) { + for (const v1::Id& child : group.children) { if (!decode_id(child.source_batch)) { return std::unexpected("DAG metadata contains an invalid child ID"); } @@ -157,7 +148,7 @@ inline std::expected decode_metadata(NodeMetadat dag::NodeMetadata result; result.group_assignment = std::move(metadata.group_assignment); result.groups.reserve(metadata.groups.size()); - for (v1::Group &group : metadata.groups) { + for (v1::Group& group : metadata.groups) { dag::Group decoded_group{ .children = {}, .error = group.error, @@ -167,31 +158,29 @@ inline std::expected decode_metadata(NodeMetadat }, }; decoded_group.children.reserve(group.children.size()); - for (const v1::Id &child : group.children) { + for (const v1::Id& child : group.children) { auto source_batch = decode_id(child.source_batch); if (!source_batch) { return std::unexpected(source_batch.error()); } - decoded_group.children.push_back({*source_batch, child.cluster_index}); + decoded_group.children.push_back({ *source_batch, child.cluster_index }); } result.groups.push_back(std::move(decoded_group)); } return result; } -inline std::expected encode_clustering( - const ::Clustering &clustering) +inline std::expected encode_clustering(const ::Clustering& clustering) { - Clustering result{ + Clustering result { .vertex_count = clustering.positions.size(), .encoded_positions = meshopt::encode_vertex_buffer(clustering.positions), .clusters = {}, .textures = {}, }; result.clusters.reserve(clustering.clusters.size()); - for (const ::Cluster &cluster : clustering.clusters) { - if (cluster.texture_id - == (std::numeric_limits::max)()) { + for (const ::Cluster& cluster : clustering.clusters) { + if (cluster.texture_id == (std::numeric_limits::max)()) { return std::unexpected("DAG texture ID collides with the on-disk null sentinel"); } result.clusters.push_back({ @@ -208,14 +197,14 @@ inline std::expected encode_clustering( } result.textures.reserve(clustering.textures.size()); try { - for (const cv::Mat &texture : clustering.textures) { + for (const cv::Mat& texture : clustering.textures) { auto jpeg = mesh::io::write_texture_to_encoded_buffer(texture, ".jpeg"); if (!texture.empty() && jpeg.empty()) { return std::unexpected("could not encode DAG texture"); } - result.textures.push_back({std::move(jpeg)}); + result.textures.push_back({ std::move(jpeg) }); } - } catch (const cv::Exception &) { + } catch (const cv::Exception&) { return std::unexpected("could not encode DAG texture"); } return result; @@ -223,11 +212,10 @@ inline std::expected encode_clustering( inline bool count_fits(const std::uint64_t count, const std::size_t element_size) { - return count <= std::numeric_limits::max() / element_size - && count * element_size <= io::envelope::default_max_decompressed_size; + return count <= std::numeric_limits::max() / element_size && count * element_size <= io::envelope::default_max_decompressed_size; } -inline std::expected validate(const Clustering &clustering) +inline std::expected validate(const Clustering& clustering) { if (!count_fits(clustering.vertex_count, sizeof(glm::dvec3))) { return std::unexpected("DAG position count exceeds the allocation limit"); @@ -235,18 +223,14 @@ inline std::expected validate(const Clustering &clustering) if (clustering.vertex_count != 0 && clustering.encoded_positions.empty()) { return std::unexpected("DAG position count and encoded data disagree"); } - std::size_t decoded_size = - static_cast(clustering.vertex_count) * sizeof(glm::dvec3); - for (const v1::Cluster &cluster : clustering.clusters) { - if (!count_fits(cluster.triangle_count, sizeof(glm::uvec3)) - || !count_fits(cluster.vertex_count, sizeof(std::uint32_t)) + std::size_t decoded_size = static_cast(clustering.vertex_count) * sizeof(glm::dvec3); + for (const v1::Cluster& cluster : clustering.clusters) { + if (!count_fits(cluster.triangle_count, sizeof(glm::uvec3)) || !count_fits(cluster.vertex_count, sizeof(std::uint32_t)) || !count_fits(cluster.uv_count, sizeof(glm::dvec2))) { return std::unexpected("DAG cluster count exceeds the allocation limit"); } - const std::size_t cluster_size = - static_cast(cluster.triangle_count) * sizeof(glm::uvec3) - + static_cast(cluster.vertex_count) * sizeof(std::uint32_t) - + static_cast(cluster.uv_count) * sizeof(glm::dvec2); + const std::size_t cluster_size = static_cast(cluster.triangle_count) * sizeof(glm::uvec3) + + static_cast(cluster.vertex_count) * sizeof(std::uint32_t) + static_cast(cluster.uv_count) * sizeof(glm::dvec2); if (cluster_size > io::envelope::default_max_decompressed_size - decoded_size) { return std::unexpected("DAG decoded data exceeds the allocation limit"); } @@ -254,13 +238,11 @@ inline std::expected validate(const Clustering &clustering) if (cluster.uv_count != 0 && cluster.uv_count != cluster.vertex_count) { return std::unexpected("DAG cluster UV count does not match its vertex count"); } - if ((cluster.triangle_count != 0 && cluster.encoded_triangles.empty()) - || (cluster.vertex_count != 0 && cluster.encoded_vertex_indices.empty()) + if ((cluster.triangle_count != 0 && cluster.encoded_triangles.empty()) || (cluster.vertex_count != 0 && cluster.encoded_vertex_indices.empty()) || (cluster.uv_count != 0 && cluster.encoded_uvs.empty())) { return std::unexpected("DAG cluster count and encoded data disagree"); } - if (cluster.texture_id != (std::numeric_limits::max)() - && cluster.texture_id >= clustering.textures.size()) { + if (cluster.texture_id != (std::numeric_limits::max)() && cluster.texture_id >= clustering.textures.size()) { return std::unexpected("DAG cluster contains an invalid texture reference"); } if (!std::isfinite(cluster.absolute_error)) { @@ -272,41 +254,25 @@ inline std::expected validate(const Clustering &clustering) template inline std::expected, std::string> decode_vertex_buffer( - const std::uint64_t count, - const std::vector &encoded, - const std::string_view label) + const std::uint64_t count, const std::vector& encoded, const std::string_view label) { if (!count_fits(count, sizeof(T))) { return std::unexpected(std::string(label) + " count exceeds the allocation limit"); } std::vector result(static_cast(count)); - if (meshopt_decodeVertexBuffer( - result.data(), - result.size(), - sizeof(T), - encoded.data(), - encoded.size()) - != 0) { + if (meshopt_decodeVertexBuffer(result.data(), result.size(), sizeof(T), encoded.data(), encoded.size()) != 0) { return std::unexpected("could not decode " + std::string(label)); } return result; } -inline std::expected, std::string> decode_triangles( - const std::uint64_t count, - const std::vector &encoded) +inline std::expected, std::string> decode_triangles(const std::uint64_t count, const std::vector& encoded) { if (!count_fits(count, sizeof(glm::uvec3))) { return std::unexpected("DAG triangle count exceeds the allocation limit"); } std::vector result(static_cast(count)); - if (meshopt_decodeIndexBuffer( - result.data(), - result.size() * 3, - sizeof(std::uint32_t), - encoded.data(), - encoded.size()) - != 0) { + if (meshopt_decodeIndexBuffer(result.data(), result.size() * 3, sizeof(std::uint32_t), encoded.data(), encoded.size()) != 0) { return std::unexpected("could not decode DAG triangles"); } return result; @@ -317,10 +283,7 @@ inline std::expected<::Clustering, std::string> decode_clustering(Clustering clu if (auto valid = validate(clustering); !valid) { return std::unexpected(valid.error()); } - auto positions = decode_vertex_buffer( - clustering.vertex_count, - clustering.encoded_positions, - "DAG positions"); + auto positions = decode_vertex_buffer(clustering.vertex_count, clustering.encoded_positions, "DAG positions"); if (!positions) { return std::unexpected(positions.error()); } @@ -328,16 +291,10 @@ inline std::expected<::Clustering, std::string> decode_clustering(Clustering clu ::Clustering result; result.positions = std::move(*positions); result.clusters.reserve(clustering.clusters.size()); - for (const v1::Cluster &encoded : clustering.clusters) { + for (const v1::Cluster& encoded : clustering.clusters) { auto triangles = decode_triangles(encoded.triangle_count, encoded.encoded_triangles); - auto vertex_indices = decode_vertex_buffer( - encoded.vertex_count, - encoded.encoded_vertex_indices, - "DAG vertex indices"); - auto uvs = decode_vertex_buffer( - encoded.uv_count, - encoded.encoded_uvs, - "DAG UVs"); + auto vertex_indices = decode_vertex_buffer(encoded.vertex_count, encoded.encoded_vertex_indices, "DAG vertex indices"); + auto uvs = decode_vertex_buffer(encoded.uv_count, encoded.encoded_uvs, "DAG UVs"); if (!triangles) { return std::unexpected(triangles.error()); } @@ -353,9 +310,7 @@ inline std::expected<::Clustering, std::string> decode_clustering(Clustering clu } } for (const glm::uvec3 triangle : *triangles) { - if (glm::any(glm::greaterThanEqual( - triangle, - glm::uvec3(static_cast(vertex_indices->size()))))) { + if (glm::any(glm::greaterThanEqual(triangle, glm::uvec3(static_cast(vertex_indices->size()))))) { return std::unexpected("DAG cluster contains an invalid local triangle index"); } } @@ -364,9 +319,8 @@ inline std::expected<::Clustering, std::string> decode_clustering(Clustering clu .vertex_indices = std::move(*vertex_indices), .local_triangles = std::move(*triangles), .uvs = std::move(*uvs), - .texture_id = encoded.texture_id == (std::numeric_limits::max)() - ? std::nullopt - : std::optional{encoded.texture_id}, + .texture_id + = encoded.texture_id == (std::numeric_limits::max)() ? std::nullopt : std::optional { encoded.texture_id }, .absolute_error = encoded.absolute_error, }); } @@ -374,14 +328,14 @@ inline std::expected<::Clustering, std::string> decode_clustering(Clustering clu std::vector textures; textures.reserve(clustering.textures.size()); try { - for (const v1::Texture &texture : clustering.textures) { + for (const v1::Texture& texture : clustering.textures) { cv::Mat image = mesh::io::read_texture_from_encoded_bytes(texture.jpeg); if (image.empty()) { return std::unexpected("could not decode DAG texture"); } textures.push_back(std::move(image)); } - } catch (const cv::Exception &) { + } catch (const cv::Exception&) { return std::unexpected("could not decode DAG texture"); } result.textures = TextureSet(std::move(textures)); diff --git a/src/dag_builder/codec/MetadataView.h b/src/dag_builder/codec/MetadataView.h index c27dfa62..f668e507 100644 --- a/src/dag_builder/codec/MetadataView.h +++ b/src/dag_builder/codec/MetadataView.h @@ -13,23 +13,20 @@ namespace dag::codec { class MetadataView final : public store::Codec { public: - std::vector paths( - const store::NodePath &node_path) const override + std::vector paths(const store::NodePath& node_path) const override { - return {store::codec::append_extension(node_path, ".dagmeta")}; + return { store::codec::append_extension(node_path, ".dagmeta") }; } - std::expected read( - const store::NodePath &node_path) const override + std::expected read(const store::NodePath& node_path) const override { - auto payload = io::envelope::read_from_path( - paths(node_path).front()); + auto payload = io::envelope::read_from_path(paths(node_path).front()); if (!payload) { return std::unexpected(file_error(store::CodecOperation::Read, payload.error())); } auto metadata = dag::format::decode_metadata(std::move(*payload)); if (!metadata) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::InvalidData, metadata.error(), diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h index 53d558d1..928d93a5 100644 --- a/src/dag_builder/codec/from_extension.h +++ b/src/dag_builder/codec/from_extension.h @@ -6,30 +6,30 @@ #include -#include "codec/MetadataView.h" #include "codec/Dag.h" +#include "codec/MetadataView.h" #include "dag_node.h" namespace dag::codec { -inline std::expected>, store::CodecError> -from_extension(const std::string_view extension) { +inline std::expected>, store::CodecError> from_extension(const std::string_view extension) +{ if (extension == ".dag") { return std::make_unique(); } - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Resolve, store::CodecErrorCategory::UnsupportedCodec, "unsupported DAG codec selector: " + std::string(extension), }); } -inline std::expected>, store::CodecError> -metadata_from_extension(const std::string_view extension) { +inline std::expected>, store::CodecError> metadata_from_extension(const std::string_view extension) +{ if (extension == ".dag") { return std::make_unique(); } - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Resolve, store::CodecErrorCategory::UnsupportedCodec, "unsupported DAG metadata codec selector: " + std::string(extension), diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index b3ee1e5f..35fbb870 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -13,134 +13,114 @@ namespace io::envelope { namespace { -using CompressionContext = std::unique_ptr; - -std::string crc32c_checksum(const Bytes &data) -{ - static constexpr auto table = [] { - constexpr std::uint32_t polynomial = 0x82f63b78u; - - std::array values{}; - for (std::size_t index = 0; index < values.size(); ++index) { - auto crc = static_cast(index); - for (int bit = 0; bit < 8; ++bit) { - crc = (crc >> 1u) ^ ((crc & 1u) != 0u ? polynomial : 0u); + using CompressionContext = std::unique_ptr; + + std::string crc32c_checksum(const Bytes& data) + { + static constexpr auto table = [] { + constexpr std::uint32_t polynomial = 0x82f63b78u; + + std::array values {}; + for (std::size_t index = 0; index < values.size(); ++index) { + auto crc = static_cast(index); + for (int bit = 0; bit < 8; ++bit) { + crc = (crc >> 1u) ^ ((crc & 1u) != 0u ? polynomial : 0u); + } + values[index] = crc; } - values[index] = crc; - } - return values; - }(); + return values; + }(); - std::uint32_t crc = 0xffffffffu; - for (const auto value : data) { - const auto index = (crc ^ std::to_integer(value)) & 0xffu; - crc = table[index] ^ (crc >> 8u); - } - crc = ~crc; + std::uint32_t crc = 0xffffffffu; + for (const auto value : data) { + const auto index = (crc ^ std::to_integer(value)) & 0xffu; + crc = table[index] ^ (crc >> 8u); + } + crc = ~crc; - constexpr char hex_digits[] = "0123456789abcdef"; - std::string checksum(8, '0'); - for (std::size_t index = 0; index < checksum.size(); ++index) { - const auto shift = static_cast((checksum.size() - index - 1) * 4); - checksum[index] = hex_digits[(crc >> shift) & 0x0fu]; - } - return checksum; -} + constexpr char hex_digits[] = "0123456789abcdef"; + std::string checksum(8, '0'); + for (std::size_t index = 0; index < checksum.size(); ++index) { + const auto shift = static_cast((checksum.size() - index - 1) * 4); + checksum[index] = hex_digits[(crc >> shift) & 0x0fu]; + } + return checksum; + } + + std::expected validate_algorithms(const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) + { + switch (checksum_algorithm) { + case ChecksumAlgorithm::None: + case ChecksumAlgorithm::HandledByCompressionLib: + case ChecksumAlgorithm::Crc32c: + break; + default: + return std::unexpected(Error { ErrorCode::UnsupportedChecksumAlgorithm }); + } -std::expected validate_algorithms( - const CompressionAlgorithm compression_algorithm, - const ChecksumAlgorithm checksum_algorithm) -{ - switch (checksum_algorithm) { - case ChecksumAlgorithm::None: - case ChecksumAlgorithm::HandledByCompressionLib: - case ChecksumAlgorithm::Crc32c: - break; - default: - return std::unexpected(Error{ErrorCode::UnsupportedChecksumAlgorithm}); - } + switch (compression_algorithm) { + case CompressionAlgorithm::None: + case CompressionAlgorithm::ZstdBestCompressionWithChecksum: + case CompressionAlgorithm::ZstdDefaultCompressionWithChecksum: + break; + default: + return std::unexpected(Error { ErrorCode::UnsupportedCompressionAlgorithm }); + } - switch (compression_algorithm) { - case CompressionAlgorithm::None: - case CompressionAlgorithm::ZstdBestCompressionWithChecksum: - case CompressionAlgorithm::ZstdDefaultCompressionWithChecksum: - break; - default: - return std::unexpected(Error{ErrorCode::UnsupportedCompressionAlgorithm}); - } + const bool no_compression = compression_algorithm == CompressionAlgorithm::None + && (checksum_algorithm == ChecksumAlgorithm::None || checksum_algorithm == ChecksumAlgorithm::Crc32c); + const bool zstd_with_checksum = (compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + || compression_algorithm == CompressionAlgorithm::ZstdDefaultCompressionWithChecksum) + && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib || checksum_algorithm == ChecksumAlgorithm::Crc32c); + if (!no_compression && !zstd_with_checksum) { + return std::unexpected(Error { ErrorCode::InvalidAlgorithmCombination }); + } - const bool no_compression = compression_algorithm == CompressionAlgorithm::None - && (checksum_algorithm == ChecksumAlgorithm::None - || checksum_algorithm == ChecksumAlgorithm::Crc32c); - const bool zstd_with_checksum = - (compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum - || compression_algorithm == CompressionAlgorithm::ZstdDefaultCompressionWithChecksum) - && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib - || checksum_algorithm == ChecksumAlgorithm::Crc32c); - if (!no_compression && !zstd_with_checksum) { - return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + return {}; } - return {}; -} - } // namespace std::expected compress_with_checksum( - const Bytes &uncompressed_data, - const CompressionAlgorithm compression_algorithm, - const ChecksumAlgorithm checksum_algorithm) + const Bytes& uncompressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) { if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { return std::unexpected(validation.error()); } if (uncompressed_data.size() > default_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + return std::unexpected(Error { ErrorCode::SizeLimitExceeded }); } - const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c - ? crc32c_checksum(uncompressed_data) - : std::string{}; + const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c ? crc32c_checksum(uncompressed_data) : std::string {}; if (compression_algorithm == CompressionAlgorithm::None) { - return CompressedData{uncompressed_data, checksum}; + return CompressedData { uncompressed_data, checksum }; } - CompressionContext context{ZSTD_createCCtx(), &ZSTD_freeCCtx}; + CompressionContext context { ZSTD_createCCtx(), &ZSTD_freeCCtx }; if (!context) { - return std::unexpected(Error{ErrorCode::CompressionFailed}); + return std::unexpected(Error { ErrorCode::CompressionFailed }); } - const int compression_level = - compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum - ? ZSTD_maxCLevel() - : 0; - if (ZSTD_isError(ZSTD_CCtx_setParameter( - context.get(), - ZSTD_c_compressionLevel, - compression_level)) + const int compression_level = compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum ? ZSTD_maxCLevel() : 0; + if (ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_compressionLevel, compression_level)) || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { - return std::unexpected(Error{ErrorCode::CompressionFailed}); + return std::unexpected(Error { ErrorCode::CompressionFailed }); } const std::size_t capacity = ZSTD_compressBound(uncompressed_data.size()); Bytes compressed_data(capacity); - const std::size_t compressed_size = ZSTD_compress2( - context.get(), - compressed_data.data(), - compressed_data.size(), - uncompressed_data.data(), - uncompressed_data.size()); + const std::size_t compressed_size + = ZSTD_compress2(context.get(), compressed_data.data(), compressed_data.size(), uncompressed_data.data(), uncompressed_data.size()); if (ZSTD_isError(compressed_size)) { - return std::unexpected(Error{ErrorCode::CompressionFailed}); + return std::unexpected(Error { ErrorCode::CompressionFailed }); } compressed_data.resize(compressed_size); - return CompressedData{std::move(compressed_data), checksum}; + return CompressedData { std::move(compressed_data), checksum }; } -std::expected checked_decompress( - const Bytes &compressed_data, +std::expected checked_decompress(const Bytes& compressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm, const std::string_view checksum, @@ -150,79 +130,65 @@ std::expected checked_decompress( return std::unexpected(validation.error()); } if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { - return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + return std::unexpected(Error { ErrorCode::InvalidAlgorithmCombination }); } - if (checksum_algorithm == ChecksumAlgorithm::Crc32c - && (checksum.size() != 8 - || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { - return (character >= '0' && character <= '9') - || (character >= 'a' && character <= 'f'); - }))) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + if (checksum_algorithm == ChecksumAlgorithm::Crc32c && (checksum.size() != 8 || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { + return (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f'); + }))) { + return std::unexpected(Error { ErrorCode::ChecksumMismatch }); } - const std::size_t effective_max_decompressed_size = - std::min(max_decompressed_size, default_max_decompressed_size); + const std::size_t effective_max_decompressed_size = std::min(max_decompressed_size, default_max_decompressed_size); Bytes uncompressed_data; if (compression_algorithm == CompressionAlgorithm::None) { if (compressed_data.size() > effective_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + return std::unexpected(Error { ErrorCode::SizeLimitExceeded }); } uncompressed_data = compressed_data; } else { - ZSTD_frameHeader frame_header{}; - const std::size_t frame_header_result = ZSTD_getFrameHeader( - &frame_header, compressed_data.data(), compressed_data.size()); + ZSTD_frameHeader frame_header {}; + const std::size_t frame_header_result = ZSTD_getFrameHeader(&frame_header, compressed_data.data(), compressed_data.size()); if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + return std::unexpected(Error { ErrorCode::DecompressionFailed }); } if (frame_header.checksumFlag == 0) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + return std::unexpected(Error { ErrorCode::ChecksumMismatch }); } - const unsigned long long frame_content_size = - ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); + const unsigned long long frame_content_size = ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); if (frame_content_size == ZSTD_CONTENTSIZE_ERROR - || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN - && frame_content_size > std::numeric_limits::max())) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN && frame_content_size > std::numeric_limits::max())) { + return std::unexpected(Error { ErrorCode::DecompressionFailed }); } const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + return std::unexpected(Error { ErrorCode::SizeLimitExceeded }); } - const std::size_t allocation_size = content_size_is_known - ? static_cast(frame_content_size) - : effective_max_decompressed_size; + const std::size_t allocation_size = content_size_is_known ? static_cast(frame_content_size) : effective_max_decompressed_size; uncompressed_data.resize(allocation_size); - const std::size_t decompressed_size = ZSTD_decompress( - uncompressed_data.data(), - uncompressed_data.size(), - compressed_data.data(), - compressed_data.size()); + const std::size_t decompressed_size + = ZSTD_decompress(uncompressed_data.data(), uncompressed_data.size(), compressed_data.data(), compressed_data.size()); if (ZSTD_isError(decompressed_size)) { if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + return std::unexpected(Error { ErrorCode::ChecksumMismatch }); } - if (!content_size_is_known - && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + if (!content_size_is_known && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { + return std::unexpected(Error { ErrorCode::SizeLimitExceeded }); } - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + return std::unexpected(Error { ErrorCode::DecompressionFailed }); } if (content_size_is_known && decompressed_size != uncompressed_data.size()) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + return std::unexpected(Error { ErrorCode::DecompressionFailed }); } uncompressed_data.resize(decompressed_size); } - if (checksum_algorithm == ChecksumAlgorithm::Crc32c - && crc32c_checksum(uncompressed_data) != checksum) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + if (checksum_algorithm == ChecksumAlgorithm::Crc32c && crc32c_checksum(uncompressed_data) != checksum) { + return std::unexpected(Error { ErrorCode::ChecksumMismatch }); } return uncompressed_data; } diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h index 536fe694..dc61a4f9 100644 --- a/src/terrainlib/io/compression.h +++ b/src/terrainlib/io/compression.h @@ -13,7 +13,7 @@ namespace io::envelope { using Bytes = std::vector; -inline constexpr std::size_t default_max_decompressed_size = std::size_t{1} << 30; +inline constexpr std::size_t default_max_decompressed_size = std::size_t { 1 } << 30; enum class ChecksumAlgorithm : std::uint8_t { None, @@ -44,9 +44,9 @@ enum class ErrorCode : std::uint8_t { struct Error { ErrorCode code; - std::errc serialization_error{}; + std::errc serialization_error {}; - constexpr bool operator==(const Error &) const = default; + constexpr bool operator==(const Error&) const = default; }; struct CompressedData { @@ -55,12 +55,9 @@ struct CompressedData { }; std::expected compress_with_checksum( - const Bytes &uncompressed_data, - CompressionAlgorithm compression_algorithm, - ChecksumAlgorithm checksum_algorithm); + const Bytes& uncompressed_data, CompressionAlgorithm compression_algorithm, ChecksumAlgorithm checksum_algorithm); -std::expected checked_decompress( - const Bytes &compressed_data, +std::expected checked_decompress(const Bytes& compressed_data, CompressionAlgorithm compression_algorithm, ChecksumAlgorithm checksum_algorithm, std::string_view checksum, diff --git a/src/terrainlib/io/conversion.h b/src/terrainlib/io/conversion.h index 246e87b0..7ddd6a14 100644 --- a/src/terrainlib/io/conversion.h +++ b/src/terrainlib/io/conversion.h @@ -25,171 +25,145 @@ enum class ErrorCode { struct Error { ErrorCode code; - int expected_cv_type{-1}; - int actual_cv_type{-1}; + int expected_cv_type { -1 }; + int actual_cv_type { -1 }; - constexpr bool operator==(const Error &) const = default; + constexpr bool operator==(const Error&) const = default; }; namespace detail { -template -inline constexpr int cv_depth = -1; - -template <> -inline constexpr int cv_depth = CV_8U; -template <> -inline constexpr int cv_depth = CV_8S; -template <> -inline constexpr int cv_depth = CV_16U; -template <> -inline constexpr int cv_depth = CV_16S; -template <> -inline constexpr int cv_depth = CV_32S; -template <> -inline constexpr int cv_depth = CV_32F; -template <> -inline constexpr int cv_depth = CV_64F; + template + inline constexpr int cv_depth = -1; + + template <> + inline constexpr int cv_depth = CV_8U; + template <> + inline constexpr int cv_depth = CV_8S; + template <> + inline constexpr int cv_depth = CV_16U; + template <> + inline constexpr int cv_depth = CV_16S; + template <> + inline constexpr int cv_depth = CV_32S; + template <> + inline constexpr int cv_depth = CV_32F; + template <> + inline constexpr int cv_depth = CV_64F; + + template + struct PixelTraits { + using channel_type = std::remove_cv_t; + static constexpr int channels = 1; + static constexpr bool supported = cv_depth >= 0; + + static channel_type& channel(Pixel& pixel, const int) { return pixel; } + + static const channel_type& channel(const Pixel& pixel, const int) { return pixel; } + }; + + template + struct PixelTraits> { + using channel_type = Channel; + static constexpr int channels = static_cast(Length); + static constexpr bool supported = channels >= 2 && channels <= 4 && cv_depth >= 0; + + static channel_type& channel(glm::vec& pixel, const int index) { return pixel[static_cast(index)]; } + + static const channel_type& channel(const glm::vec& pixel, const int index) + { + return pixel[static_cast(index)]; + } + }; -template -struct PixelTraits { - using channel_type = std::remove_cv_t; - static constexpr int channels = 1; - static constexpr bool supported = cv_depth >= 0; + template + using Traits = PixelTraits>; - static channel_type &channel(Pixel &pixel, const int) - { - return pixel; - } + template + inline constexpr int cv_type = Traits::supported ? CV_MAKETYPE(cv_depth::channel_type>, Traits::channels) : -1; - static const channel_type &channel(const Pixel &pixel, const int) - { - return pixel; - } -}; + template + inline constexpr bool has_packed_layout = Traits::supported + && std::is_trivially_copyable_v && sizeof(Pixel) == sizeof(typename Traits::channel_type) * Traits::channels; -template -struct PixelTraits> { - using channel_type = Channel; - static constexpr int channels = static_cast(Length); - static constexpr bool supported = channels >= 2 && channels <= 4 - && cv_depth >= 0; - - static channel_type &channel(glm::vec &pixel, const int index) + template + void copy_mat_to_raster(const cv::Mat& source, radix::Raster& destination) { - return pixel[static_cast(index)]; - } - - static const channel_type &channel( - const glm::vec &pixel, - const int index) - { - return pixel[static_cast(index)]; - } -}; - -template -using Traits = PixelTraits>; - -template -inline constexpr int cv_type = Traits::supported - ? CV_MAKETYPE(cv_depth::channel_type>, Traits::channels) - : -1; + using PixelInfo = Traits; + using Channel = typename PixelInfo::channel_type; -template -inline constexpr bool has_packed_layout = Traits::supported - && std::is_trivially_copyable_v - && sizeof(Pixel) == sizeof(typename Traits::channel_type) * Traits::channels; - -template -void copy_mat_to_raster(const cv::Mat &source, radix::Raster &destination) -{ - using PixelInfo = Traits; - using Channel = typename PixelInfo::channel_type; - - const auto row_bytes = static_cast(source.cols) * sizeof(Pixel); - if constexpr (has_packed_layout) { - for (int row = 0; row < source.rows; ++row) { - std::memcpy( - destination.data() + static_cast(row) * source.cols, - source.ptr(row), - row_bytes); - } - } else { - for (int row = 0; row < source.rows; ++row) { - const Channel *source_row = source.ptr(row); - for (int column = 0; column < source.cols; ++column) { - Pixel &pixel = destination.pixel({ - static_cast(column), - static_cast(row), - }); - for (int channel = 0; channel < PixelInfo::channels; ++channel) { - PixelInfo::channel(pixel, channel) = - source_row[column * PixelInfo::channels + channel]; + const auto row_bytes = static_cast(source.cols) * sizeof(Pixel); + if constexpr (has_packed_layout) { + for (int row = 0; row < source.rows; ++row) { + std::memcpy(destination.data() + static_cast(row) * source.cols, source.ptr(row), row_bytes); + } + } else { + for (int row = 0; row < source.rows; ++row) { + const Channel* source_row = source.ptr(row); + for (int column = 0; column < source.cols; ++column) { + Pixel& pixel = destination.pixel({ + static_cast(column), + static_cast(row), + }); + for (int channel = 0; channel < PixelInfo::channels; ++channel) { + PixelInfo::channel(pixel, channel) = source_row[column * PixelInfo::channels + channel]; + } } } } } -} -template -void copy_raster_to_mat(const radix::Raster &source, cv::Mat &destination) -{ - using PixelInfo = Traits; - using Channel = typename PixelInfo::channel_type; - - const auto row_bytes = static_cast(destination.cols) * sizeof(Pixel); - if constexpr (has_packed_layout) { - for (int row = 0; row < destination.rows; ++row) { - std::memcpy( - destination.ptr(row), - source.data() + static_cast(row) * destination.cols, - row_bytes); - } - } else { - for (int row = 0; row < destination.rows; ++row) { - Channel *destination_row = destination.ptr(row); - for (int column = 0; column < destination.cols; ++column) { - const Pixel &pixel = source.pixel({ - static_cast(column), - static_cast(row), - }); - for (int channel = 0; channel < PixelInfo::channels; ++channel) { - destination_row[column * PixelInfo::channels + channel] = - PixelInfo::channel(pixel, channel); + template + void copy_raster_to_mat(const radix::Raster& source, cv::Mat& destination) + { + using PixelInfo = Traits; + using Channel = typename PixelInfo::channel_type; + + const auto row_bytes = static_cast(destination.cols) * sizeof(Pixel); + if constexpr (has_packed_layout) { + for (int row = 0; row < destination.rows; ++row) { + std::memcpy(destination.ptr(row), source.data() + static_cast(row) * destination.cols, row_bytes); + } + } else { + for (int row = 0; row < destination.rows; ++row) { + Channel* destination_row = destination.ptr(row); + for (int column = 0; column < destination.cols; ++column) { + const Pixel& pixel = source.pixel({ + static_cast(column), + static_cast(row), + }); + for (int channel = 0; channel < PixelInfo::channels; ++channel) { + destination_row[column * PixelInfo::channels + channel] = PixelInfo::channel(pixel, channel); + } } } } } -} } // namespace detail template -std::expected, Error> to_raster(const cv::Mat &source) +std::expected, Error> to_raster(const cv::Mat& source) { if constexpr (!detail::Traits::supported) { - return std::unexpected(Error{ErrorCode::UnsupportedPixelType}); + return std::unexpected(Error { ErrorCode::UnsupportedPixelType }); } else { if (source.empty()) { - return std::unexpected(Error{ErrorCode::EmptyInput}); + return std::unexpected(Error { ErrorCode::EmptyInput }); } if (source.dims != 2) { - return std::unexpected(Error{ErrorCode::UnsupportedDimensions}); + return std::unexpected(Error { ErrorCode::UnsupportedDimensions }); } if (source.type() != detail::cv_type) { - return std::unexpected(Error{ + return std::unexpected(Error { ErrorCode::PixelTypeMismatch, detail::cv_type, source.type(), }); } - if (source.cols < 0 || source.rows < 0 - || static_cast(source.cols) - > std::numeric_limits::max() - || static_cast(source.rows) - > std::numeric_limits::max()) { - return std::unexpected(Error{ErrorCode::DimensionsOutOfRange}); + if (source.cols < 0 || source.rows < 0 || static_cast(source.cols) > std::numeric_limits::max() + || static_cast(source.rows) > std::numeric_limits::max()) { + return std::unexpected(Error { ErrorCode::DimensionsOutOfRange }); } try { @@ -199,39 +173,36 @@ std::expected, Error> to_raster(const cv::Mat &source) }); detail::copy_mat_to_raster(source, destination); return destination; - } catch (const std::bad_alloc &) { - return std::unexpected(Error{ErrorCode::AllocationFailed}); - } catch (const cv::Exception &) { - return std::unexpected(Error{ErrorCode::OpenCvFailure}); + } catch (const std::bad_alloc&) { + return std::unexpected(Error { ErrorCode::AllocationFailed }); + } catch (const cv::Exception&) { + return std::unexpected(Error { ErrorCode::OpenCvFailure }); } } } template -std::expected to_mat(const radix::Raster &source) +std::expected to_mat(const radix::Raster& source) { if constexpr (!detail::Traits::supported) { - return std::unexpected(Error{ErrorCode::UnsupportedPixelType}); + return std::unexpected(Error { ErrorCode::UnsupportedPixelType }); } else { if (source.width() == 0 || source.height() == 0) { - return std::unexpected(Error{ErrorCode::EmptyInput}); + return std::unexpected(Error { ErrorCode::EmptyInput }); } if (source.width() > static_cast(std::numeric_limits::max()) || source.height() > static_cast(std::numeric_limits::max())) { - return std::unexpected(Error{ErrorCode::DimensionsOutOfRange}); + return std::unexpected(Error { ErrorCode::DimensionsOutOfRange }); } try { - cv::Mat destination( - static_cast(source.height()), - static_cast(source.width()), - detail::cv_type); + cv::Mat destination(static_cast(source.height()), static_cast(source.width()), detail::cv_type); detail::copy_raster_to_mat(source, destination); return destination; - } catch (const std::bad_alloc &) { - return std::unexpected(Error{ErrorCode::AllocationFailed}); - } catch (const cv::Exception &) { - return std::unexpected(Error{ErrorCode::OpenCvFailure}); + } catch (const std::bad_alloc&) { + return std::unexpected(Error { ErrorCode::AllocationFailed }); + } catch (const cv::Exception&) { + return std::unexpected(Error { ErrorCode::OpenCvFailure }); } } } diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h index d643e383..bf9a6561 100644 --- a/src/terrainlib/io/envelope.h +++ b/src/terrainlib/io/envelope.h @@ -32,12 +32,9 @@ template struct FixedString { char value[Size]; - constexpr FixedString(const char (&text)[Size]) - { - std::copy_n(text, Size, value); - } + constexpr FixedString(const char (&text)[Size]) { std::copy_n(text, Size, value); } - constexpr auto operator<=>(const FixedString &) const = default; + constexpr auto operator<=>(const FixedString&) const = default; }; template @@ -48,52 +45,50 @@ struct Version { namespace detail { -template -struct FindVersion; + template + struct FindVersion; -template -struct FindVersion { - using type = std::conditional_t::type>; -}; + template + struct FindVersion { + using type = std::conditional_t::type>; + }; -template -struct FindVersion { - using type = void; -}; + template + struct FindVersion { + using type = void; + }; -template -consteval bool versions_are_strictly_increasing() -{ - constexpr std::uint32_t numbers[] = {Versions::number...}; - for (std::size_t index = 1; index < sizeof...(Versions); ++index) { - if (numbers[index - 1] >= numbers[index]) { - return false; + template + consteval bool versions_are_strictly_increasing() + { + constexpr std::uint32_t numbers[] = { Versions::number... }; + for (std::size_t index = 1; index < sizeof...(Versions); ++index) { + if (numbers[index - 1] >= numbers[index]) { + return false; + } } + return true; } - return true; -} -template -concept ConvertsFromPrevious = requires(From previous) { - { To::from_previous(std::move(previous)) } -> std::same_as; -}; + template + concept ConvertsFromPrevious = requires(From previous) { + { To::from_previous(std::move(previous)) } -> std::same_as; + }; -template -consteval bool conversions_are_valid(std::index_sequence) -{ - return (ConvertsFromPrevious< - typename std::tuple_element_t::payload_type, - typename std::tuple_element_t::payload_type> + template + consteval bool conversions_are_valid(std::index_sequence) + { + return (ConvertsFromPrevious::payload_type, + typename std::tuple_element_t::payload_type> && ...); -} + } } // namespace detail template struct PayloadSchema { static_assert(sizeof...(Versions) > 0, "a payload schema requires at least one version"); - static_assert(detail::versions_are_strictly_increasing(), - "payload versions must be strictly increasing"); + static_assert(detail::versions_are_strictly_increasing(), "payload versions must be strictly increasing"); using version_tuple = std::tuple; static constexpr std::size_t version_count = sizeof...(Versions); @@ -104,43 +99,34 @@ struct PayloadSchema { using latest_version_descriptor = version_at; using latest_type = typename latest_version_descriptor::payload_type; - static constexpr std::string_view class_name{ClassName.value, sizeof(ClassName.value) - 1}; + static constexpr std::string_view class_name { ClassName.value, sizeof(ClassName.value) - 1 }; static constexpr std::uint32_t latest_version = latest_version_descriptor::number; template - static constexpr bool supports_version = - !std::is_void_v::type>; + static constexpr bool supports_version = !std::is_void_v::type>; - static constexpr bool supports_version_number(const std::uint32_t number) - { - return ((number == Versions::number) || ...); - } + static constexpr bool supports_version_number(const std::uint32_t number) { return ((number == Versions::number) || ...); } template using payload_type = typename detail::FindVersion::type::payload_type; - static_assert(version_count == 1 - || detail::conversions_are_valid( - std::make_index_sequence{}), - "each payload version must provide from_previous for the preceding version"); + static_assert(version_count == 1 || detail::conversions_are_valid(std::make_index_sequence {}), + "each payload version must provide from_previous for the preceding version"); }; template -std::expected serialize( - const typename Schema::template payload_type &payload, +std::expected serialize(const typename Schema::template payload_type& payload, CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template -std::expected serialize( - const typename Schema::latest_type &payload, +std::expected serialize(const typename Schema::latest_type& payload, CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template std::expected deserialize( - std::span bytes, - std::size_t max_decompressed_size = default_max_decompressed_size); + std::span bytes, std::size_t max_decompressed_size = default_max_decompressed_size); } // namespace io::envelope diff --git a/src/terrainlib/io/envelope_file.h b/src/terrainlib/io/envelope_file.h index f5e3cd6a..20c1b3e2 100644 --- a/src/terrainlib/io/envelope_file.h +++ b/src/terrainlib/io/envelope_file.h @@ -17,15 +17,12 @@ namespace io::envelope { using FileError = std::variant<::io::Error, Error>; -inline std::string describe_error(const Error &error) -{ - return "envelope error " + std::to_string(static_cast(error.code)); -} +inline std::string describe_error(const Error& error) { return "envelope error " + std::to_string(static_cast(error.code)); } -inline std::string describe_error(const FileError &error) +inline std::string describe_error(const FileError& error) { return std::visit( - [](const auto &value) -> std::string { + [](const auto& value) -> std::string { using Value = std::remove_cvref_t; if constexpr (std::same_as) { return value.to_string(); @@ -38,41 +35,38 @@ inline std::string describe_error(const FileError &error) template std::expected read_from_path( - const std::filesystem::path &path, - const std::size_t max_decompressed_size = default_max_decompressed_size) + const std::filesystem::path& path, const std::size_t max_decompressed_size = default_max_decompressed_size) { auto file_bytes = ::io::read_bytes_from_path(path); if (!file_bytes) { - return std::unexpected(FileError{file_bytes.error()}); + return std::unexpected(FileError { file_bytes.error() }); } - const auto bytes = std::as_bytes(std::span{*file_bytes}); + const auto bytes = std::as_bytes(std::span { *file_bytes }); auto result = deserialize(bytes, max_decompressed_size); if (!result) { - return std::unexpected(FileError{result.error()}); + return std::unexpected(FileError { result.error() }); } return std::move(*result); } template -std::expected write_to_path( - const typename Schema::latest_type &payload, - const std::filesystem::path &path, +std::expected write_to_path(const typename Schema::latest_type& payload, + const std::filesystem::path& path, const bool make_dirs = true, - const CompressionAlgorithm compression_algorithm = - CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, + const CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, const ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib) { auto serialized = serialize(payload, compression_algorithm, checksum_algorithm); if (!serialized) { - return std::unexpected(FileError{serialized.error()}); + return std::unexpected(FileError { serialized.error() }); } - const auto bytes = std::span{ - reinterpret_cast(serialized->data()), + const auto bytes = std::span { + reinterpret_cast(serialized->data()), serialized->size(), }; auto result = ::io::write_bytes_to_path(bytes, path, make_dirs); if (!result) { - return std::unexpected(FileError{result.error()}); + return std::unexpected(FileError { result.error() }); } return {}; } diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index f2ea337a..dd83de61 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -19,50 +19,49 @@ enum class GltfContainer { class Gltf final : public store::Codec { public: - explicit Gltf(const GltfContainer container) : _container(container) {} + explicit Gltf(const GltfContainer container) + : m_container(container) + { + } - std::vector paths(const store::NodePath &node_path) const override { - return {store::codec::append_extension( - node_path, - _container == GltfContainer::Binary ? ".glb" : ".gltf")}; + std::vector paths(const store::NodePath& node_path) const override + { + return { store::codec::append_extension(node_path, m_container == GltfContainer::Binary ? ".glb" : ".gltf") }; } - std::expected read( - const store::NodePath &node_path) const override { + std::expected read(const store::NodePath& node_path) const override + { const auto result = mesh::io::gltf::load_from_path(paths(node_path).front()); if (!result.has_value()) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, - result.error() == mesh::io::LoadMeshErrorKind::FileNotFound - ? store::CodecErrorCategory::FileNotFound - : store::CodecErrorCategory::InvalidData, + result.error() == mesh::io::LoadMeshErrorKind::FileNotFound ? store::CodecErrorCategory::FileNotFound : store::CodecErrorCategory::InvalidData, result.error().description(), }); } return result.value(); } - std::expected write( - const store::NodePath &node_path, - const mesh::Simple &mesh) const override { + std::expected write(const store::NodePath& node_path, const mesh::Simple& mesh) const override + { try { const auto result = mesh::io::gltf::save_to_path(mesh, paths(node_path).front()); if (!result.has_value()) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, result.error().description(), }); } return {}; - } catch (const std::exception &error) { - return std::unexpected(store::CodecError{ + } catch (const std::exception& error) { + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, error.what(), }); } catch (...) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, "unknown glTF writer exception", @@ -70,12 +69,10 @@ class Gltf final : public store::Codec { } } - GltfContainer container() const { - return _container; - } + GltfContainer container() const { return m_container; } private: - GltfContainer _container; + GltfContainer m_container; }; } // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h index 33fb2c80..e9f4ada3 100644 --- a/src/terrainlib/mesh/codec/SfMesh.h +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -13,29 +13,25 @@ namespace mesh::codec { class SfMesh final : public store::Codec { public: - std::vector paths( - const store::NodePath &node_path) const override + std::vector paths(const store::NodePath& node_path) const override { - return {store::codec::append_extension(node_path, ".sfmesh")}; + return { store::codec::append_extension(node_path, ".sfmesh") }; } - std::expected read( - const store::NodePath &node_path) const override + std::expected read(const store::NodePath& node_path) const override { - auto payload = ::io::envelope::read_from_path( - paths(node_path).front()); + auto payload = ::io::envelope::read_from_path(paths(node_path).front()); if (!payload) { - const bool missing = std::holds_alternative<::io::Error>(payload.error()) - && std::get<::io::Error>(payload.error()) == ::io::Error(::io::Error::OpenFile); - return std::unexpected(store::CodecError{ + const bool missing + = std::holds_alternative<::io::Error>(payload.error()) && std::get<::io::Error>(payload.error()) == ::io::Error(::io::Error::OpenFile); + return std::unexpected(store::CodecError { store::CodecOperation::Read, - missing ? store::CodecErrorCategory::FileNotFound - : store::CodecErrorCategory::InvalidData, + missing ? store::CodecErrorCategory::FileNotFound : store::CodecErrorCategory::InvalidData, ::io::envelope::describe_error(payload.error()), }); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::InvalidData, valid.error(), @@ -43,7 +39,7 @@ class SfMesh final : public store::Codec { } auto decoded = mesh::sf::decode_payload(std::move(*payload)); if (!decoded) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::InvalidData, "could not decode SF mesh payload", @@ -52,38 +48,31 @@ class SfMesh final : public store::Codec { return std::move(*decoded); } - std::expected write( - const store::NodePath &node_path, - const mesh::Simple &mesh) const override + std::expected write(const store::NodePath& node_path, const mesh::Simple& mesh) const override { return write(node_path, mesh, {}); } - std::expected write( - const store::NodePath &node_path, - const mesh::Simple &mesh, - const mesh::EncodeOptions options) const + std::expected write(const store::NodePath& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const { auto payload = mesh::sf::encode_payload(mesh, options); if (!payload) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, "could not encode SF mesh payload", }); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, valid.error(), }); } - auto result = ::io::envelope::write_to_path( - *payload, - paths(node_path).front()); + auto result = ::io::envelope::write_to_path(*payload, paths(node_path).front()); if (!result) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Io, ::io::envelope::describe_error(result.error()), diff --git a/src/terrainlib/mesh/codec/SfMeshFormat.h b/src/terrainlib/mesh/codec/SfMeshFormat.h index b1461505..c2e17de8 100644 --- a/src/terrainlib/mesh/codec/SfMeshFormat.h +++ b/src/terrainlib/mesh/codec/SfMeshFormat.h @@ -15,34 +15,29 @@ namespace mesh::sf { namespace v1 { -struct Payload { - std::uint32_t vertex_count; - std::uint32_t face_count; - std::vector triangles; - std::vector positions; - std::vector uvs; - std::vector texture; -}; + struct Payload { + std::uint32_t vertex_count; + std::uint32_t face_count; + std::vector triangles; + std::vector positions; + std::vector uvs; + std::vector texture; + }; } // namespace v1 -using Schema = ::io::envelope::PayloadSchema< - "mesh.SfMesh", - ::io::envelope::Version<1, v1::Payload>>; +using Schema = ::io::envelope::PayloadSchema<"mesh.SfMesh", ::io::envelope::Version<1, v1::Payload>>; using Payload = Schema::latest_type; -inline std::expected validate(const Payload &payload) +inline std::expected validate(const Payload& payload) { - if (payload.vertex_count - > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Position)) { + if (payload.vertex_count > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Position)) { return std::unexpected("SF mesh vertex count exceeds the allocation limit"); } - if (payload.face_count - > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Triangle)) { + if (payload.face_count > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Triangle)) { return std::unexpected("SF mesh face count exceeds the allocation limit"); } - if (payload.vertex_count == 0 - && (!payload.positions.empty() || !payload.uvs.empty())) { + if (payload.vertex_count == 0 && (!payload.positions.empty() || !payload.uvs.empty())) { return std::unexpected("empty mesh contains vertex data"); } if (payload.vertex_count != 0 && payload.positions.empty()) { @@ -57,15 +52,13 @@ inline std::expected validate(const Payload &payload) return {}; } -inline std::expected encode_payload( - const mesh::Simple &mesh, - const mesh::EncodeOptions options = {}) +inline std::expected encode_payload(const mesh::Simple& mesh, const mesh::EncodeOptions options = {}) { auto encoded = mesh::encode(mesh, options); if (!encoded) { return std::unexpected(encoded.error()); } - return Payload{ + return Payload { .vertex_count = encoded->header.vertex_count, .face_count = encoded->header.face_count, .triangles = std::move(encoded->triangles), diff --git a/src/terrainlib/mesh/codec/from_extension.h b/src/terrainlib/mesh/codec/from_extension.h index 21209dbc..ed3dca40 100644 --- a/src/terrainlib/mesh/codec/from_extension.h +++ b/src/terrainlib/mesh/codec/from_extension.h @@ -12,8 +12,8 @@ namespace mesh::codec { -inline std::expected>, store::CodecError> -from_extension(const std::string_view extension) { +inline std::expected>, store::CodecError> from_extension(const std::string_view extension) +{ if (extension == ".sfmesh") { return std::make_unique(); } @@ -23,7 +23,7 @@ from_extension(const std::string_view extension) { if (extension == ".gltf") { return std::make_unique(GltfContainer::Json); } - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Resolve, store::CodecErrorCategory::UnsupportedCodec, "unsupported mesh codec selector: " + std::string(extension), diff --git a/src/terrainlib/mesh/storage.h b/src/terrainlib/mesh/storage.h index a26ac905..ebae84ed 100644 --- a/src/terrainlib/mesh/storage.h +++ b/src/terrainlib/mesh/storage.h @@ -18,35 +18,19 @@ using Storage = store::Storage; using IndexedStorage = store::IndexedStorage; using OpenOptions = octree::storage::OpenOptions; -inline std::expected> open_index( - const std::filesystem::path &path) { - return store::open_index( - path, - octree::storage::index_format(), - payload_class, - mesh::codec::from_extension); +inline std::expected> open_index(const std::filesystem::path& path) +{ + return store::open_index(path, octree::storage::index_format(), payload_class, mesh::codec::from_extension); } -inline std::expected> open_folder( - const std::filesystem::path &path, - OpenOptions options = {}) { - return octree::storage::open_folder( - path, - std::string(payload_class), - ".sfmesh", - mesh::codec::from_extension, - std::move(options)); +inline std::expected> open_folder(const std::filesystem::path& path, OpenOptions options = {}) +{ + return octree::storage::open_folder(path, std::string(payload_class), ".sfmesh", mesh::codec::from_extension, std::move(options)); } -inline std::expected> open_folder_indexed( - const std::filesystem::path &path, - OpenOptions options = {}) { - return octree::storage::open_folder_indexed( - path, - std::string(payload_class), - ".sfmesh", - mesh::codec::from_extension, - std::move(options)); +inline std::expected> open_folder_indexed(const std::filesystem::path& path, OpenOptions options = {}) +{ + return octree::storage::open_folder_indexed(path, std::string(payload_class), ".sfmesh", mesh::codec::from_extension, std::move(options)); } } // namespace mesh::storage diff --git a/src/terrainlib/octree/StoreTraits.h b/src/terrainlib/octree/StoreTraits.h index 655656a4..54252824 100644 --- a/src/terrainlib/octree/StoreTraits.h +++ b/src/terrainlib/octree/StoreTraits.h @@ -10,19 +10,10 @@ struct StoreTraits { using Key = Id; using Hasher = std::hash; - static constexpr Key root() { - return Key::root(); - } - static constexpr auto parent(const Key &key) { - return key.parent(); - } - static constexpr auto children(const Key &key) { - return key.children(); - } - static constexpr bool is_valid(const Key &key) { - return key.level() <= Key::max_level() - && key.index_on_level() <= Key::max_index_on_level(key.level()); - } + static constexpr Key root() { return Key::root(); } + static constexpr auto parent(const Key& key) { return key.parent(); } + static constexpr auto children(const Key& key) { return key.children(); } + static constexpr bool is_valid(const Key& key) { return key.level() <= Key::max_level() && key.index_on_level() <= Key::max_index_on_level(key.level()); } }; } // namespace octree diff --git a/src/terrainlib/octree/storage/Format.h b/src/terrainlib/octree/storage/Format.h index e7e48121..40325e8f 100644 --- a/src/terrainlib/octree/storage/Format.h +++ b/src/terrainlib/octree/storage/Format.h @@ -18,35 +18,31 @@ inline constexpr std::string_view index_file_name = "octree.storeindex"; namespace v1 { -struct StoreMetadata { - std::string layout_id; - std::string payload_class; - std::string codec_selector; -}; - -struct IndexEntry { - std::uint8_t level; - std::uint64_t index; - std::uint8_t status; -}; - -struct StoreIndex { - std::vector entries; -}; + struct StoreMetadata { + std::string layout_id; + std::string payload_class; + std::string codec_selector; + }; + + struct IndexEntry { + std::uint8_t level; + std::uint64_t index; + std::uint8_t status; + }; + + struct StoreIndex { + std::vector entries; + }; } // namespace v1 -using StoreMetadataSchema = io::envelope::PayloadSchema< - "octree.StoreMetadata", - io::envelope::Version<1, v1::StoreMetadata>>; -using StoreIndexSchema = io::envelope::PayloadSchema< - "octree.StoreIndex", - io::envelope::Version<1, v1::StoreIndex>>; +using StoreMetadataSchema = io::envelope::PayloadSchema<"octree.StoreMetadata", io::envelope::Version<1, v1::StoreMetadata>>; +using StoreIndexSchema = io::envelope::PayloadSchema<"octree.StoreIndex", io::envelope::Version<1, v1::StoreIndex>>; using StoreMetadata = StoreMetadataSchema::latest_type; using StoreIndex = StoreIndexSchema::latest_type; -inline std::expected validate(const StoreMetadata &metadata) +inline std::expected validate(const StoreMetadata& metadata) { if (metadata.layout_id.empty()) { return std::unexpected("layout ID is empty"); @@ -60,11 +56,11 @@ inline std::expected validate(const StoreMetadata &metadata) return {}; } -inline StoreIndex encode_index(const store::Index &index) +inline StoreIndex encode_index(const store::Index& index) { StoreIndex result; result.entries.reserve(index.size()); - for (const auto &[id, status] : index) { + for (const auto& [id, status] : index) { result.entries.push_back({ .level = id.level(), .index = id.index_on_level(), @@ -74,15 +70,12 @@ inline StoreIndex encode_index(const store::Index &index) return result; } -inline std::expected, std::string> decode_index( - const StoreIndex &encoded) +inline std::expected, std::string> decode_index(const StoreIndex& encoded) { store::Index result; std::unordered_set seen; - for (const v1::IndexEntry &entry : encoded.entries) { - const auto id = Id::try_make( - static_cast(entry.level), - static_cast(entry.index)); + for (const v1::IndexEntry& entry : encoded.entries) { + const auto id = Id::try_make(static_cast(entry.level), static_cast(entry.index)); if (!id) { return std::unexpected("index contains an invalid octree ID"); } @@ -92,19 +85,17 @@ inline std::expected, std::string> decode_index( if (entry.status > static_cast(store::NodeStatus::Virtual)) { return std::unexpected("index contains an invalid node status"); } - const auto set = result.set_raw( - *id, - store::NodeStatus{static_cast(entry.status)}); + const auto set = result.set_raw(*id, store::NodeStatus { static_cast(entry.status) }); if (!set) { return std::unexpected("index contains an invalid octree ID"); } } - for (const auto &[id, status] : result) { + for (const auto& [id, status] : result) { const auto children = StoreTraits::children(id); bool has_child = false; if (children) { - for (const Id &child : *children) { + for (const Id& child : *children) { const auto child_status = result.get(child); if (!child_status) { return std::unexpected("index child lookup failed"); @@ -112,10 +103,7 @@ inline std::expected, std::string> decode_index( has_child = has_child || child_status->has_value(); } } - if ((status == store::NodeStatus::Leaf && has_child) - || ((status == store::NodeStatus::Inner - || status == store::NodeStatus::Virtual) - && !has_child)) { + if ((status == store::NodeStatus::Leaf && has_child) || ((status == store::NodeStatus::Inner || status == store::NodeStatus::Virtual) && !has_child)) { return std::unexpected("index contains inconsistent node topology"); } @@ -127,8 +115,7 @@ inline std::expected, std::string> decode_index( continue; } const auto parent_status = result.get(*parent); - if (!parent_status || !parent_status->has_value() - || parent_status->value() == store::NodeStatus::Leaf) { + if (!parent_status || !parent_status->has_value() || parent_status->value() == store::NodeStatus::Leaf) { return std::unexpected("index contains a node without a valid indexed parent"); } } diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 1ef8f9b5..56d8a301 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -10,22 +10,17 @@ namespace octree::storage { -inline store::IndexFormatError read_error( - const std::filesystem::path &path, - const io::envelope::FileError &error) +inline store::IndexFormatError read_error(const std::filesystem::path& path, const io::envelope::FileError& error) { - const bool missing = std::holds_alternative(error) - && std::get(error) == io::Error(io::Error::OpenFile); + const bool missing = std::holds_alternative(error) && std::get(error) == io::Error(io::Error::OpenFile); return { - missing ? store::IndexFormatErrorCategory::Open - : store::IndexFormatErrorCategory::Malformed, + missing ? store::IndexFormatErrorCategory::Open : store::IndexFormatErrorCategory::Malformed, path, io::envelope::describe_error(error), }; } -inline std::expected read_store_metadata( - const std::filesystem::path &base_path) +inline std::expected read_store_metadata(const std::filesystem::path& base_path) { const std::filesystem::path path = base_path / metadata_file_name; auto metadata = io::envelope::read_from_path(path); @@ -33,7 +28,7 @@ inline std::expected read_store_metadata return std::unexpected(read_error(path, metadata.error())); } if (auto valid = validate(*metadata); !valid) { - return std::unexpected(store::IndexFormatError{ + return std::unexpected(store::IndexFormatError { store::IndexFormatErrorCategory::Malformed, path, valid.error(), @@ -42,8 +37,7 @@ inline std::expected read_store_metadata return std::move(*metadata); } -inline std::expected, store::IndexFormatError> -read_index_file(const std::filesystem::path &path) +inline std::expected, store::IndexFormatError> read_index_file(const std::filesystem::path& path) { auto metadata = read_store_metadata(path.parent_path()); if (!metadata) { @@ -56,13 +50,13 @@ read_index_file(const std::filesystem::path &path) } auto index = decode_index(*encoded_index); if (!index) { - return std::unexpected(store::IndexFormatError{ + return std::unexpected(store::IndexFormatError { store::IndexFormatErrorCategory::Malformed, path, index.error(), }); } - return store::IndexMetadata{ + return store::IndexMetadata { std::move(*index), std::move(metadata->layout_id), std::move(metadata->payload_class), @@ -70,28 +64,23 @@ read_index_file(const std::filesystem::path &path) }; } -inline std::expected write_index_file( - const std::filesystem::path &path, - const store::IndexMetadata &metadata) +inline std::expected write_index_file(const std::filesystem::path& path, const store::IndexMetadata& metadata) { - const StoreMetadata store_metadata{ + const StoreMetadata store_metadata { .layout_id = metadata.layout_id, .payload_class = metadata.payload_class, .codec_selector = metadata.codec_selector, }; if (auto valid = validate(store_metadata); !valid) { - return std::unexpected(store::IndexFormatError{ + return std::unexpected(store::IndexFormatError { store::IndexFormatErrorCategory::Write, path.parent_path() / metadata_file_name, valid.error(), }); } const std::filesystem::path metadata_path = path.parent_path() / metadata_file_name; - if (auto written = io::envelope::write_to_path( - store_metadata, - metadata_path); - !written) { - return std::unexpected(store::IndexFormatError{ + if (auto written = io::envelope::write_to_path(store_metadata, metadata_path); !written) { + return std::unexpected(store::IndexFormatError { store::IndexFormatErrorCategory::Write, metadata_path, io::envelope::describe_error(written.error()), @@ -99,9 +88,8 @@ inline std::expected write_index_file( } const StoreIndex encoded_index = encode_index(metadata.index); - if (auto written = io::envelope::write_to_path(encoded_index, path); - !written) { - return std::unexpected(store::IndexFormatError{ + if (auto written = io::envelope::write_to_path(encoded_index, path); !written) { + return std::unexpected(store::IndexFormatError { store::IndexFormatErrorCategory::Write, path, io::envelope::describe_error(written.error()), @@ -110,10 +98,7 @@ inline std::expected write_index_file( return {}; } -inline store::PathMapping default_mapping() -{ - return store_layout::level_and_coordinate_directories(); -} +inline store::PathMapping default_mapping() { return store_layout::level_and_coordinate_directories(); } inline store::IndexFormat index_format() { diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index 1994bfd9..49d2fc4d 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -19,9 +19,8 @@ struct OpenOptions { std::optional preferred_extension; }; -template -std::expected, store::OpenError> open_folder( - const std::filesystem::path &base_path, +template +std::expected, store::OpenError> open_folder(const std::filesystem::path& base_path, std::string payload_class, std::string default_codec_selector, CodecResolver resolve_codec, @@ -30,14 +29,14 @@ std::expected, store::OpenError> open_ std::error_code error; const bool base_exists = std::filesystem::exists(base_path, error); if (error) { - return std::unexpected(store::OpenError(store::FilesystemError{ + return std::unexpected(store::OpenError(store::FilesystemError { base_path, "exists", error, })); } if (base_exists && !std::filesystem::is_directory(base_path, error)) { - return std::unexpected(store::OpenError(store::FilesystemError{ + return std::unexpected(store::OpenError(store::FilesystemError { base_path, "is_directory", error ? error : std::make_error_code(std::errc::not_a_directory), @@ -46,7 +45,7 @@ std::expected, store::OpenError> open_ if (!base_exists) { std::filesystem::create_directories(base_path, error); if (error) { - return std::unexpected(store::OpenError(store::FilesystemError{ + return std::unexpected(store::OpenError(store::FilesystemError { base_path, "create_directories", error, @@ -59,7 +58,7 @@ std::expected, store::OpenError> open_ const std::filesystem::path metadata_path = base_path / metadata_file_name; const bool index_exists = std::filesystem::exists(index_path, error); if (error) { - return std::unexpected(store::OpenError(store::FilesystemError{ + return std::unexpected(store::OpenError(store::FilesystemError { index_path, "exists", error, @@ -67,64 +66,45 @@ std::expected, store::OpenError> open_ } const bool metadata_exists = std::filesystem::exists(metadata_path, error); if (error) { - return std::unexpected(store::OpenError(store::FilesystemError{ + return std::unexpected(store::OpenError(store::FilesystemError { metadata_path, "exists", error, })); } if (index_exists != metadata_exists) { - return std::unexpected(store::OpenError(store::IndexFormatError{ + return std::unexpected(store::OpenError(store::IndexFormatError { store::IndexFormatErrorCategory::Open, index_exists ? metadata_path : index_path, "octree dataset metadata and index must either both exist or both be absent", })); } if (index_exists) { - auto indexed = store::open_index( - index_path, - format, - payload_class, - resolve_codec); + auto indexed = store::open_index(index_path, format, payload_class, resolve_codec); if (!indexed) { return std::unexpected(indexed.error()); } return store::Storage(std::move(*indexed)); } - const store::PathMapping mapping = - options.default_mapping.value_or(format.default_mapping()); - std::string codec_selector = - options.preferred_extension.value_or(std::move(default_codec_selector)); + const store::PathMapping mapping = options.default_mapping.value_or(format.default_mapping()); + std::string codec_selector = options.preferred_extension.value_or(std::move(default_codec_selector)); auto codec = resolve_codec(codec_selector); if (!codec) { return std::unexpected(store::OpenError(codec.error())); } - return store::make_storage( - base_path, - format, - mapping, - std::move(payload_class), - std::move(codec_selector), - std::move(*codec)); + return store::make_storage(base_path, format, mapping, std::move(payload_class), std::move(codec_selector), std::move(*codec)); } -template -std::expected, store::OpenError> -open_folder_indexed( - const std::filesystem::path &base_path, +template +std::expected, store::OpenError> open_folder_indexed(const std::filesystem::path& base_path, std::string payload_class, std::string default_codec_selector, CodecResolver resolve_codec, OpenOptions options = {}) { - auto result = open_folder( - base_path, - std::move(payload_class), - std::move(default_codec_selector), - std::move(resolve_codec), - std::move(options)); + auto result = open_folder(base_path, std::move(payload_class), std::move(default_codec_selector), std::move(resolve_codec), std::move(options)); if (!result) { return std::unexpected(result.error()); } diff --git a/src/terrainlib/octree/store_layout/Flat.h b/src/terrainlib/octree/store_layout/Flat.h index c7f58a7f..e05f7f14 100644 --- a/src/terrainlib/octree/store_layout/Flat.h +++ b/src/terrainlib/octree/store_layout/Flat.h @@ -13,12 +13,11 @@ namespace octree::store_layout { -inline store::NodePath flat_key_to_node_path(const Id &id) { - return store::NodePath(fmt::format("{}-{}", id.level(), id.index_on_level())); -} +inline store::NodePath flat_key_to_node_path(const Id& id) { return store::NodePath(fmt::format("{}-{}", id.level(), id.index_on_level())); } -inline std::optional flat_node_path_to_key(const store::NodePath &node_path) { - const std::filesystem::path &path = node_path.path(); +inline std::optional flat_node_path_to_key(const store::NodePath& node_path) +{ + const std::filesystem::path& path = node_path.path(); if (path.empty() || path.is_absolute() || path.has_parent_path() || path.has_extension()) { return std::nullopt; } diff --git a/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h index 7036443a..9545adad 100644 --- a/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h +++ b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h @@ -12,19 +12,15 @@ namespace octree::store_layout { -inline store::NodePath level_and_coordinate_key_to_node_path(const Id &id) { +inline store::NodePath level_and_coordinate_key_to_node_path(const Id& id) +{ const Id::Coords coordinates = id.coords(); - return store::NodePath(fmt::format( - "{}/{}/{}/{}", - id.level(), - coordinates.x, - coordinates.y, - coordinates.z)); + return store::NodePath(fmt::format("{}/{}/{}/{}", id.level(), coordinates.x, coordinates.y, coordinates.z)); } -inline std::optional level_and_coordinate_node_path_to_key( - const store::NodePath &node_path) { - const std::filesystem::path &path = node_path.path(); +inline std::optional level_and_coordinate_node_path_to_key(const store::NodePath& node_path) +{ + const std::filesystem::path& path = node_path.path(); if (path.empty() || path.is_absolute() || std::distance(path.begin(), path.end()) != 4) { return std::nullopt; } @@ -43,7 +39,7 @@ inline std::optional level_and_coordinate_node_path_to_key( if (!level.has_value() || !x.has_value() || !y.has_value() || !z.has_value()) { return std::nullopt; } - return Id::try_make(level.value(), {x.value(), y.value(), z.value()}); + return Id::try_make(level.value(), { x.value(), y.value(), z.value() }); } } // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/Mappings.h b/src/terrainlib/octree/store_layout/Mappings.h index 1b46bed8..2cdab912 100644 --- a/src/terrainlib/octree/store_layout/Mappings.h +++ b/src/terrainlib/octree/store_layout/Mappings.h @@ -10,11 +10,10 @@ namespace octree::store_layout { -inline store::PathMapping flat() { - return {"flat", flat_key_to_node_path, flat_node_path_to_key}; -} +inline store::PathMapping flat() { return { "flat", flat_key_to_node_path, flat_node_path_to_key }; } -inline store::PathMapping level_and_coordinate_directories() { +inline store::PathMapping level_and_coordinate_directories() +{ return { "level_and_coordinate_directories", level_and_coordinate_key_to_node_path, @@ -22,11 +21,10 @@ inline store::PathMapping level_and_coordinate_directories() { }; } -inline std::array, 2> all() { - return {flat(), level_and_coordinate_directories()}; -} +inline std::array, 2> all() { return { flat(), level_and_coordinate_directories() }; } -inline std::optional> from_id(const std::string_view id) { +inline std::optional> from_id(const std::string_view id) +{ for (const auto mapping : all()) { if (mapping.id == id) { return mapping; diff --git a/src/terrainlib/raster_store/StoreTraits.h b/src/terrainlib/raster_store/StoreTraits.h index a6ba4a8b..bd1dea66 100644 --- a/src/terrainlib/raster_store/StoreTraits.h +++ b/src/terrainlib/raster_store/StoreTraits.h @@ -15,29 +15,30 @@ struct StoreTraits { static constexpr unsigned max_zoom_level = std::numeric_limits::digits; - static Key root() { - return {0, {0, 0}}; - } - static std::optional parent(const Key &key) { + static Key root() { return { 0, { 0, 0 } }; } + static std::optional parent(const Key& key) + { if (key.zoom_level == 0) { return std::nullopt; } return key.parent(); } - static std::optional> children(const Key &key) { + static std::optional> children(const Key& key) + { if (key.zoom_level >= max_zoom_level) { return std::nullopt; } return key.children(); } - static bool is_valid(const Key &key) { + static bool is_valid(const Key& key) + { if (key.zoom_level > max_zoom_level) { return false; } if (key.zoom_level == max_zoom_level) { return true; } - const uint32_t extent = uint32_t{1} << key.zoom_level; + const uint32_t extent = uint32_t { 1 } << key.zoom_level; return key.coords.x < extent && key.coords.y < extent; } }; diff --git a/src/terrainlib/sf/Error.h b/src/terrainlib/sf/Error.h index a8274c72..337a58c3 100644 --- a/src/terrainlib/sf/Error.h +++ b/src/terrainlib/sf/Error.h @@ -13,23 +13,20 @@ namespace sf { using FinalizeError = std::variant; -using ProcessingError = std::variant< - InvalidTopology, +using ProcessingError = std::variant, store::FileOperationError, store::SaveError, store::CopyError, store::IndexFormatError>; -inline std::string describe_error(const InvalidTopology &error) { - return "Structura Fundamentalis topology contains Inner node " - + error.key.to_string(); -} +inline std::string describe_error(const InvalidTopology& error) { return "Structura Fundamentalis topology contains Inner node " + error.key.to_string(); } -template -std::string describe_error(const std::variant &error) { +template +std::string describe_error(const std::variant& error) +{ return std::visit( - [](const auto &value) -> std::string { + [](const auto& value) -> std::string { if constexpr (std::is_same_v, InvalidTopology>) { return describe_error(value); } else { diff --git a/src/terrainlib/sf/InvalidTopology.h b/src/terrainlib/sf/InvalidTopology.h index 643d769d..a6010691 100644 --- a/src/terrainlib/sf/InvalidTopology.h +++ b/src/terrainlib/sf/InvalidTopology.h @@ -7,7 +7,7 @@ namespace sf { struct InvalidTopology { octree::Id key; - bool operator==(const InvalidTopology &) const = default; + bool operator==(const InvalidTopology&) const = default; }; } // namespace sf diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h index 157d9852..c2e6386e 100644 --- a/src/terrainlib/sf/finalize_storage.h +++ b/src/terrainlib/sf/finalize_storage.h @@ -10,7 +10,8 @@ namespace sf { -inline std::expected finalize_storage(mesh::storage::Storage &storage) { +inline std::expected finalize_storage(mesh::storage::Storage& storage) +{ const auto save_result = storage.save_or_create_index(); if (!save_result.has_value()) { return std::unexpected(FinalizeError(save_result.error())); diff --git a/src/terrainlib/sf/validate_index.h b/src/terrainlib/sf/validate_index.h index 1cf33719..da04a239 100644 --- a/src/terrainlib/sf/validate_index.h +++ b/src/terrainlib/sf/validate_index.h @@ -8,11 +8,11 @@ namespace sf { -inline std::expected validate_index( - const store::Index &index) { - for (const auto &[key, status] : index) { +inline std::expected validate_index(const store::Index& index) +{ + for (const auto& [key, status] : index) { if (status == store::NodeStatus::Inner) { - return std::unexpected(InvalidTopology{key}); + return std::unexpected(InvalidTopology { key }); } } return {}; diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h index c9611589..d112f69f 100644 --- a/src/terrainlib/store/Codec.h +++ b/src/terrainlib/store/Codec.h @@ -10,21 +10,21 @@ namespace store { -template +template class Codec { public: virtual ~Codec() = default; - virtual std::vector paths(const NodePath &node_path) const = 0; + virtual std::vector paths(const NodePath& node_path) const = 0; - virtual std::expected read(const NodePath &) const { - return std::unexpected( - CodecError::unsupported_operation(CodecOperation::Read, "read")); + virtual std::expected read(const NodePath&) const + { + return std::unexpected(CodecError::unsupported_operation(CodecOperation::Read, "read")); } - virtual std::expected write(const NodePath &, const NodeData &) const { - return std::unexpected( - CodecError::unsupported_operation(CodecOperation::Write, "write")); + virtual std::expected write(const NodePath&, const NodeData&) const + { + return std::unexpected(CodecError::unsupported_operation(CodecOperation::Write, "write")); } }; diff --git a/src/terrainlib/store/CodecError.h b/src/terrainlib/store/CodecError.h index ba4adeb8..835069bd 100644 --- a/src/terrainlib/store/CodecError.h +++ b/src/terrainlib/store/CodecError.h @@ -26,13 +26,12 @@ struct CodecError { CodecErrorCategory category; std::string message; - static CodecError unsupported_operation( - const CodecOperation operation, - const std::string_view name) { - return {operation, CodecErrorCategory::UnsupportedOperation, std::string(name)}; + static CodecError unsupported_operation(const CodecOperation operation, const std::string_view name) + { + return { operation, CodecErrorCategory::UnsupportedOperation, std::string(name) }; } - bool operator==(const CodecError &) const = default; + bool operator==(const CodecError&) const = default; }; } // namespace store diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h index 024e3b2b..bb7998f5 100644 --- a/src/terrainlib/store/Index.h +++ b/src/terrainlib/store/Index.h @@ -5,15 +5,15 @@ #include #include -#include #include "log.h" #include "store/InvalidKey.h" #include "store/NodeStatus.h" #include "store/Traits.h" +#include namespace store { -template +template class Index { public: using Key = typename Traits::Key; @@ -22,23 +22,25 @@ class Index { using iterator = typename Container::iterator; using const_iterator = typename Container::const_iterator; - std::expected, Error> get(const Key &key) const { + std::expected, Error> get(const Key& key) const + { if (!Traits::is_valid(key)) { - return std::unexpected(Error{key}); + return std::unexpected(Error { key }); } - const NodeStatus *status = get_raw_unchecked(key); + const NodeStatus* status = get_raw_unchecked(key); if (status == nullptr) { return std::nullopt; } return *status; } - std::expected add(const Key &key) { + std::expected add(const Key& key) + { if (!Traits::is_valid(key)) { - return std::unexpected(Error{key}); + return std::unexpected(Error { key }); } - NodeStatus *status = get_raw_unchecked(key); + NodeStatus* status = get_raw_unchecked(key); if (status != nullptr) { switch (*status) { case NodeStatus::Inner: @@ -61,9 +63,9 @@ class Index { } if (!Traits::is_valid(parent.value())) { - return std::unexpected(Error{parent.value()}); + return std::unexpected(Error { parent.value() }); } - NodeStatus *parent_status = get_raw_unchecked(parent.value()); + NodeStatus* parent_status = get_raw_unchecked(parent.value()); if (parent_status == nullptr) { const auto parent_result = add(parent.value()); if (!parent_result.has_value()) { @@ -87,12 +89,13 @@ class Index { return true; } - std::expected remove(const Key &key) { + std::expected remove(const Key& key) + { if (!Traits::is_valid(key)) { - return std::unexpected(Error{key}); + return std::unexpected(Error { key }); } - NodeStatus *status = get_raw_unchecked(key); + NodeStatus* status = get_raw_unchecked(key); if (status == nullptr) { return false; } @@ -109,20 +112,23 @@ class Index { UNREACHABLE(); } - std::expected is_present(const Key &key) const { + std::expected is_present(const Key& key) const + { if (!Traits::is_valid(key)) { - return std::unexpected(Error{key}); + return std::unexpected(Error { key }); } - return _index.contains(key); + return m_index.contains(key); } - std::expected is_absent(const Key &key) const { + std::expected is_absent(const Key& key) const + { const auto present = is_present(key); if (!present.has_value()) { return std::unexpected(present.error()); } return !present.value(); } - std::expected is(const NodeStatus status, const Key &key) const { + std::expected is(const NodeStatus status, const Key& key) const + { const auto result = get(key); if (!result.has_value()) { return std::unexpected(result.error()); @@ -130,86 +136,75 @@ class Index { return result.value() == status; } - std::expected get_raw(const Key &key) { + std::expected get_raw(const Key& key) + { if (!Traits::is_valid(key)) { - return std::unexpected(Error{key}); + return std::unexpected(Error { key }); } return get_raw_unchecked(key); } - std::expected get_raw(const Key &key) const { + std::expected get_raw(const Key& key) const + { if (!Traits::is_valid(key)) { - return std::unexpected(Error{key}); + return std::unexpected(Error { key }); } return get_raw_unchecked(key); } - std::expected set_raw(const Key &key, const NodeStatus status) { + std::expected set_raw(const Key& key, const NodeStatus status) + { if (!Traits::is_valid(key)) { - return std::unexpected(Error{key}); + return std::unexpected(Error { key }); } set_raw_unchecked(key, status); return {}; } - std::expected remove_raw(const Key &key) { + std::expected remove_raw(const Key& key) + { if (!Traits::is_valid(key)) { - return std::unexpected(Error{key}); + return std::unexpected(Error { key }); } remove_raw_unchecked(key); return {}; } - void clear() { - _index.clear(); - } - bool empty() const { - return _index.empty(); - } - size_t size() const { - return _index.size(); - } + void clear() { m_index.clear(); } + bool empty() const { return m_index.empty(); } + size_t size() const { return m_index.size(); } - const_iterator begin() const { - return _index.begin(); - } - const_iterator end() const { - return _index.end(); - } - const_iterator cbegin() const { - return _index.cbegin(); - } - const_iterator cend() const { - return _index.cend(); - } + const_iterator begin() const { return m_index.begin(); } + const_iterator end() const { return m_index.end(); } + const_iterator cbegin() const { return m_index.cbegin(); } + const_iterator cend() const { return m_index.cend(); } private: - NodeStatus *get_raw_unchecked(const Key &key) { - const auto iterator = _index.find(key); - return iterator == _index.end() ? nullptr : &iterator->second; - } - const NodeStatus *get_raw_unchecked(const Key &key) const { - const auto iterator = _index.find(key); - return iterator == _index.end() ? nullptr : &iterator->second; - } - void set_raw_unchecked(const Key &key, const NodeStatus status) { - _index[key] = status; + NodeStatus* get_raw_unchecked(const Key& key) + { + const auto iterator = m_index.find(key); + return iterator == m_index.end() ? nullptr : &iterator->second; } - void remove_raw_unchecked(const Key &key) { - _index.erase(key); + const NodeStatus* get_raw_unchecked(const Key& key) const + { + const auto iterator = m_index.find(key); + return iterator == m_index.end() ? nullptr : &iterator->second; } + void set_raw_unchecked(const Key& key, const NodeStatus status) { m_index[key] = status; } + void remove_raw_unchecked(const Key& key) { m_index.erase(key); } - std::expected update_parent_after_remove(const Key &key) { + std::expected update_parent_after_remove(const Key& key) + { const auto parent = Traits::parent(key); if (!parent.has_value()) { return true; } if (!Traits::is_valid(parent.value())) { - return std::unexpected(Error{parent.value()}); + return std::unexpected(Error { parent.value() }); } - NodeStatus *parent_status = get_raw_unchecked(parent.value()); + NodeStatus* parent_status = get_raw_unchecked(parent.value()); DEBUG_ASSERT(parent_status != nullptr); const auto siblings = Traits::children(parent.value()); DEBUG_ASSERT(siblings.has_value()); - for (const Key &sibling : siblings.value()) { + for (const Key& sibling : siblings.value()) { if (sibling != key && get_raw_unchecked(sibling) != nullptr) { return true; } @@ -228,7 +223,7 @@ class Index { return true; } - Container _index; + Container m_index; }; } // namespace store diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h index d307f14e..57e2d893 100644 --- a/src/terrainlib/store/IndexFormat.h +++ b/src/terrainlib/store/IndexFormat.h @@ -23,10 +23,10 @@ struct IndexFormatError { std::filesystem::path path; std::string message; - bool operator==(const IndexFormatError &) const = default; + bool operator==(const IndexFormatError&) const = default; }; -template +template struct IndexMetadata { Index index; std::string layout_id; @@ -34,15 +34,12 @@ struct IndexMetadata { std::string codec_selector; }; -template +template struct IndexFormat { std::string_view index_filename; - std::expected, IndexFormatError> (*read)( - const std::filesystem::path &index_path); - std::expected (*write)( - const std::filesystem::path &index_path, - const IndexMetadata &metadata); + std::expected, IndexFormatError> (*read)(const std::filesystem::path& index_path); + std::expected (*write)(const std::filesystem::path& index_path, const IndexMetadata& metadata); std::optional> (*mapping_from_id)(std::string_view id); PathMapping (*default_mapping)(); }; diff --git a/src/terrainlib/store/IndexedStorage.h b/src/terrainlib/store/IndexedStorage.h index 9a963a10..6aac6138 100644 --- a/src/terrainlib/store/IndexedStorage.h +++ b/src/terrainlib/store/IndexedStorage.h @@ -6,31 +6,29 @@ namespace store { -template +template class IndexedStorage : public Storage { public: using Base = Storage; using typename Base::Persistence; explicit IndexedStorage(Storage storage) - : Base(std::move(storage)) { + : Base(std::move(storage)) + { this->ensure_indexed(); } - IndexedStorage( - RawStorage raw, - Index index, - Persistence persistence) - : Base(std::move(raw), std::move(index), std::move(persistence)) {} + IndexedStorage(RawStorage raw, Index index, Persistence persistence) + : Base(std::move(raw), std::move(index), std::move(persistence)) + { + } - IndexedStorage(const IndexedStorage &) = delete; - IndexedStorage &operator=(const IndexedStorage &) = delete; - IndexedStorage(IndexedStorage &&) noexcept = default; - IndexedStorage &operator=(IndexedStorage &&) noexcept = default; + IndexedStorage(const IndexedStorage&) = delete; + IndexedStorage& operator=(const IndexedStorage&) = delete; + IndexedStorage(IndexedStorage&&) noexcept = default; + IndexedStorage& operator=(IndexedStorage&&) noexcept = default; - const Index &index() const { - return this->index_ref(); - } + const Index& index() const { return this->index_ref(); } }; } // namespace store diff --git a/src/terrainlib/store/InvalidKey.h b/src/terrainlib/store/InvalidKey.h index d8caad27..6b23ee9c 100644 --- a/src/terrainlib/store/InvalidKey.h +++ b/src/terrainlib/store/InvalidKey.h @@ -2,11 +2,11 @@ namespace store { -template +template struct InvalidKey { Key key; - bool operator==(const InvalidKey &) const = default; + bool operator==(const InvalidKey&) const = default; }; } // namespace store diff --git a/src/terrainlib/store/Layout.h b/src/terrainlib/store/Layout.h index b987c535..0bbec70d 100644 --- a/src/terrainlib/store/Layout.h +++ b/src/terrainlib/store/Layout.h @@ -9,36 +9,33 @@ namespace store { -template +template class Layout { public: Layout(std::filesystem::path base_path, PathMapping mapping) - : _base_path(std::move(base_path)), _mapping(mapping) {} - - NodePath node_path(const Key &key) const { - return NodePath(_base_path / _mapping.key_to_node_path(key).path()); + : m_base_path(std::move(base_path)) + , m_mapping(mapping) + { } - std::optional key_from_node_path(const NodePath &node_path) const { + NodePath node_path(const Key& key) const { return NodePath(m_base_path / m_mapping.key_to_node_path(key).path()); } + + std::optional key_from_node_path(const NodePath& node_path) const + { std::error_code error; - const std::filesystem::path relative = - std::filesystem::relative(node_path.path(), _base_path, error); + const std::filesystem::path relative = std::filesystem::relative(node_path.path(), m_base_path, error); if (error || relative.empty() || relative.is_absolute()) { return std::nullopt; } - return _mapping.node_path_to_key(NodePath(relative)); + return m_mapping.node_path_to_key(NodePath(relative)); } - const std::filesystem::path &base_path() const { - return _base_path; - } - PathMapping mapping() const { - return _mapping; - } + const std::filesystem::path& base_path() const { return m_base_path; } + PathMapping mapping() const { return m_mapping; } private: - std::filesystem::path _base_path; - PathMapping _mapping; + std::filesystem::path m_base_path; + PathMapping m_mapping; }; } // namespace store diff --git a/src/terrainlib/store/NodePath.h b/src/terrainlib/store/NodePath.h index 31673862..3307ec9a 100644 --- a/src/terrainlib/store/NodePath.h +++ b/src/terrainlib/store/NodePath.h @@ -8,16 +8,17 @@ namespace store { class NodePath { public: NodePath() = default; - explicit NodePath(std::filesystem::path path) : _path(std::move(path)) {} - - const std::filesystem::path &path() const { - return _path; + explicit NodePath(std::filesystem::path path) + : m_path(std::move(path)) + { } - bool operator==(const NodePath &) const = default; + const std::filesystem::path& path() const { return m_path; } + + bool operator==(const NodePath&) const = default; private: - std::filesystem::path _path; + std::filesystem::path m_path; }; } // namespace store diff --git a/src/terrainlib/store/NodeStatus.h b/src/terrainlib/store/NodeStatus.h index 5a012f3f..4cd17241 100644 --- a/src/terrainlib/store/NodeStatus.h +++ b/src/terrainlib/store/NodeStatus.h @@ -4,9 +4,9 @@ #include #include +#include "log.h" #include #include -#include "log.h" namespace store { @@ -20,42 +20,37 @@ class NodeStatus { }; constexpr NodeStatus() = default; - constexpr NodeStatus(const Value value) : _value(value) {} - - constexpr operator Value() const { - return _value; + constexpr NodeStatus(const Value value) + : m_value(value) + { } + + constexpr operator Value() const { return m_value; } explicit operator bool() const = delete; - constexpr bool operator==(const NodeStatus other) const { - return _value == other._value; - } - constexpr bool operator!=(const NodeStatus other) const { - return !(*this == other); - } - constexpr bool operator==(const Value other) const { - return _value == other; - } - constexpr bool operator!=(const Value other) const { - return _value != other; - } + constexpr bool operator==(const NodeStatus other) const { return m_value == other.m_value; } + constexpr bool operator!=(const NodeStatus other) const { return !(*this == other); } + constexpr bool operator==(const Value other) const { return m_value == other; } + constexpr bool operator!=(const Value other) const { return m_value != other; } std::string to_string() const; private: - Value _value; + Value m_value; }; } // namespace store -template<> +template <> struct fmt::formatter { - template - constexpr auto parse(ParseContext &context) { + template + constexpr auto parse(ParseContext& context) + { return context.begin(); } - template - auto format(const store::NodeStatus &status, FormatContext &context) const { + template + auto format(const store::NodeStatus& status, FormatContext& context) const + { switch (status) { case store::NodeStatus::Leaf: return fmt::format_to(context.out(), "Leaf"); @@ -69,11 +64,10 @@ struct fmt::formatter { } }; -inline std::string store::NodeStatus::to_string() const { - return fmt::format("{}", *this); -} +inline std::string store::NodeStatus::to_string() const { return fmt::format("{}", *this); } -inline std::ostream &operator<<(std::ostream &stream, const store::NodeStatus &status) { +inline std::ostream& operator<<(std::ostream& stream, const store::NodeStatus& status) +{ fmt::print(stream, "{}", status); return stream; } diff --git a/src/terrainlib/store/NodeStatusOrMissing.h b/src/terrainlib/store/NodeStatusOrMissing.h index 4157b57d..1b69fd28 100644 --- a/src/terrainlib/store/NodeStatusOrMissing.h +++ b/src/terrainlib/store/NodeStatusOrMissing.h @@ -27,88 +27,82 @@ class NodeStatusOrMissing { }; constexpr NodeStatusOrMissing() = default; - constexpr NodeStatusOrMissing(const Value value) : _value(value) {} - NodeStatusOrMissing(const NodeStatus status) : _value(from_status(status)) {} - NodeStatusOrMissing(const std::optional status) : _value(from_optional_status(status)) {} - - constexpr operator Value() const { - return _value; - } - constexpr operator std::optional() const { - return to_optional_status(); - } - explicit operator bool() const = delete; - constexpr bool operator==(const NodeStatusOrMissing other) const { - return _value == other._value; + constexpr NodeStatusOrMissing(const Value value) + : m_value(value) + { } - constexpr bool operator!=(const NodeStatusOrMissing other) const { - return !(*this == other); + NodeStatusOrMissing(const NodeStatus status) + : m_value(from_status(status)) + { } - constexpr bool operator==(const Value other) const { - return _value == other; - } - constexpr bool operator!=(const Value other) const { - return _value != other; + NodeStatusOrMissing(const std::optional status) + : m_value(from_optional_status(status)) + { } + constexpr operator Value() const { return m_value; } + constexpr operator std::optional() const { return to_optional_status(); } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatusOrMissing other) const { return m_value == other.m_value; } + constexpr bool operator!=(const NodeStatusOrMissing other) const { return !(*this == other); } + constexpr bool operator==(const Value other) const { return m_value == other; } + constexpr bool operator!=(const Value other) const { return m_value != other; } + std::string to_string() const; private: - static constexpr NodeStatusOrMissing from_status(const NodeStatus status) noexcept { - return static_cast(static_cast(status)); + static constexpr NodeStatusOrMissing from_status(const NodeStatus status) noexcept { return static_cast(static_cast(status)); } + static constexpr NodeStatusOrMissing from_optional_status(const std::optional status) noexcept + { + return status.has_value() ? from_status(status.value()) : NodeStatusOrMissing(NodeStatusOrMissing::Missing); } - static constexpr NodeStatusOrMissing from_optional_status(const std::optional status) noexcept { - return status.has_value() - ? from_status(status.value()) - : NodeStatusOrMissing(NodeStatusOrMissing::Missing); - } - constexpr std::optional to_optional_status() const noexcept { - if (_value == Missing) { + constexpr std::optional to_optional_status() const noexcept + { + if (m_value == Missing) { return std::nullopt; } - return static_cast(_value); + return static_cast(m_value); } public: - Value _value; + Value m_value; }; namespace detail { -template -consteval bool verify_node_status_values(std::index_sequence) { - constexpr auto status_values = magic_enum::enum_values(); - constexpr auto optional_values = magic_enum::enum_values(); - return ((magic_enum::enum_underlying(status_values[Indices]) - == magic_enum::enum_underlying(optional_values[Indices])) - && ...); -} + template + consteval bool verify_node_status_values(std::index_sequence) + { + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + return ((magic_enum::enum_underlying(status_values[Indices]) == magic_enum::enum_underlying(optional_values[Indices])) && ...); + } -consteval bool verify_node_status_enums() { - static_assert( - std::is_same_v< - magic_enum::underlying_type_t, - magic_enum::underlying_type_t>); - constexpr auto status_values = magic_enum::enum_values(); - constexpr auto optional_values = magic_enum::enum_values(); - static_assert(optional_values.size() == status_values.size() + 1); - static_assert(verify_node_status_values(std::make_index_sequence())); - return true; -} + consteval bool verify_node_status_enums() + { + static_assert(std::is_same_v, magic_enum::underlying_type_t>); + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + static_assert(optional_values.size() == status_values.size() + 1); + static_assert(verify_node_status_values(std::make_index_sequence())); + return true; + } -static_assert(verify_node_status_enums()); + static_assert(verify_node_status_enums()); } // namespace detail } // namespace store -template<> +template <> struct fmt::formatter { - template - constexpr auto parse(ParseContext &context) { + template + constexpr auto parse(ParseContext& context) + { return context.begin(); } - template - auto format(const store::NodeStatusOrMissing &status, FormatContext &context) const { + template + auto format(const store::NodeStatusOrMissing& status, FormatContext& context) const + { switch (status) { case store::NodeStatusOrMissing::Leaf: return fmt::format_to(context.out(), "Leaf"); @@ -124,11 +118,10 @@ struct fmt::formatter { } }; -inline std::string store::NodeStatusOrMissing::to_string() const { - return fmt::format("{}", *this); -} +inline std::string store::NodeStatusOrMissing::to_string() const { return fmt::format("{}", *this); } -inline std::ostream &operator<<(std::ostream &stream, const store::NodeStatusOrMissing &status) { +inline std::ostream& operator<<(std::ostream& stream, const store::NodeStatusOrMissing& status) +{ fmt::print(stream, "{}", status); return stream; } diff --git a/src/terrainlib/store/OpenError.h b/src/terrainlib/store/OpenError.h index 2622b8a5..29b21bc2 100644 --- a/src/terrainlib/store/OpenError.h +++ b/src/terrainlib/store/OpenError.h @@ -10,15 +10,10 @@ namespace store { struct UnknownLayout { std::string id; - bool operator==(const UnknownLayout &) const = default; + bool operator==(const UnknownLayout&) const = default; }; -template -using OpenError = std::variant< - IndexFormatError, - FilesystemError, - UnknownLayout, - CodecError, - InvalidKey>; +template +using OpenError = std::variant>; } // namespace store diff --git a/src/terrainlib/store/PathMapping.h b/src/terrainlib/store/PathMapping.h index 0a2fe725..771f3381 100644 --- a/src/terrainlib/store/PathMapping.h +++ b/src/terrainlib/store/PathMapping.h @@ -7,13 +7,13 @@ namespace store { -template +template struct PathMapping { std::string_view id; - NodePath (*key_to_node_path)(const Key &); - std::optional (*node_path_to_key)(const NodePath &); + NodePath (*key_to_node_path)(const Key&); + std::optional (*node_path_to_key)(const NodePath&); - bool operator==(const PathMapping &) const = default; + bool operator==(const PathMapping&) const = default; }; } // namespace store diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index 9245c0dc..4b747fe8 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -16,51 +16,57 @@ namespace store { -template +template class RawStorage { public: using Key = typename Traits::Key; using value_type = NodeData; RawStorage(Layout layout, std::unique_ptr> codec) - : _layout(std::move(layout)), _codec(std::move(codec)) {} + : m_layout(std::move(layout)) + , m_codec(std::move(codec)) + { + } - RawStorage(const RawStorage &) = delete; - RawStorage &operator=(const RawStorage &) = delete; - RawStorage(RawStorage &&) noexcept = default; - RawStorage &operator=(RawStorage &&) noexcept = default; + RawStorage(const RawStorage&) = delete; + RawStorage& operator=(const RawStorage&) = delete; + RawStorage(RawStorage&&) noexcept = default; + RawStorage& operator=(RawStorage&&) noexcept = default; - std::expected> load(const Key &key) const { + std::expected> load(const Key& key) const + { if (!Traits::is_valid(key)) { - return std::unexpected(LoadError(InvalidKey{key})); + return std::unexpected(LoadError(InvalidKey { key })); } - const auto result = _codec->read(_layout.node_path(key)); + const auto result = m_codec->read(m_layout.node_path(key)); if (!result.has_value()) { return std::unexpected(LoadError(result.error())); } return std::move(*result); } - std::expected> save(const Key &key, const NodeData &data) const { + std::expected> save(const Key& key, const NodeData& data) const + { if (!Traits::is_valid(key)) { - return std::unexpected(SaveError(InvalidKey{key})); + return std::unexpected(SaveError(InvalidKey { key })); } - const auto result = _codec->write(_layout.node_path(key), data); + const auto result = m_codec->write(m_layout.node_path(key), data); if (!result.has_value()) { return std::unexpected(SaveError(result.error())); } return {}; } - std::expected, InvalidKey> paths( - const Key &key) const { + std::expected, InvalidKey> paths(const Key& key) const + { if (!Traits::is_valid(key)) { - return std::unexpected(InvalidKey{key}); + return std::unexpected(InvalidKey { key }); } - return _codec->paths(_layout.node_path(key)); + return m_codec->paths(m_layout.node_path(key)); } - std::expected> has(const Key &key) const { + std::expected> has(const Key& key) const + { const auto node_paths = paths(key); if (!node_paths.has_value()) { return std::unexpected(FileOperationError(node_paths.error())); @@ -68,11 +74,11 @@ class RawStorage { if (node_paths->empty()) { return false; } - for (const auto &path : node_paths.value()) { + for (const auto& path : node_paths.value()) { std::error_code error; const bool exists = std::filesystem::exists(path, error); if (error) { - return std::unexpected(FileOperationError(FilesystemError{ + return std::unexpected(FileOperationError(FilesystemError { path, "exists", error, @@ -85,17 +91,18 @@ class RawStorage { return true; } - std::expected> remove(const Key &key) const { + std::expected> remove(const Key& key) const + { const auto node_paths = paths(key); if (!node_paths.has_value()) { return std::unexpected(FileOperationError(node_paths.error())); } bool removed = false; - for (const auto &path : node_paths.value()) { + for (const auto& path : node_paths.value()) { std::error_code error; removed = std::filesystem::remove(path, error) || removed; if (error) { - return std::unexpected(FileOperationError(FilesystemError{ + return std::unexpected(FileOperationError(FilesystemError { path, "remove", error, @@ -105,35 +112,28 @@ class RawStorage { return removed; } - std::expected> copy_from( - const Key &key, - const RawStorage &source) const { - return copy_from(key, source, []() -> std::expected> { - return {}; - }); + std::expected> copy_from(const Key& key, const RawStorage& source) const + { + return copy_from(key, source, []() -> std::expected> { return {}; }); } - template - std::expected> copy_from( - const Key &key, - const RawStorage &source, - BeforeModify &&before_modify) const { + template + std::expected> copy_from(const Key& key, const RawStorage& source, BeforeModify&& before_modify) const + { if (!Traits::is_valid(key)) { - return std::unexpected(CopyError(InvalidKey{key})); + return std::unexpected(CopyError(InvalidKey { key })); } const auto source_exists = source.has(key); if (!source_exists.has_value()) { - return std::unexpected(std::visit( - [](const auto &error) -> CopyError { return error; }, - source_exists.error())); + return std::unexpected(std::visit([](const auto& error) -> CopyError { return error; }, source_exists.error())); } if (!source_exists.value()) { - return std::unexpected(CopyError(MissingSource{key})); + return std::unexpected(CopyError(MissingSource { key })); } const NodePath probe("__codec_probe__/node"); - if (_codec->paths(probe) != source._codec->paths(probe)) { - const auto loaded = source._codec->read(source._layout.node_path(key)); + if (m_codec->paths(probe) != source.m_codec->paths(probe)) { + const auto loaded = source.m_codec->read(source.m_layout.node_path(key)); if (!loaded.has_value()) { return std::unexpected(CopyError(loaded.error())); } @@ -141,17 +141,17 @@ class RawStorage { if (!prepared.has_value()) { return prepared; } - const auto written = _codec->write(_layout.node_path(key), loaded.value()); + const auto written = m_codec->write(m_layout.node_path(key), loaded.value()); if (!written.has_value()) { return std::unexpected(CopyError(written.error())); } return {}; } - const auto source_paths = source._codec->paths(source._layout.node_path(key)); - const auto target_paths = _codec->paths(_layout.node_path(key)); + const auto source_paths = source.m_codec->paths(source.m_layout.node_path(key)); + const auto target_paths = m_codec->paths(m_layout.node_path(key)); if (source_paths.size() != target_paths.size()) { - return std::unexpected(CopyError(CodecError{ + return std::unexpected(CopyError(CodecError { CodecOperation::Write, CodecErrorCategory::Domain, "matching codec probes produced different actual path counts", @@ -165,7 +165,7 @@ class RawStorage { std::error_code error; std::filesystem::remove(target_paths[index], error); if (error) { - return std::unexpected(CopyError(FilesystemError{ + return std::unexpected(CopyError(FilesystemError { target_paths[index], "remove", error, @@ -173,7 +173,7 @@ class RawStorage { } std::filesystem::create_directories(target_paths[index].parent_path(), error); if (error) { - return std::unexpected(CopyError(FilesystemError{ + return std::unexpected(CopyError(FilesystemError { target_paths[index].parent_path(), "create_directories", error, @@ -181,7 +181,7 @@ class RawStorage { } std::filesystem::create_hard_link(source_paths[index], target_paths[index], error); if (error) { - return std::unexpected(CopyError(FilesystemError{ + return std::unexpected(CopyError(FilesystemError { target_paths[index], "create_hard_link", error, @@ -191,16 +191,12 @@ class RawStorage { return {}; } - const Layout &layout() const { - return _layout; - } - const Codec &codec() const { - return *_codec; - } + const Layout& layout() const { return m_layout; } + const Codec& codec() const { return *m_codec; } private: - Layout _layout; - std::unique_ptr> _codec; + Layout m_layout; + std::unique_ptr> m_codec; }; } // namespace store diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index e043d9a5..7df1628d 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -15,7 +15,7 @@ namespace store { -template +template class Storage { public: using Key = typename Traits::Key; @@ -32,298 +32,273 @@ class Storage { std::string codec_selector; }; - explicit Storage(RawStorage raw) : _raw(std::move(raw)) {} - - Storage( - RawStorage raw, - Index index, - Persistence persistence, - const bool dirty = false) - : _raw(std::move(raw)), - _index(std::move(index)), - _persistence(std::move(persistence)), - _dirty(dirty) {} + explicit Storage(RawStorage raw) + : m_raw(std::move(raw)) + { + } - virtual ~Storage() { - finalize_displaced_state(); + Storage(RawStorage raw, Index index, Persistence persistence, const bool dirty = false) + : m_raw(std::move(raw)) + , m_index(std::move(index)) + , m_persistence(std::move(persistence)) + , m_dirty(dirty) + { } - Storage(const Storage &) = delete; - Storage &operator=(const Storage &) = delete; + virtual ~Storage() { finalize_displaced_state(); } + + Storage(const Storage&) = delete; + Storage& operator=(const Storage&) = delete; - Storage(Storage &&other) noexcept - : _raw(std::move(other._raw)), - _index(std::move(other._index)), - _persistence(std::move(other._persistence)), - _settings(other._settings), - _dirty(std::exchange(other._dirty, false)) { - other._index.reset(); - other._persistence.reset(); + Storage(Storage&& other) noexcept + : m_raw(std::move(other.m_raw)) + , m_index(std::move(other.m_index)) + , m_persistence(std::move(other.m_persistence)) + , m_settings(other.m_settings) + , m_dirty(std::exchange(other.m_dirty, false)) + { + other.m_index.reset(); + other.m_persistence.reset(); } - Storage &operator=(Storage &&other) noexcept { + Storage& operator=(Storage&& other) noexcept + { if (this == &other) { return *this; } finalize_displaced_state(); - _raw = std::move(other._raw); - _index = std::move(other._index); - _persistence = std::move(other._persistence); - _settings = other._settings; - _dirty = std::exchange(other._dirty, false); - other._index.reset(); - other._persistence.reset(); + m_raw = std::move(other.m_raw); + m_index = std::move(other.m_index); + m_persistence = std::move(other.m_persistence); + m_settings = other.m_settings; + m_dirty = std::exchange(other.m_dirty, false); + other.m_index.reset(); + other.m_persistence.reset(); return *this; } - std::expected> load(const Key &key) const { + std::expected> load(const Key& key) const + { if (!Traits::is_valid(key)) { - return std::unexpected(LoadError(InvalidKey{key})); + return std::unexpected(LoadError(InvalidKey { key })); } - if (_index.has_value()) { - const auto status = _index->get(key); + if (m_index.has_value()) { + const auto status = m_index->get(key); if (!status.has_value()) { return std::unexpected(LoadError(status.error())); } - if (!status->has_value() - || status->value() == NodeStatus::Virtual) { - return std::unexpected(LoadError(CodecError{ + if (!status->has_value() || status->value() == NodeStatus::Virtual) { + return std::unexpected(LoadError(CodecError { CodecOperation::Read, CodecErrorCategory::FileNotFound, "node is not physically present in the index", })); } } - return _raw.load(key); + return m_raw.load(key); } - std::expected> save(const Key &key, const NodeData &data) { + std::expected> save(const Key& key, const NodeData& data) + { if (!Traits::is_valid(key)) { - return std::unexpected(SaveError(InvalidKey{key})); + return std::unexpected(SaveError(InvalidKey { key })); } const auto exists = has(key); if (!exists.has_value()) { - return std::unexpected(std::visit( - [](const auto &error) -> SaveError { return error; }, - exists.error())); + return std::unexpected(std::visit([](const auto& error) -> SaveError { return error; }, exists.error())); } - if (exists.value() && !_settings.allow_overwrite) { - const auto node_paths = _raw.paths(key).value(); - return std::unexpected(SaveError(AlreadyExists{ - node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), + if (exists.value() && !m_settings.allow_overwrite) { + const auto node_paths = m_raw.paths(key).value(); + return std::unexpected(SaveError(AlreadyExists { + node_paths.empty() ? m_raw.layout().node_path(key).path() : node_paths.front(), })); } - const auto result = _raw.save(key, data); + const auto result = m_raw.save(key, data); if (!result.has_value()) { return result; } - if (_index.has_value()) { - const auto added = _index->add(key); + if (m_index.has_value()) { + const auto added = m_index->add(key); if (!added.has_value()) { return std::unexpected(SaveError(added.error())); } - _dirty = _dirty || added.value(); + m_dirty = m_dirty || added.value(); } return {}; } - std::expected> copy_from( - const Key &key, - const Storage &source) { + std::expected> copy_from(const Key& key, const Storage& source) + { if (!Traits::is_valid(key)) { - return std::unexpected(CopyError(InvalidKey{key})); + return std::unexpected(CopyError(InvalidKey { key })); } const auto exists = has(key); if (!exists.has_value()) { - return std::unexpected(std::visit( - [](const auto &error) -> CopyError { return error; }, - exists.error())); + return std::unexpected(std::visit([](const auto& error) -> CopyError { return error; }, exists.error())); } - if (exists.value() && !_settings.allow_overwrite) { - const auto node_paths = _raw.paths(key).value(); - return std::unexpected(CopyError(AlreadyExists{ - node_paths.empty() ? _raw.layout().node_path(key).path() : node_paths.front(), + if (exists.value() && !m_settings.allow_overwrite) { + const auto node_paths = m_raw.paths(key).value(); + return std::unexpected(CopyError(AlreadyExists { + node_paths.empty() ? m_raw.layout().node_path(key).path() : node_paths.front(), })); } const auto prepare_target = [&]() -> std::expected> { - if (!exists.value() || !_index.has_value()) { + if (!exists.value() || !m_index.has_value()) { return {}; } - const auto removed = _index->remove(key); + const auto removed = m_index->remove(key); if (!removed.has_value()) { return std::unexpected(CopyError(removed.error())); } - _dirty = _dirty || removed.value(); + m_dirty = m_dirty || removed.value(); return {}; }; - const auto result = _raw.copy_from(key, source._raw, prepare_target); + const auto result = m_raw.copy_from(key, source.m_raw, prepare_target); if (!result.has_value()) { return result; } - if (_index.has_value()) { - const auto added = _index->add(key); + if (m_index.has_value()) { + const auto added = m_index->add(key); if (!added.has_value()) { return std::unexpected(CopyError(added.error())); } - _dirty = _dirty || added.value(); + m_dirty = m_dirty || added.value(); } return {}; } - std::expected> remove(const Key &key) { - const auto removed = _raw.remove(key); + std::expected> remove(const Key& key) + { + const auto removed = m_raw.remove(key); if (!removed.has_value()) { return removed; } - if (_index.has_value()) { - const auto index_removed = _index->remove(key); + if (m_index.has_value()) { + const auto index_removed = m_index->remove(key); if (!index_removed.has_value()) { return std::unexpected(FileOperationError(index_removed.error())); } - _dirty = _dirty || index_removed.value(); + m_dirty = m_dirty || index_removed.value(); } return removed.value(); } - std::expected> has(const Key &key) const { + std::expected> has(const Key& key) const + { if (!Traits::is_valid(key)) { - return std::unexpected(FileOperationError(InvalidKey{key})); + return std::unexpected(FileOperationError(InvalidKey { key })); } - if (!_index.has_value()) { - return _raw.has(key); + if (!m_index.has_value()) { + return m_raw.has(key); } - const auto status = _index->get(key); + const auto status = m_index->get(key); if (!status.has_value()) { return std::unexpected(FileOperationError(status.error())); } return status->has_value() && status->value() != NodeStatus::Virtual; } - std::expected, InvalidKey> paths( - const Key &key) const { - return _raw.paths(key); - } + std::expected, InvalidKey> paths(const Key& key) const { return m_raw.paths(key); } - std::expected> path_for( - const Key &key) const { + std::expected> path_for(const Key& key) const + { const auto node_paths = paths(key); if (!node_paths.has_value()) { return std::unexpected(node_paths.error()); } - return node_paths->empty() - ? _raw.layout().node_path(key).path() - : node_paths->front(); + return node_paths->empty() ? m_raw.layout().node_path(key).path() : node_paths->front(); } - const std::filesystem::path &base_path() const { - return _raw.layout().base_path(); - } - const Layout &layout() const { - return _raw.layout(); - } - const Codec &codec() const { - return _raw.codec(); - } - std::optional codec_selector() const { - if (!_persistence.has_value()) { + const std::filesystem::path& base_path() const { return m_raw.layout().base_path(); } + const Layout& layout() const { return m_raw.layout(); } + const Codec& codec() const { return m_raw.codec(); } + std::optional codec_selector() const + { + if (!m_persistence.has_value()) { return std::nullopt; } - return _persistence->codec_selector; + return m_persistence->codec_selector; } - bool is_indexed() const { - return _index.has_value(); - } - void ensure_indexed() { - if (!_index.has_value()) { - _index.emplace(); - _dirty = true; + bool is_indexed() const { return m_index.has_value(); } + void ensure_indexed() + { + if (!m_index.has_value()) { + m_index.emplace(); + m_dirty = true; } } - std::optional>> index() const { - if (!_index.has_value()) { + std::optional>> index() const + { + if (!m_index.has_value()) { return std::nullopt; } - return std::cref(_index.value()); + return std::cref(m_index.value()); } - std::expected save_index() const { - if (!_dirty) { + std::expected save_index() const + { + if (!m_dirty) { return {}; } - if (!_index.has_value() || !_persistence.has_value()) { - return std::unexpected(IndexFormatError{ + if (!m_index.has_value() || !m_persistence.has_value()) { + return std::unexpected(IndexFormatError { IndexFormatErrorCategory::Write, {}, "indexed storage has no persistence configuration", }); } - const IndexMetadata metadata{ - _index.value(), - _persistence->layout_id, - _persistence->payload_class, - _persistence->codec_selector, + const IndexMetadata metadata { + m_index.value(), + m_persistence->layout_id, + m_persistence->payload_class, + m_persistence->codec_selector, }; - const auto result = _persistence->format.write( - _persistence->index_path, - metadata); + const auto result = m_persistence->format.write(m_persistence->index_path, metadata); if (result.has_value()) { - _dirty = false; + m_dirty = false; } return result; } - std::expected save_or_create_index() { - if (!_index.has_value()) { - _index.emplace(); - _dirty = true; + std::expected save_or_create_index() + { + if (!m_index.has_value()) { + m_index.emplace(); + m_dirty = true; } return save_index(); } - const StorageSettings &settings() const { - return _settings; - } - StorageSettings &settings() { - return _settings; - } + const StorageSettings& settings() const { return m_settings; } + StorageSettings& settings() { return m_settings; } protected: - Index &index_mut() { - return _index.value(); - } - const Index &index_ref() const { - return _index.value(); - } - bool is_index_dirty() const { - return _dirty; - } - void set_index_dirty(const bool dirty = true) const { - _dirty = dirty; - } + Index& index_mut() { return m_index.value(); } + const Index& index_ref() const { return m_index.value(); } + bool is_index_dirty() const { return m_dirty; } + void set_index_dirty(const bool dirty = true) const { m_dirty = dirty; } private: - void finalize_displaced_state() noexcept { - if (!_dirty || !_index.has_value()) { + void finalize_displaced_state() noexcept + { + if (!m_dirty || !m_index.has_value()) { return; } const auto result = save_index(); if (!result.has_value()) { - LOG_ERROR( - "Failed to automatically save index {}: {}", - result.error().path, - result.error().message); + LOG_ERROR("Failed to automatically save index {}: {}", result.error().path, result.error().message); } } - RawStorage _raw; - std::optional> _index; - std::optional _persistence; - StorageSettings _settings; - mutable bool _dirty = false; + RawStorage m_raw; + std::optional> m_index; + std::optional m_persistence; + StorageSettings m_settings; + mutable bool m_dirty = false; }; } // namespace store diff --git a/src/terrainlib/store/StorageError.h b/src/terrainlib/store/StorageError.h index ffd6d69b..a641456c 100644 --- a/src/terrainlib/store/StorageError.h +++ b/src/terrainlib/store/StorageError.h @@ -12,41 +12,32 @@ namespace store { struct AlreadyExists { std::filesystem::path path; - bool operator==(const AlreadyExists &) const = default; + bool operator==(const AlreadyExists&) const = default; }; struct FilesystemError { std::filesystem::path path; std::string operation; std::error_code error; - bool operator==(const FilesystemError &) const = default; + bool operator==(const FilesystemError&) const = default; }; -template +template struct MissingSource { Key key; - bool operator==(const MissingSource &) const = default; + bool operator==(const MissingSource&) const = default; }; -template +template using LoadError = std::variant, CodecError>; -template -using SaveError = std::variant< - InvalidKey, - CodecError, - AlreadyExists, - FilesystemError>; +template +using SaveError = std::variant, CodecError, AlreadyExists, FilesystemError>; -template +template using FileOperationError = std::variant, FilesystemError>; -template -using CopyError = std::variant< - InvalidKey, - MissingSource, - AlreadyExists, - FilesystemError, - CodecError>; +template +using CopyError = std::variant, MissingSource, AlreadyExists, FilesystemError, CodecError>; } // namespace store diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h index 80fe88a7..d0b8e7f1 100644 --- a/src/terrainlib/store/Traits.h +++ b/src/terrainlib/store/Traits.h @@ -4,7 +4,7 @@ namespace store { -template +template concept HierarchyTraits = requires(typename Traits::Key key) { typename Traits::Key; typename Traits::Hasher; diff --git a/src/terrainlib/store/cache/Dummy.h b/src/terrainlib/store/cache/Dummy.h index bcb7c7e7..ab7c2990 100644 --- a/src/terrainlib/store/cache/Dummy.h +++ b/src/terrainlib/store/cache/Dummy.h @@ -4,22 +4,14 @@ namespace store::cache { -template +template class Dummy final : public Interface { public: using Key = typename Traits::Key; - std::optional get(const Key &) noexcept override { - return std::nullopt; - } - bool put(const Key &, const NodeData &) noexcept override { - return false; - } - bool remove(const Key &) noexcept override { - return false; - } - bool contains(const Key &) const noexcept override { - return false; - } + std::optional get(const Key&) noexcept override { return std::nullopt; } + bool put(const Key&, const NodeData&) noexcept override { return false; } + bool remove(const Key&) noexcept override { return false; } + bool contains(const Key&) const noexcept override { return false; } }; } // namespace store::cache diff --git a/src/terrainlib/store/cache/Interface.h b/src/terrainlib/store/cache/Interface.h index 4ffd070d..ea42a4f2 100644 --- a/src/terrainlib/store/cache/Interface.h +++ b/src/terrainlib/store/cache/Interface.h @@ -6,16 +6,16 @@ namespace store::cache { -template +template class Interface { public: using Key = typename Traits::Key; virtual ~Interface() = default; - virtual std::optional get(const Key &key) noexcept = 0; - virtual bool put(const Key &key, const NodeData &value) noexcept = 0; - virtual bool remove(const Key &key) noexcept = 0; - virtual bool contains(const Key &key) const noexcept = 0; + virtual std::optional get(const Key& key) noexcept = 0; + virtual bool put(const Key& key, const NodeData& value) noexcept = 0; + virtual bool remove(const Key& key) noexcept = 0; + virtual bool contains(const Key& key) const noexcept = 0; }; } // namespace store::cache diff --git a/src/terrainlib/store/cache/Lru.h b/src/terrainlib/store/cache/Lru.h index 0e8d8abf..1eb8f0a4 100644 --- a/src/terrainlib/store/cache/Lru.h +++ b/src/terrainlib/store/cache/Lru.h @@ -8,66 +8,67 @@ namespace store::cache { -template +template class Lru final : public Interface { public: using Key = typename Traits::Key; - explicit Lru(const size_t capacity) : _capacity(capacity) {} + explicit Lru(const size_t capacity) + : m_capacity(capacity) + { + } - std::optional get(const Key &key) noexcept override { - const auto iterator = _values.find(key); - if (iterator == _values.end()) { + std::optional get(const Key& key) noexcept override + { + const auto iterator = m_values.find(key); + if (iterator == m_values.end()) { return std::nullopt; } - _usage.splice(_usage.begin(), _usage, iterator->second.second); + m_usage.splice(m_usage.begin(), m_usage, iterator->second.second); return iterator->second.first; } - bool put(const Key &key, const NodeData &value) noexcept override { - const auto iterator = _values.find(key); - if (iterator != _values.end()) { + bool put(const Key& key, const NodeData& value) noexcept override + { + const auto iterator = m_values.find(key); + if (iterator != m_values.end()) { iterator->second.first = value; - _usage.splice(_usage.begin(), _usage, iterator->second.second); + m_usage.splice(m_usage.begin(), m_usage, iterator->second.second); return false; } - if (_capacity == 0) { + if (m_capacity == 0) { return false; } - if (_values.size() == _capacity) { - _values.erase(_usage.back()); - _usage.pop_back(); + if (m_values.size() == m_capacity) { + m_values.erase(m_usage.back()); + m_usage.pop_back(); } - _usage.push_front(key); - _values.emplace(key, std::pair::iterator>{ - value, - _usage.begin(), - }); + m_usage.push_front(key); + m_values.emplace(key, + std::pair::iterator> { + value, + m_usage.begin(), + }); return true; } - bool remove(const Key &key) noexcept override { - const auto iterator = _values.find(key); - if (iterator == _values.end()) { + bool remove(const Key& key) noexcept override + { + const auto iterator = m_values.find(key); + if (iterator == m_values.end()) { return false; } - _usage.erase(iterator->second.second); - _values.erase(iterator); + m_usage.erase(iterator->second.second); + m_values.erase(iterator); return true; } - bool contains(const Key &key) const noexcept override { - return _values.contains(key); - } + bool contains(const Key& key) const noexcept override { return m_values.contains(key); } private: - size_t _capacity; - std::list _usage; - std::unordered_map< - Key, - std::pair::iterator>, - typename Traits::Hasher> - _values; + size_t m_capacity; + std::list m_usage; + std::unordered_map::iterator>, typename Traits::Hasher> m_values; }; } // namespace store::cache diff --git a/src/terrainlib/store/codec/Path.h b/src/terrainlib/store/codec/Path.h index 4715291b..81529800 100644 --- a/src/terrainlib/store/codec/Path.h +++ b/src/terrainlib/store/codec/Path.h @@ -7,9 +7,7 @@ namespace store::codec { -inline std::filesystem::path append_extension( - const NodePath &node_path, - const std::string_view extension) +inline std::filesystem::path append_extension(const NodePath& node_path, const std::string_view extension) { std::filesystem::path result = node_path.path(); result += extension; diff --git a/src/terrainlib/store/describe_error.h b/src/terrainlib/store/describe_error.h index aa61d2a1..f280feed 100644 --- a/src/terrainlib/store/describe_error.h +++ b/src/terrainlib/store/describe_error.h @@ -8,43 +8,32 @@ namespace store { -inline std::string describe_error(const CodecError &error) { - return error.message; -} +inline std::string describe_error(const CodecError& error) { return error.message; } -inline std::string describe_error(const IndexFormatError &error) { - return error.path.empty() - ? error.message - : error.path.string() + ": " + error.message; -} +inline std::string describe_error(const IndexFormatError& error) { return error.path.empty() ? error.message : error.path.string() + ": " + error.message; } -inline std::string describe_error(const FilesystemError &error) { - return error.operation + " " + error.path.string() + ": " + error.error.message(); -} +inline std::string describe_error(const FilesystemError& error) { return error.operation + " " + error.path.string() + ": " + error.error.message(); } -inline std::string describe_error(const UnknownLayout &error) { - return "unknown layout: " + error.id; -} +inline std::string describe_error(const UnknownLayout& error) { return "unknown layout: " + error.id; } -inline std::string describe_error(const AlreadyExists &error) { - return "path already exists: " + error.path.string(); -} +inline std::string describe_error(const AlreadyExists& error) { return "path already exists: " + error.path.string(); } -template -std::string describe_error(const InvalidKey &) { +template +std::string describe_error(const InvalidKey&) +{ return "invalid hierarchy key"; } -template -std::string describe_error(const MissingSource &) { +template +std::string describe_error(const MissingSource&) +{ return "source node is missing"; } -template -std::string describe_error(const std::variant &error) { - return std::visit( - [](const auto &value) { return describe_error(value); }, - error); +template +std::string describe_error(const std::variant& error) +{ + return std::visit([](const auto& value) { return describe_error(value); }, error); } } // namespace store diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h index 5bf36352..b26a65ef 100644 --- a/src/terrainlib/store/open.h +++ b/src/terrainlib/store/open.h @@ -11,12 +11,10 @@ namespace store { -template +template std::expected, OpenError> open_index( - const std::filesystem::path &index_path, - const IndexFormat format, - const std::string_view expected_payload_class, - CodecResolver &&resolve_codec) { + const std::filesystem::path& index_path, const IndexFormat format, const std::string_view expected_payload_class, CodecResolver&& resolve_codec) +{ using Key = typename Traits::Key; const auto metadata_result = format.read(index_path); if (!metadata_result.has_value()) { @@ -24,22 +22,22 @@ std::expected, OpenError> } IndexMetadata metadata = std::move(metadata_result.value()); if (metadata.payload_class != expected_payload_class) { - return std::unexpected(OpenError(CodecError{ + return std::unexpected(OpenError(CodecError { CodecOperation::Resolve, CodecErrorCategory::UnsupportedCodec, "unexpected payload class: " + metadata.payload_class, })); } - for (const auto &[key, status] : metadata.index) { + for (const auto& [key, status] : metadata.index) { static_cast(status); if (!Traits::is_valid(key)) { - return std::unexpected(OpenError(InvalidKey{key})); + return std::unexpected(OpenError(InvalidKey { key })); } } const auto mapping = format.mapping_from_id(metadata.layout_id); if (!mapping.has_value()) { - return std::unexpected(OpenError(UnknownLayout{metadata.layout_id})); + return std::unexpected(OpenError(UnknownLayout { metadata.layout_id })); } auto codec = resolve_codec(metadata.codec_selector); if (!codec.has_value()) { @@ -48,12 +46,9 @@ std::expected, OpenError> const std::filesystem::path base_path = index_path.parent_path(); using Storage = store::Storage; - return IndexedStorage( - RawStorage( - Layout(base_path, mapping.value()), - std::move(codec.value())), + return IndexedStorage(RawStorage(Layout(base_path, mapping.value()), std::move(codec.value())), std::move(metadata.index), - typename Storage::Persistence{ + typename Storage::Persistence { format, index_path, std::move(metadata.layout_id), @@ -62,40 +57,34 @@ std::expected, OpenError> }); } -template -std::expected, OpenError> make_storage( - const std::filesystem::path &base_path, +template +std::expected, OpenError> make_storage(const std::filesystem::path& base_path, const IndexFormat format, const PathMapping mapping, std::string payload_class, std::string codec_selector, - std::unique_ptr> codec) { + std::unique_ptr> codec) +{ using Key = typename Traits::Key; std::error_code error; std::filesystem::create_directories(base_path, error); if (error) { - return std::unexpected(OpenError(FilesystemError{ + return std::unexpected(OpenError(FilesystemError { base_path, "create_directories", error, })); } - typename Storage::Persistence persistence{ + typename Storage::Persistence persistence { format, base_path / format.index_filename, std::string(mapping.id), std::move(payload_class), std::move(codec_selector), }; - RawStorage raw( - Layout(base_path, mapping), - std::move(codec)); - return Storage( - std::move(raw), - Index{}, - std::move(persistence), - true); + RawStorage raw(Layout(base_path, mapping), std::move(codec)); + return Storage(std::move(raw), Index {}, std::move(persistence), true); } } // namespace store diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h index 8dbfe948..811988ad 100644 --- a/src/terrainlib/store/traverse.h +++ b/src/terrainlib/store/traverse.h @@ -14,24 +14,25 @@ enum class TraversalOrder { }; struct AlwaysRefine { - template - constexpr bool operator()(const Key &) const { + template + constexpr bool operator()(const Key&) const + { return true; } }; -template -std::expected> traverse( - const Index &index, - VisitFn &&visit, - RefineFn &&refine = {}, - const typename Traits::Key &root = Traits::root(), - const TraversalOrder order = TraversalOrder::DepthFirst) { +template +std::expected> traverse(const Index& index, + VisitFn&& visit, + RefineFn&& refine = {}, + const typename Traits::Key& root = Traits::root(), + const TraversalOrder order = TraversalOrder::DepthFirst) +{ using Key = typename Traits::Key; using Error = InvalidKey; if (!Traits::is_valid(root)) { - return std::unexpected(Error{root}); + return std::unexpected(Error { root }); } const auto root_status = index.get(root); if (!root_status.has_value()) { @@ -42,8 +43,8 @@ std::expected> traverse( } if (order == TraversalOrder::DepthFirst) { - std::function(const Key &)> depth_first; - depth_first = [&](const Key ¤t) -> std::expected { + std::function(const Key&)> depth_first; + depth_first = [&](const Key& current) -> std::expected { const auto status = index.get(current); if (!status.has_value()) { return std::unexpected(status.error()); @@ -56,7 +57,7 @@ std::expected> traverse( if (refine(current)) { const auto children = Traits::children(current); if (children.has_value()) { - for (const Key &child : children.value()) { + for (const Key& child : children.value()) { const auto result = depth_first(child); if (!result.has_value()) { return result; @@ -87,7 +88,7 @@ std::expected> traverse( if (refine(current)) { const auto children = Traits::children(current); if (children.has_value()) { - for (const Key &child : children.value()) { + for (const Key& child : children.value()) { queue.push(child); } } diff --git a/src/terrainlib/tile_path.h b/src/terrainlib/tile_path.h index 8f866d80..e669057e 100644 --- a/src/terrainlib/tile_path.h +++ b/src/terrainlib/tile_path.h @@ -6,9 +6,7 @@ #include [[nodiscard]] inline std::filesystem::path google_tile_path( - const std::filesystem::path& base_path, - const radix::tile::Id& tile_id, - const std::string& extension) + const std::filesystem::path& base_path, const radix::tile::Id& tile_id, const std::string& extension) { return base_path / std::to_string(tile_id.zoom_level) / std::to_string(tile_id.coords.x) / (std::to_string(tile_id.coords.y) + extension); } diff --git a/src/tile_builder/alpine_raster.cpp b/src/tile_builder/alpine_raster.cpp index c5d8a6b0..f1e387d9 100644 --- a/src/tile_builder/alpine_raster.cpp +++ b/src/tile_builder/alpine_raster.cpp @@ -38,5 +38,5 @@ ParallelTileGenerator alpine_raster::make_generator(const std::string& input_dat void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const radix::Raster& heights) const { - image::saveImageAsPng(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); + image::save_image_as_png(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); } diff --git a/src/tile_builder/image_writer.cpp b/src/tile_builder/image_writer.cpp index b3135f90..91ee199c 100644 --- a/src/tile_builder/image_writer.cpp +++ b/src/tile_builder/image_writer.cpp @@ -25,7 +25,7 @@ #include #include -void image::saveImageAsPng(const radix::Raster& input_image, const std::string& path) +void image::save_image_as_png(const radix::Raster& input_image, const std::string& path) { auto converted = io::conversion::to_mat(input_image); if (!converted) { diff --git a/src/tile_builder/image_writer.h b/src/tile_builder/image_writer.h index db8ff8f9..37d697da 100644 --- a/src/tile_builder/image_writer.h +++ b/src/tile_builder/image_writer.h @@ -31,20 +31,21 @@ namespace image { -void saveImageAsPng(const radix::Raster& image, const std::string& path); +void save_image_as_png(const radix::Raster& image, const std::string& path); template -void debugOut(const radix::Raster& image, const std::string& path) +void debug_out(const radix::Raster& image, const std::string& path) { if (image.buffer().empty()) throw std::invalid_argument("Can't write an empty raster to " + path); const auto [min, max] = std::ranges::minmax(image); const auto range = float(max) - float(min); - saveImageAsPng(radix::raster::transform(image, [min, range](const auto value) { - const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); - return glm::u8vec3(intensity); - }), + save_image_as_png(radix::raster::transform(image, + [min, range](const auto value) { + const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); + return glm::u8vec3(intensity); + }), path); } diff --git a/src/tile_downloader/write_file.h b/src/tile_downloader/write_file.h index ab2d07f2..b74e7ebe 100644 --- a/src/tile_downloader/write_file.h +++ b/src/tile_downloader/write_file.h @@ -9,7 +9,7 @@ namespace tile_downloader_detail { -inline void write_file_checked_direct(const std::filesystem::path &path, const std::vector &data) +inline void write_file_checked_direct(const std::filesystem::path& path, const std::vector& data) { std::ofstream output(path, std::ios::binary); if (!output) { @@ -27,23 +27,23 @@ inline void write_file_checked_direct(const std::filesystem::path &path, const s } } -} +} // namespace tile_downloader_detail -[[nodiscard]] inline std::filesystem::path partial_tile_path(const std::filesystem::path &path) +[[nodiscard]] inline std::filesystem::path partial_tile_path(const std::filesystem::path& path) { auto partial_path = path; partial_path += ".part"; return partial_path; } -[[nodiscard]] inline std::filesystem::path children_pending_tile_path(const std::filesystem::path &path) +[[nodiscard]] inline std::filesystem::path children_pending_tile_path(const std::filesystem::path& path) { auto pending_path = path; pending_path += ".children-pending"; return pending_path; } -inline void write_file_children_pending(const std::filesystem::path &path, const std::vector &data) +inline void write_file_children_pending(const std::filesystem::path& path, const std::vector& data) { const auto partial_path = partial_tile_path(path); const auto pending_path = children_pending_tile_path(path); @@ -58,7 +58,7 @@ inline void write_file_children_pending(const std::filesystem::path &path, const } } -inline void mark_tile_children_complete(const std::filesystem::path &path) +inline void mark_tile_children_complete(const std::filesystem::path& path) { const auto pending_path = children_pending_tile_path(path); std::filesystem::rename(pending_path, path); diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 22d6a704..00ea1317 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -18,59 +18,55 @@ namespace { class TemporaryDirectory { public: - TemporaryDirectory() { + TemporaryDirectory() + { static std::atomic_uint64_t counter = 0; const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - _path = std::filesystem::temp_directory_path() - / ("atb-dag-codec-" + std::to_string(timestamp) + "-" - + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(_path)); + m_path = std::filesystem::temp_directory_path() / ("atb-dag-codec-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(m_path)); } - ~TemporaryDirectory() { + ~TemporaryDirectory() + { std::error_code error; - std::filesystem::remove_all(_path, error); - } - const std::filesystem::path &path() const { - return _path; + std::filesystem::remove_all(m_path, error); } + const std::filesystem::path& path() const { return m_path; } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -dag::ClusterBatch sample_batch() { +dag::ClusterBatch sample_batch() +{ dag::ClusterBatch batch; - batch.metadata.group_assignment = {0}; - batch.metadata.groups = {{ - .children = {{octree::Id::root().child(4).value(), 3}}, + batch.metadata.group_assignment = { 0 }; + batch.metadata.groups = { { + .children = { { octree::Id::root().child(4).value(), 3 } }, .error = 12.5, .bounds = {}, - }}; + } }; batch.clustering.positions = { - {0.0, 0.0, 0.0}, - {1.0, 0.0, 0.0}, - {0.0, 1.0, 0.0}, + { 0.0, 0.0, 0.0 }, + { 1.0, 0.0, 0.0 }, + { 0.0, 1.0, 0.0 }, }; - batch.clustering.clusters = {{ + batch.clustering.clusters = { { .id = 17, - .vertex_indices = {0, 1, 2}, - .local_triangles = {{0, 1, 2}}, + .vertex_indices = { 0, 1, 2 }, + .local_triangles = { { 0, 1, 2 } }, .uvs = {}, .texture_id = std::nullopt, .absolute_error = 0.0, - }}; + } }; return batch; } -mesh::Simple sample_mesh() { - return mesh::Simple( - {{0, 1, 2}}, - {{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}}); -} +mesh::Simple sample_mesh() { return mesh::Simple({ { 0, 1, 2 } }, { { 0.0, 0.0, 0.0 }, { 1.0, 0.0, 0.0 }, { 0.0, 1.0, 0.0 } }); } } // namespace -TEST_CASE("runtime DAG envelope codec is reentrant") { +TEST_CASE("runtime DAG envelope codec is reentrant") +{ const dag::ClusterBatch batch = sample_batch(); TemporaryDirectory output; @@ -83,23 +79,23 @@ TEST_CASE("runtime DAG envelope codec is reentrant") { return false; } const auto loaded = codec.read(path); - return loaded.has_value() - && loaded->metadata.group_assignment == batch.metadata.group_assignment; + return loaded.has_value() && loaded->metadata.group_assignment == batch.metadata.group_assignment; })); } - for (auto &operation : operations) { + for (auto& operation : operations) { REQUIRE(operation.get()); } } -TEST_CASE("versioned DAG payload validation is free-standing") { - dag::format::NodeMetadata invalid_metadata{ - .group_assignment = {0}, +TEST_CASE("versioned DAG payload validation is free-standing") +{ + dag::format::NodeMetadata invalid_metadata { + .group_assignment = { 0 }, .groups = {}, }; CHECK_FALSE(dag::format::validate(invalid_metadata).has_value()); - dag::format::Clustering oversized_clustering{ + dag::format::Clustering oversized_clustering { .vertex_count = std::numeric_limits::max(), .encoded_positions = {}, .clusters = {}, @@ -108,19 +104,16 @@ TEST_CASE("versioned DAG payload validation is free-standing") { CHECK_FALSE(dag::format::validate(oversized_clustering).has_value()); } -TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") { +TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") +{ const auto batch_codec = dag::codec::from_extension(".dag"); REQUIRE(batch_codec.has_value()); - CHECK(batch_codec.value()->paths(store::NodePath("node")) - == std::vector{"node.dag", "node.dagmeta"}); + CHECK(batch_codec.value()->paths(store::NodePath("node")) == std::vector { "node.dag", "node.dagmeta" }); const auto metadata_codec = dag::codec::metadata_from_extension(".dag"); REQUIRE(metadata_codec.has_value()); - CHECK(metadata_codec.value()->paths(store::NodePath("node")) - == std::vector{"node.dagmeta"}); - const auto write_result = metadata_codec.value()->write( - store::NodePath("node"), - dag::NodeMetadata{}); + CHECK(metadata_codec.value()->paths(store::NodePath("node")) == std::vector { "node.dagmeta" }); + const auto write_result = metadata_codec.value()->write(store::NodePath("node"), dag::NodeMetadata {}); REQUIRE_FALSE(write_result.has_value()); CHECK(write_result.error().operation == store::CodecOperation::Write); CHECK(write_result.error().category == store::CodecErrorCategory::UnsupportedOperation); @@ -130,7 +123,8 @@ TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[stor CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); } -TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") { +TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") +{ const dag::ClusterBatch batch = sample_batch(); TemporaryDirectory output; @@ -161,7 +155,8 @@ TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[s CHECK(metadata->groups.front().error == batch.metadata.groups.front().error); } -TEST_CASE("DAG storage hard-links both files for the same format", "[store][copy]") { +TEST_CASE("DAG storage hard-links both files for the same format", "[store][copy]") +{ TemporaryDirectory source_directory; TemporaryDirectory target_directory; auto source_result = dag::storage::open_folder_indexed(source_directory.path()); @@ -184,7 +179,8 @@ TEST_CASE("DAG storage hard-links both files for the same format", "[store][copy CHECK(std::filesystem::equivalent((*source_paths)[1], (*target_paths)[1])); } -TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") { +TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") +{ TemporaryDirectory input_directory; TemporaryDirectory output_directory; auto input_result = octree::open_folder_indexed(input_directory.path()); @@ -199,7 +195,7 @@ TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf auto output_result = dag::storage::open_folder_indexed(output_directory.path()); REQUIRE(output_result.has_value()); auto output = std::move(output_result.value()); - const dag::BuildOptions options{ + const dag::BuildOptions options { .clusters_per_partition = 1, .target_ratio = 1.0f, .relative_target_error = std::nullopt, diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp index d864949a..8c92a884 100644 --- a/unittests/sf_builder/storage_finalization.cpp +++ b/unittests/sf_builder/storage_finalization.cpp @@ -11,37 +11,32 @@ namespace { class TemporaryDirectory { public: - TemporaryDirectory() { + TemporaryDirectory() + { static std::atomic_uint64_t counter = 0; const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - _path = std::filesystem::temp_directory_path() - / ("atb-sfbuilder-finalize-" + std::to_string(timestamp) + "-" - + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(_path)); + m_path = std::filesystem::temp_directory_path() / ("atb-sfbuilder-finalize-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(m_path)); } - ~TemporaryDirectory() { + ~TemporaryDirectory() + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); } - const std::filesystem::path &path() const { - return _path; - } + const std::filesystem::path& path() const { return m_path; } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -mesh::Simple sample_mesh() { - return mesh::Simple( - {{0, 1, 2}}, - {{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}}); -} +mesh::Simple sample_mesh() { return mesh::Simple({ { 0, 1, 2 } }, { { 0.0, 0.0, 0.0 }, { 1.0, 0.0, 0.0 }, { 0.0, 1.0, 0.0 } }); } } // namespace -TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuilder][sf]") { +TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuilder][sf]") +{ TemporaryDirectory directory; auto storage_result = octree::open_folder(directory.path()); REQUIRE(storage_result.has_value()); @@ -53,9 +48,8 @@ TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuil CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storemeta")); } -TEST_CASE( - "SF builder finalization retains an invalid written index for diagnosis", - "[sfbuilder][sf]") { +TEST_CASE("SF builder finalization retains an invalid written index for diagnosis", "[sfbuilder][sf]") +{ TemporaryDirectory directory; auto storage_result = octree::open_folder(directory.path()); REQUIRE(storage_result.has_value()); diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp index 05de6686..f583705f 100644 --- a/unittests/sf_merger/storage_boundaries.cpp +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -12,67 +12,75 @@ namespace { class TemporaryDirectory { public: - explicit TemporaryDirectory(const std::string_view label) { + explicit TemporaryDirectory(const std::string_view label) + { static std::atomic_uint64_t counter = 0; const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - _path = std::filesystem::temp_directory_path() - / ("atb-sfmerger-" + std::string(label) + "-" - + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(_path)); + m_path = std::filesystem::temp_directory_path() + / ("atb-sfmerger-" + std::string(label) + "-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(m_path)); } - ~TemporaryDirectory() { + ~TemporaryDirectory() + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); } - const std::filesystem::path &path() const { - return _path; - } + const std::filesystem::path& path() const { return m_path; } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -mesh::Simple triangle(const double x_offset = 0.0) { - return mesh::Simple( - {{0, 1, 2}}, +mesh::Simple triangle(const double x_offset = 0.0) +{ + return mesh::Simple({ { 0, 1, 2 } }, { - {-1.0 + x_offset, 0.0, 0.0}, - {1.0 + x_offset, 0.0, 0.0}, - {x_offset, 2.0, 0.0}, + { -1.0 + x_offset, 0.0, 0.0 }, + { 1.0 + x_offset, 0.0, 0.0 }, + { x_offset, 2.0, 0.0 }, }); } -mesh::Simple clipping_box() { +mesh::Simple clipping_box() +{ return mesh::Simple( { - {0, 1, 2}, {0, 2, 3}, - {1, 5, 6}, {1, 6, 2}, - {5, 4, 7}, {5, 7, 6}, - {4, 0, 3}, {4, 3, 7}, - {3, 2, 6}, {3, 6, 7}, - {4, 5, 1}, {4, 1, 0}, + { 0, 1, 2 }, + { 0, 2, 3 }, + { 1, 5, 6 }, + { 1, 6, 2 }, + { 5, 4, 7 }, + { 5, 7, 6 }, + { 4, 0, 3 }, + { 4, 3, 7 }, + { 3, 2, 6 }, + { 3, 6, 7 }, + { 4, 5, 1 }, + { 4, 1, 0 }, }, { - {0.0, -1.0, -1.0}, - {2.0, -1.0, -1.0}, - {2.0, 3.0, -1.0}, - {0.0, 3.0, -1.0}, - {0.0, -1.0, 1.0}, - {2.0, -1.0, 1.0}, - {2.0, 3.0, 1.0}, - {0.0, 3.0, 1.0}, + { 0.0, -1.0, -1.0 }, + { 2.0, -1.0, -1.0 }, + { 2.0, 3.0, -1.0 }, + { 0.0, 3.0, -1.0 }, + { 0.0, -1.0, 1.0 }, + { 2.0, -1.0, 1.0 }, + { 2.0, 3.0, 1.0 }, + { 0.0, 3.0, 1.0 }, }); } -mesh::storage::IndexedStorage indexed_storage(const std::filesystem::path &path) { +mesh::storage::IndexedStorage indexed_storage(const std::filesystem::path& path) +{ auto result = octree::open_folder_indexed(path); REQUIRE(result.has_value()); return std::move(result.value()); } -mesh::storage::Storage unindexed_storage(const std::filesystem::path &path) { +mesh::storage::Storage unindexed_storage(const std::filesystem::path& path) +{ auto result = octree::open_folder(path); REQUIRE(result.has_value()); return std::move(result.value()); @@ -80,7 +88,8 @@ mesh::storage::Storage unindexed_storage(const std::filesystem::path &path) { } // namespace -TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][validation]") { +TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][validation]") +{ TemporaryDirectory left_directory("invalid-left"); TemporaryDirectory right_directory("invalid-right"); TemporaryDirectory output_directory("invalid-output"); @@ -98,9 +107,8 @@ TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][valida CHECK_FALSE(std::filesystem::exists(output_directory.path() / "octree.storeindex")); } -TEST_CASE( - "SF merge retains an invalid output index for diagnosis", - "[sfmerger][sf][validation]") { +TEST_CASE("SF merge retains an invalid output index for diagnosis", "[sfmerger][sf][validation]") +{ TemporaryDirectory left_directory("valid-left"); TemporaryDirectory right_directory("valid-right"); TemporaryDirectory output_directory("bad-final-output"); @@ -118,10 +126,12 @@ TEST_CASE( CHECK(std::filesystem::is_regular_file(output_directory.path() / "octree.storeindex")); } -TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfmerger][sf][copy]") { +TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfmerger][sf][copy]") +{ const octree::Id root = octree::Id::root(); - SECTION("unchanged") { + SECTION("unchanged") + { TemporaryDirectory left_directory("unchanged-left"); TemporaryDirectory right_directory("unchanged-right"); TemporaryDirectory output_directory("unchanged-output"); @@ -132,12 +142,11 @@ TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfme REQUIRE(merge_datasets(left, right, output).has_value()); REQUIRE(output.has(root).value()); - CHECK(std::filesystem::equivalent( - left.paths(root)->front(), - output.paths(root)->front())); + CHECK(std::filesystem::equivalent(left.paths(root)->front(), output.paths(root)->front())); } - SECTION("changed") { + SECTION("changed") + { TemporaryDirectory left_directory("changed-left"); TemporaryDirectory right_directory("changed-right"); TemporaryDirectory output_directory("changed-output"); @@ -149,46 +158,41 @@ TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfme REQUIRE(merge_datasets(left, right, output).has_value()); REQUIRE(output.has(root).value()); - CHECK_FALSE(std::filesystem::equivalent( - left.paths(root)->front(), - output.paths(root)->front())); - CHECK_FALSE(std::filesystem::equivalent( - right.paths(root)->front(), - output.paths(root)->front())); + CHECK_FALSE(std::filesystem::equivalent(left.paths(root)->front(), output.paths(root)->front())); + CHECK_FALSE(std::filesystem::equivalent(right.paths(root)->front(), output.paths(root)->front())); REQUIRE(output.load(root).has_value()); CHECK(output.load(root)->face_count() == 2); } } -TEST_CASE("SF cut hard-links unchanged leaves and writes clipped leaves", "[sfmerger][sf][copy]") { +TEST_CASE("SF cut hard-links unchanged leaves and writes clipped leaves", "[sfmerger][sf][copy]") +{ const octree::Id root = octree::Id::root(); - SECTION("unchanged") { + SECTION("unchanged") + { TemporaryDirectory input_directory("cut-unchanged-input"); TemporaryDirectory output_directory("cut-unchanged-output"); auto input = indexed_storage(input_directory.path()); auto output = unindexed_storage(output_directory.path()); REQUIRE(input.save(root, triangle()).has_value()); - REQUIRE(cut_dataset(input, MeshMask{}, output, false).has_value()); + REQUIRE(cut_dataset(input, MeshMask {}, output, false).has_value()); REQUIRE(output.has(root).value()); - CHECK(std::filesystem::equivalent( - input.paths(root)->front(), - output.paths(root)->front())); + CHECK(std::filesystem::equivalent(input.paths(root)->front(), output.paths(root)->front())); } - SECTION("clipped") { + SECTION("clipped") + { TemporaryDirectory input_directory("cut-clipped-input"); TemporaryDirectory output_directory("cut-clipped-output"); auto input = indexed_storage(input_directory.path()); auto output = unindexed_storage(output_directory.path()); REQUIRE(input.save(root, triangle()).has_value()); - REQUIRE(cut_dataset(input, MeshMask{.components = {clipping_box()}}, output, true).has_value()); + REQUIRE(cut_dataset(input, MeshMask { .components = { clipping_box() } }, output, true).has_value()); REQUIRE(output.has(root).value()); - CHECK_FALSE(std::filesystem::equivalent( - input.paths(root)->front(), - output.paths(root)->front())); + CHECK_FALSE(std::filesystem::equivalent(input.paths(root)->front(), output.paths(root)->front())); const auto clipped = output.load(root); REQUIRE(clipped.has_value()); CHECK_FALSE(clipped->is_empty()); diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index ab461b7a..da6ab2d7 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -2,8 +2,8 @@ #include "io/envelope.h" -#include #include +#include #include #include @@ -14,66 +14,63 @@ namespace { namespace v1 { -struct Payload { - std::uint32_t id; - std::string name; + struct Payload { + std::uint32_t id; + std::string name; - bool operator==(const Payload &) const = default; -}; + bool operator==(const Payload&) const = default; + }; } // namespace v1 namespace v2 { -struct Payload { - std::uint64_t id; - std::string name; - bool enabled; - - static Payload from_previous(v1::Payload previous) - { - return { - .id = previous.id, - .name = std::move(previous.name), - .enabled = true, - }; - } + struct Payload { + std::uint64_t id; + std::string name; + bool enabled; + + static Payload from_previous(v1::Payload previous) + { + return { + .id = previous.id, + .name = std::move(previous.name), + .enabled = true, + }; + } - bool operator==(const Payload &) const = default; -}; + bool operator==(const Payload&) const = default; + }; } // namespace v2 namespace v3 { -struct Payload { - std::uint64_t id; - std::string label; - bool enabled; - std::vector samples; - - static Payload from_previous(v2::Payload previous) - { - return { - .id = previous.id, - .label = std::move(previous.name), - .enabled = previous.enabled, - .samples = {}, - }; - } + struct Payload { + std::uint64_t id; + std::string label; + bool enabled; + std::vector samples; + + static Payload from_previous(v2::Payload previous) + { + return { + .id = previous.id, + .label = std::move(previous.name), + .enabled = previous.enabled, + .samples = {}, + }; + } - bool operator==(const Payload &) const = default; -}; + bool operator==(const Payload&) const = default; + }; } // namespace v3 -using Schema = io::envelope::PayloadSchema< - "test.Payload", - io::envelope::Version<1, v1::Payload>, - io::envelope::Version<2, v2::Payload>, - io::envelope::Version<3, v3::Payload>>; +using Schema = io::envelope:: + PayloadSchema<"test.Payload", io::envelope::Version<1, v1::Payload>, io::envelope::Version<2, v2::Payload>, io::envelope::Version<3, v3::Payload>>; -constexpr std::array zstd_compression_algorithms{ +constexpr std::array zstd_compression_algorithms { io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, }; @@ -87,7 +84,7 @@ static_assert(std::same_as); static_assert(std::same_as, v1::Payload>); template -io::envelope::Bytes encode_value(const Value &value) +io::envelope::Bytes encode_value(const Value& value) { io::envelope::Bytes bytes; zpp::bits::out output(bytes); @@ -95,45 +92,36 @@ io::envelope::Bytes encode_value(const Value &value) return bytes; } -io::envelope::Bytes encode_envelope(const io::envelope::Envelope &envelope) -{ - return encode_value(envelope); -} +io::envelope::Bytes encode_envelope(const io::envelope::Envelope& envelope) { return encode_value(envelope); } -io::envelope::Envelope decode_envelope(const io::envelope::Bytes &bytes) +io::envelope::Envelope decode_envelope(const io::envelope::Bytes& bytes) { - io::envelope::Envelope envelope{}; + io::envelope::Envelope envelope {}; zpp::bits::in input(bytes); input(envelope).or_throw(); return envelope; } -io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes &uncompressed_data) +io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes& uncompressed_data) { - const std::unique_ptr context{ - ZSTD_createCCtx(), &ZSTD_freeCCtx}; - if (!context - || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_contentSizeFlag, 0)) + const std::unique_ptr context { ZSTD_createCCtx(), &ZSTD_freeCCtx }; + if (!context || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_contentSizeFlag, 0)) || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { - throw std::runtime_error{"could not configure zstd test context"}; + throw std::runtime_error { "could not configure zstd test context" }; } io::envelope::Bytes compressed_data(ZSTD_compressBound(uncompressed_data.size())); - const std::size_t compressed_size = ZSTD_compress2( - context.get(), - compressed_data.data(), - compressed_data.size(), - uncompressed_data.data(), - uncompressed_data.size()); + const std::size_t compressed_size + = ZSTD_compress2(context.get(), compressed_data.data(), compressed_data.size(), uncompressed_data.data(), uncompressed_data.size()); if (ZSTD_isError(compressed_size)) { - throw std::runtime_error{"could not create zstd test data"}; + throw std::runtime_error { "could not create zstd test data" }; } compressed_data.resize(compressed_size); return compressed_data; } template -void check_error(const Result &result, const io::envelope::ErrorCode expected) +void check_error(const Result& result, const io::envelope::ErrorCode expected) { REQUIRE_FALSE(result.has_value()); CHECK(result.error().code == expected); @@ -153,11 +141,11 @@ io::envelope::Bytes bytes_from_string(const std::string_view text) TEST_CASE("Envelope round trips the latest payload version") { - const v3::Payload expected{ + const v3::Payload expected { .id = 42, .label = "latest", .enabled = false, - .samples = {1, 2, 3}, + .samples = { 1, 2, 3 }, }; const auto bytes = io::envelope::serialize(expected); @@ -169,8 +157,7 @@ TEST_CASE("Envelope round trips the latest payload version") CHECK(envelope.class_version == 3); CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::HandledByCompressionLib); CHECK(envelope.checksum.empty()); - CHECK(envelope.compression_algorithm - == io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum); + CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum); CHECK(envelope.uncompressed_size == encode_value(expected).size()); const auto result = io::envelope::deserialize(*bytes); @@ -182,49 +169,48 @@ TEST_CASE("Envelope upgrades older payload versions") { SECTION("version 1 is upgraded through every subsequent version") { - const v1::Payload original{.id = 7, .name = "version one"}; + const v1::Payload original { .id = 7, .name = "version one" }; const auto bytes = io::envelope::serialize(original); REQUIRE(bytes.has_value()); const auto result = io::envelope::deserialize(*bytes); REQUIRE(result.has_value()); - CHECK(*result == v3::Payload{ - .id = 7, - .label = "version one", - .enabled = true, - .samples = {}, - }); + CHECK(*result + == v3::Payload { + .id = 7, + .label = "version one", + .enabled = true, + .samples = {}, + }); } SECTION("version 2 is upgraded to the latest version") { - const v2::Payload original{.id = 9, .name = "version two", .enabled = false}; + const v2::Payload original { .id = 9, .name = "version two", .enabled = false }; const auto bytes = io::envelope::serialize(original); REQUIRE(bytes.has_value()); const auto result = io::envelope::deserialize(*bytes); REQUIRE(result.has_value()); - CHECK(*result == v3::Payload{ - .id = 9, - .label = "version two", - .enabled = false, - .samples = {}, - }); + CHECK(*result + == v3::Payload { + .id = 9, + .label = "version two", + .enabled = false, + .samples = {}, + }); } } TEST_CASE("Envelope supports uncompressed data without a checksum") { - const v3::Payload expected{ + const v3::Payload expected { .id = 11, .label = "plain", .enabled = true, - .samples = {5, 8}, + .samples = { 5, 8 }, }; - const auto bytes = io::envelope::serialize( - expected, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::None); + const auto bytes = io::envelope::serialize(expected, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::None); REQUIRE(bytes.has_value()); const auto envelope = decode_envelope(*bytes); @@ -243,19 +229,14 @@ TEST_CASE("CRC-32C uses its canonical hexadecimal representation") SECTION("standard test vector") { const auto compressed = io::envelope::compress_with_checksum( - bytes_from_string("123456789"), - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + bytes_from_string("123456789"), io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(compressed.has_value()); CHECK(compressed->checksum == "e3069283"); } SECTION("empty input") { - const auto compressed = io::envelope::compress_with_checksum( - {}, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + const auto compressed = io::envelope::compress_with_checksum({}, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(compressed.has_value()); CHECK(compressed->checksum == "00000000"); } @@ -267,28 +248,19 @@ TEST_CASE("CRC-32C protects independently compressed data") SECTION("without compression") { - const auto compressed = io::envelope::compress_with_checksum( - original, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + const auto compressed + = io::envelope::compress_with_checksum(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(compressed.has_value()); const auto result = io::envelope::checked_decompress( - compressed->compressed_data, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c, - compressed->checksum); + compressed->compressed_data, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, compressed->checksum); REQUIRE(result.has_value()); CHECK(*result == original); auto corrupted = compressed->compressed_data; - corrupted.front() ^= std::byte{0x01}; - check_error( - io::envelope::checked_decompress( - corrupted, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c, - compressed->checksum), + corrupted.front() ^= std::byte { 0x01 }; + check_error(io::envelope::checked_decompress( + corrupted, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, compressed->checksum), io::envelope::ErrorCode::ChecksumMismatch); } @@ -296,48 +268,28 @@ TEST_CASE("CRC-32C protects independently compressed data") { for (const auto compression_algorithm : zstd_compression_algorithms) { CAPTURE(compression_algorithm); - const auto compressed = io::envelope::compress_with_checksum( - original, - compression_algorithm, - io::envelope::ChecksumAlgorithm::Crc32c); + const auto compressed = io::envelope::compress_with_checksum(original, compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(compressed.has_value()); REQUIRE(compressed->checksum.size() == 8); const auto result = io::envelope::checked_decompress( - compressed->compressed_data, - compression_algorithm, - io::envelope::ChecksumAlgorithm::Crc32c, - compressed->checksum); + compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c, compressed->checksum); REQUIRE(result.has_value()); CHECK(*result == original); auto wrong_checksum = compressed->checksum; wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; check_error( - io::envelope::checked_decompress( - compressed->compressed_data, - compression_algorithm, - io::envelope::ChecksumAlgorithm::Crc32c, - wrong_checksum), + io::envelope::checked_decompress(compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c, wrong_checksum), io::envelope::ErrorCode::ChecksumMismatch); } } SECTION("malformed checksum") { - check_error( - io::envelope::checked_decompress( - original, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c, - "E3069283"), + check_error(io::envelope::checked_decompress(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, "E3069283"), io::envelope::ErrorCode::ChecksumMismatch); - check_error( - io::envelope::checked_decompress( - original, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c, - {}), + check_error(io::envelope::checked_decompress(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, {}), io::envelope::ErrorCode::ChecksumMismatch); } } @@ -346,16 +298,13 @@ TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") { SECTION("latest version") { - const v3::Payload expected{ + const v3::Payload expected { .id = 12, .label = "crc", .enabled = true, - .samples = {3, 5, 8}, + .samples = { 3, 5, 8 }, }; - const auto bytes = io::envelope::serialize( - expected, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + const auto bytes = io::envelope::serialize(expected, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(bytes.has_value()); const auto envelope = decode_envelope(*bytes); @@ -368,116 +317,82 @@ TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") CHECK(*result == expected); auto corrupted = envelope; - corrupted.compressed_data.front() ^= std::byte{0x01}; - check_error(io::envelope::deserialize(encode_envelope(corrupted)), - io::envelope::ErrorCode::ChecksumMismatch); + corrupted.compressed_data.front() ^= std::byte { 0x01 }; + check_error(io::envelope::deserialize(encode_envelope(corrupted)), io::envelope::ErrorCode::ChecksumMismatch); } SECTION("older version") { - const v1::Payload original{.id = 13, .name = "crc version one"}; - const auto bytes = io::envelope::serialize( - original, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + const v1::Payload original { .id = 13, .name = "crc version one" }; + const auto bytes = io::envelope::serialize(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(bytes.has_value()); const auto result = io::envelope::deserialize(*bytes); REQUIRE(result.has_value()); - CHECK(*result == v3::Payload{ - .id = 13, - .label = "crc version one", - .enabled = true, - .samples = {}, - }); + CHECK(*result + == v3::Payload { + .id = 13, + .label = "crc version one", + .enabled = true, + .samples = {}, + }); } } TEST_CASE("Checked compression round trips and validates its checksum") { - const io::envelope::Bytes original(4096, std::byte{0x2a}); + const io::envelope::Bytes original(4096, std::byte { 0x2a }); for (const auto compression_algorithm : zstd_compression_algorithms) { CAPTURE(compression_algorithm); - const auto compressed = io::envelope::compress_with_checksum( - original, - compression_algorithm, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + const auto compressed = io::envelope::compress_with_checksum(original, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib); REQUIRE(compressed.has_value()); CHECK(compressed->compressed_data.size() < original.size()); CHECK(compressed->checksum.empty()); const auto result = io::envelope::checked_decompress( - compressed->compressed_data, - compression_algorithm, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - compressed->checksum); + compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, compressed->checksum); REQUIRE(result.has_value()); CHECK(*result == original); - const auto empty_compressed = io::envelope::compress_with_checksum( - {}, - compression_algorithm, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + const auto empty_compressed = io::envelope::compress_with_checksum({}, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib); REQUIRE(empty_compressed.has_value()); const auto empty_result = io::envelope::checked_decompress( - empty_compressed->compressed_data, - compression_algorithm, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - empty_compressed->checksum); + empty_compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, empty_compressed->checksum); REQUIRE(empty_result.has_value()); CHECK(empty_result->empty()); auto corrupted = compressed->compressed_data; - corrupted.back() ^= std::byte{0x01}; - check_error( - io::envelope::checked_decompress( - corrupted, - compression_algorithm, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}), + corrupted.back() ^= std::byte { 0x01 }; + check_error(io::envelope::checked_decompress(corrupted, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}), io::envelope::ErrorCode::ChecksumMismatch); - check_error( - io::envelope::checked_decompress( - compressed->compressed_data, - compression_algorithm, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}, - original.size() - 1), + check_error(io::envelope::checked_decompress( + compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size() - 1), io::envelope::ErrorCode::SizeLimitExceeded); } } TEST_CASE("Checked decompression uses its maximum when the format omits the content size") { - const io::envelope::Bytes original(4096, std::byte{0x37}); + const io::envelope::Bytes original(4096, std::byte { 0x37 }); const auto compressed_data = compress_without_content_size(original); for (const auto compression_algorithm : zstd_compression_algorithms) { CAPTURE(compression_algorithm); const auto result = io::envelope::checked_decompress( - compressed_data, - compression_algorithm, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}, - original.size()); + compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size()); REQUIRE(result.has_value()); CHECK(*result == original); - check_error( - io::envelope::checked_decompress( - compressed_data, - compression_algorithm, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}, - original.size() - 1), + check_error(io::envelope::checked_decompress( + compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size() - 1), io::envelope::ErrorCode::SizeLimitExceeded); } } TEST_CASE("Envelope rejects incompatible metadata") { - const v3::Payload payload{.id = 1, .label = "metadata", .enabled = true, .samples = {}}; + const v3::Payload payload { .id = 1, .label = "metadata", .enabled = true, .samples = {} }; const auto serialized = io::envelope::serialize(payload); REQUIRE(serialized.has_value()); @@ -485,56 +400,49 @@ TEST_CASE("Envelope rejects incompatible metadata") { auto envelope = decode_envelope(*serialized); ++envelope.magic; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::InvalidMagic); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::InvalidMagic); } SECTION("class name") { auto envelope = decode_envelope(*serialized); envelope.class_name = "other.Payload"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::WrongClassName); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::WrongClassName); } SECTION("class version") { auto envelope = decode_envelope(*serialized); envelope.class_version = 99; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::UnsupportedClassVersion); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::UnsupportedClassVersion); } SECTION("checksum algorithm") { auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = static_cast(99); - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::UnsupportedChecksumAlgorithm); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::UnsupportedChecksumAlgorithm); } SECTION("compression algorithm") { auto envelope = decode_envelope(*serialized); envelope.compression_algorithm = static_cast(99); - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::UnsupportedCompressionAlgorithm); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::UnsupportedCompressionAlgorithm); } SECTION("algorithm combination") { auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::None; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::InvalidAlgorithmCombination); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::InvalidAlgorithmCombination); } SECTION("external checksum") { auto envelope = decode_envelope(*serialized); envelope.checksum = "not used by zstd"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::InvalidAlgorithmCombination); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::InvalidAlgorithmCombination); } SECTION("malformed CRC-32C checksum") @@ -542,8 +450,7 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; envelope.checksum = "1234"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::ChecksumMismatch); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::ChecksumMismatch); } SECTION("incorrect CRC-32C checksum") @@ -551,76 +458,63 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; envelope.checksum = "00000000"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::ChecksumMismatch); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::ChecksumMismatch); } SECTION("uncompressed size is smaller than the payload") { auto envelope = decode_envelope(*serialized); --envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); } SECTION("uncompressed size is larger than the payload") { auto envelope = decode_envelope(*serialized); ++envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); } SECTION("zero uncompressed size does not mean unspecified") { auto envelope = decode_envelope(*serialized); envelope.uncompressed_size = 0; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); } SECTION("uncompressed size exceeds the hard limit") { auto envelope = decode_envelope(*serialized); envelope.uncompressed_size = io::envelope::default_max_decompressed_size + 1; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::SizeLimitExceeded); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::SizeLimitExceeded); } SECTION("uncompressed size exceeds a caller limit") { auto envelope = decode_envelope(*serialized); - check_error( - io::envelope::deserialize( - encode_envelope(envelope), - static_cast(envelope.uncompressed_size - 1)), + check_error(io::envelope::deserialize(encode_envelope(envelope), static_cast(envelope.uncompressed_size - 1)), io::envelope::ErrorCode::SizeLimitExceeded); } } TEST_CASE("Envelope validates the size of uncompressed data") { - const v3::Payload payload{.id = 2, .label = "plain", .enabled = true, .samples = {1}}; - const auto serialized = io::envelope::serialize( - payload, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::None); + const v3::Payload payload { .id = 2, .label = "plain", .enabled = true, .samples = { 1 } }; + const auto serialized = io::envelope::serialize(payload, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::None); REQUIRE(serialized.has_value()); SECTION("declared size is smaller") { auto envelope = decode_envelope(*serialized); --envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); } SECTION("declared size is larger") { auto envelope = decode_envelope(*serialized); ++envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); } } @@ -628,24 +522,22 @@ TEST_CASE("Envelope reports malformed serialized data") { SECTION("envelope") { - const io::envelope::Bytes malformed{std::byte{0x01}, std::byte{0x02}}; - check_error(io::envelope::deserialize(malformed), - io::envelope::ErrorCode::DeserializationFailed); + const io::envelope::Bytes malformed { std::byte { 0x01 }, std::byte { 0x02 } }; + check_error(io::envelope::deserialize(malformed), io::envelope::ErrorCode::DeserializationFailed); } SECTION("payload") { - const io::envelope::Envelope envelope{ + const io::envelope::Envelope envelope { .magic = io::envelope::magic, - .class_name = std::string{Schema::class_name}, + .class_name = std::string { Schema::class_name }, .class_version = 3, .checksum_algorithm = io::envelope::ChecksumAlgorithm::None, .checksum = {}, .compression_algorithm = io::envelope::CompressionAlgorithm::None, .uncompressed_size = 1, - .compressed_data = {std::byte{0x01}}, + .compressed_data = { std::byte { 0x01 } }, }; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DeserializationFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DeserializationFailed); } } diff --git a/unittests/terrainlib/io_conversion.cpp b/unittests/terrainlib/io_conversion.cpp index 2dd7c7e8..5fd818c8 100644 --- a/unittests/terrainlib/io_conversion.cpp +++ b/unittests/terrainlib/io_conversion.cpp @@ -51,10 +51,8 @@ TEST_CASE("io::conversion copies non-contiguous matrices row by row") cv::Mat backing(4, 6, CV_8UC3); for (int row = 0; row < backing.rows; ++row) { for (int column = 0; column < backing.cols; ++column) { - backing.at(row, column) = cv::Vec3b( - static_cast(row), - static_cast(column), - static_cast(row * backing.cols + column)); + backing.at(row, column) + = cv::Vec3b(static_cast(row), static_cast(column), static_cast(row * backing.cols + column)); } } const cv::Mat source = backing(cv::Rect(1, 1, 3, 2)); @@ -62,11 +60,11 @@ TEST_CASE("io::conversion copies non-contiguous matrices row by row") auto raster = io::conversion::to_raster(source); REQUIRE(raster.has_value()); - CHECK(raster->pixel({0, 0}) == glm::u8vec3(1, 1, 7)); - CHECK(raster->pixel({2, 1}) == glm::u8vec3(2, 3, 15)); + CHECK(raster->pixel({ 0, 0 }) == glm::u8vec3(1, 1, 7)); + CHECK(raster->pixel({ 2, 1 }) == glm::u8vec3(2, 3, 15)); backing.setTo(cv::Scalar::all(0)); - CHECK(raster->pixel({0, 0}) == glm::u8vec3(1, 1, 7)); + CHECK(raster->pixel({ 0, 0 }) == glm::u8vec3(1, 1, 7)); const auto round_trip = io::conversion::to_mat(*raster); REQUIRE(round_trip.has_value()); @@ -78,14 +76,14 @@ TEST_CASE("io::conversion reports invalid inputs") { SECTION("empty matrix") { - const auto result = io::conversion::to_raster(cv::Mat{}); + const auto result = io::conversion::to_raster(cv::Mat {}); REQUIRE_FALSE(result.has_value()); CHECK(result.error().code == io::conversion::ErrorCode::EmptyInput); } SECTION("multidimensional matrix") { - const std::array sizes{2, 2, 2}; + const std::array sizes { 2, 2, 2 }; const cv::Mat source(static_cast(sizes.size()), sizes.data(), CV_8UC1); const auto result = io::conversion::to_raster(source); REQUIRE_FALSE(result.has_value()); @@ -97,7 +95,7 @@ TEST_CASE("io::conversion reports invalid inputs") const cv::Mat source(2, 2, CV_8UC3); const auto result = io::conversion::to_raster(source); REQUIRE_FALSE(result.has_value()); - const io::conversion::Error expected{ + const io::conversion::Error expected { io::conversion::ErrorCode::PixelTypeMismatch, CV_8UC1, CV_8UC3, @@ -118,7 +116,7 @@ TEST_CASE("io::conversion reports invalid inputs") SECTION("empty raster") { - const auto result = io::conversion::to_mat(radix::Raster{}); + const auto result = io::conversion::to_mat(radix::Raster {}); REQUIRE_FALSE(result.has_value()); CHECK(result.error().code == io::conversion::ErrorCode::EmptyInput); } diff --git a/unittests/terrainlib/sf_validate_index.cpp b/unittests/terrainlib/sf_validate_index.cpp index b8f7e2ec..1afd61e3 100644 --- a/unittests/terrainlib/sf_validate_index.cpp +++ b/unittests/terrainlib/sf_validate_index.cpp @@ -3,7 +3,8 @@ #include "octree/StoreTraits.h" #include "sf/validate_index.h" -TEST_CASE("Structura Fundamentalis validation accepts Leaf and Virtual nodes", "[sf][store]") { +TEST_CASE("Structura Fundamentalis validation accepts Leaf and Virtual nodes", "[sf][store]") +{ store::Index index; const octree::Id leaf = octree::Id::root().child(2).value().child(5).value(); REQUIRE(index.add(leaf).has_value()); @@ -11,7 +12,8 @@ TEST_CASE("Structura Fundamentalis validation accepts Leaf and Virtual nodes", " CHECK(sf::validate_index(index).has_value()); } -TEST_CASE("Structura Fundamentalis validation reports the first Inner node", "[sf][store]") { +TEST_CASE("Structura Fundamentalis validation reports the first Inner node", "[sf][store]") +{ store::Index index; const octree::Id root = octree::Id::root(); const octree::Id child = root.child(3).value(); diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp index 47dfdb46..bbdfa2dd 100644 --- a/unittests/terrainlib/store_codec.cpp +++ b/unittests/terrainlib/store_codec.cpp @@ -16,47 +16,47 @@ namespace { class TemporaryDirectory { public: - explicit TemporaryDirectory(const std::string_view label) { + explicit TemporaryDirectory(const std::string_view label) + { static std::atomic_uint64_t counter = 0; const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - _path = std::filesystem::temp_directory_path() - / ("atb-store-codec-" + std::string(label) + "-" - + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(_path)); + m_path = std::filesystem::temp_directory_path() + / ("atb-store-codec-" + std::string(label) + "-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(m_path)); } - ~TemporaryDirectory() { + ~TemporaryDirectory() + { std::error_code error; - std::filesystem::remove_all(_path, error); - } - const std::filesystem::path &path() const { - return _path; + std::filesystem::remove_all(m_path, error); } + const std::filesystem::path& path() const { return m_path; } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; class MultiFileCodec final : public store::Codec { public: - std::vector paths(const store::NodePath &node_path) const override { + std::vector paths(const store::NodePath& node_path) const override + { std::filesystem::path data = node_path.path(); std::filesystem::path metadata = node_path.path(); data += ".data"; metadata += ".metadata"; - return {data, metadata}; + return { data, metadata }; } }; class WriteOnlyCodec final : public store::Codec { public: - std::vector paths(const store::NodePath &node_path) const override { + std::vector paths(const store::NodePath& node_path) const override + { std::filesystem::path path = node_path.path(); path += ".out"; - return {path}; + return { path }; } - std::expected write( - const store::NodePath &, - const int &) const override { + std::expected write(const store::NodePath&, const int&) const override + { ++writes; return {}; } @@ -64,50 +64,46 @@ class WriteOnlyCodec final : public store::Codec { mutable std::atomic_uint32_t writes = 0; }; -mesh::Simple triangle_mesh(const double offset) { - return mesh::Simple( - {{0, 1, 2}}, - {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); +mesh::Simple triangle_mesh(const double offset) +{ + return mesh::Simple({ { 0, 1, 2 } }, { { offset, 0.0, 0.0 }, { offset + 1.0, 0.0, 0.0 }, { offset, 1.0, 0.0 } }); } -template -void check_mesh_codec_reentrancy(const Codec &codec, const std::filesystem::path &base) { +template +void check_mesh_codec_reentrancy(const Codec& codec, const std::filesystem::path& base) +{ std::vector> writes; for (int index = 0; index < 4; ++index) { - writes.push_back(std::async(std::launch::async, [&codec, base, index] { - return codec.write( - store::NodePath(base / std::to_string(index) / "node"), - triangle_mesh(index)) - .has_value(); - })); + writes.push_back(std::async(std::launch::async, + [&codec, base, index] { return codec.write(store::NodePath(base / std::to_string(index) / "node"), triangle_mesh(index)).has_value(); })); } - for (auto &write : writes) { + for (auto& write : writes) { REQUIRE(write.get()); } std::vector> reads; for (int index = 0; index < 4; ++index) { reads.push_back(std::async(std::launch::async, [&codec, base, index] { - const auto result = codec.read( - store::NodePath(base / std::to_string(index) / "node")); + const auto result = codec.read(store::NodePath(base / std::to_string(index) / "node")); return result.has_value() && result->face_count() == 1; })); } - for (auto &read : reads) { + for (auto& read : reads) { REQUIRE(read.get()); } } } // namespace -TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec]") { +TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec]") +{ MultiFileCodec codec; const store::NodePath path("12/34/56/78"); CHECK(codec.paths(path) - == std::vector{ - "12/34/56/78.data", - "12/34/56/78.metadata", - }); + == std::vector { + "12/34/56/78.data", + "12/34/56/78.metadata", + }); const auto read = codec.read(path); REQUIRE_FALSE(read.has_value()); @@ -120,7 +116,8 @@ TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec CHECK(write.error().category == store::CodecErrorCategory::UnsupportedOperation); } -TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") { +TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") +{ WriteOnlyCodec codec; REQUIRE(codec.write(store::NodePath("node"), 42).has_value()); CHECK(codec.writes == 1); @@ -129,25 +126,17 @@ TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][cod CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); } -TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") { +TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") +{ TemporaryDirectory directory("mesh-reentrant"); - SECTION("SF mesh") { - check_mesh_codec_reentrancy(mesh::codec::SfMesh{}, directory.path() / "sfmesh"); - } - SECTION("binary glTF") { - check_mesh_codec_reentrancy( - mesh::codec::Gltf(mesh::codec::GltfContainer::Binary), - directory.path() / "glb"); - } - SECTION("JSON glTF") { - check_mesh_codec_reentrancy( - mesh::codec::Gltf(mesh::codec::GltfContainer::Json), - directory.path() / "gltf"); - } + SECTION("SF mesh") { check_mesh_codec_reentrancy(mesh::codec::SfMesh {}, directory.path() / "sfmesh"); } + SECTION("binary glTF") { check_mesh_codec_reentrancy(mesh::codec::Gltf(mesh::codec::GltfContainer::Binary), directory.path() / "glb"); } + SECTION("JSON glTF") { check_mesh_codec_reentrancy(mesh::codec::Gltf(mesh::codec::GltfContainer::Json), directory.path() / "gltf"); } } -TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") { +TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") +{ TemporaryDirectory directory("gltf-error"); mesh::codec::Gltf codec(mesh::codec::GltfContainer::Binary); const store::NodePath node_path(directory.path() / "blocked"); diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 9b5c8fad..94c565d9 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -14,54 +14,48 @@ #include "mesh/codec/SfMeshFormat.h" #include "mesh/codec/from_extension.h" #include "mesh/storage.h" -#include "octree/store_layout/Mappings.h" #include "octree/storage/IndexFile.h" #include "octree/storage/open.h" +#include "octree/store_layout/Mappings.h" namespace { class TemporaryDirectory { public: - explicit TemporaryDirectory(const std::string_view label) { + explicit TemporaryDirectory(const std::string_view label) + { static std::atomic_uint64_t counter = 0; const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - _path = std::filesystem::temp_directory_path() - / ("atb-raster-store-" + std::string(label) + "-" - + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(_path)); + m_path = std::filesystem::temp_directory_path() + / ("atb-raster-store-" + std::string(label) + "-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(m_path)); } - ~TemporaryDirectory() { + ~TemporaryDirectory() + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); } - const std::filesystem::path &path() const { - return _path; - } + const std::filesystem::path& path() const { return m_path; } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -mesh::Simple triangle_mesh(const double offset) { - return mesh::Simple( - {{0, 1, 2}}, - {{offset, 0.0, 0.0}, {offset + 1.0, 0.0, 0.0}, {offset, 1.0, 0.0}}); +mesh::Simple triangle_mesh(const double offset) +{ + return mesh::Simple({ { 0, 1, 2 } }, { { offset, 0.0, 0.0 }, { offset + 1.0, 0.0, 0.0 }, { offset, 1.0, 0.0 } }); } -mesh::storage::IndexedStorage make_storage( - const std::filesystem::path &path, - const store::PathMapping mapping, - const std::string &extension) { +mesh::storage::IndexedStorage make_storage(const std::filesystem::path& path, const store::PathMapping mapping, const std::string& extension) +{ auto codec = mesh::codec::from_extension(extension); REQUIRE(codec.has_value()); return mesh::storage::IndexedStorage( - store::RawStorage( - store::Layout(path, mapping), - std::move(codec.value())), - store::Index{}, - mesh::storage::Storage::Persistence{ + store::RawStorage(store::Layout(path, mapping), std::move(codec.value())), + store::Index {}, + mesh::storage::Storage::Persistence { octree::storage::index_format(), path / "octree.storeindex", std::string(mapping.id), @@ -72,10 +66,11 @@ mesh::storage::IndexedStorage make_storage( } // namespace -TEST_CASE("octree layouts round-trip boundary IDs") { - const std::vector ids{ +TEST_CASE("octree layouts round-trip boundary IDs") +{ + const std::vector ids { octree::Id::root(), - octree::Id(octree::Id::max_level(), octree::Id::Index{0}), + octree::Id(octree::Id::max_level(), octree::Id::Index { 0 }), octree::Id(octree::Id::max_level(), octree::Id::max_index_on_level(octree::Id::max_level())), }; @@ -90,36 +85,32 @@ TEST_CASE("octree layouts round-trip boundary IDs") { } } -TEST_CASE("versioned octree and SF payload validation is free-standing") { - const octree::storage::StoreIndex invalid_index{{{ +TEST_CASE("versioned octree and SF payload validation is free-standing") +{ + const octree::storage::StoreIndex invalid_index { { { .level = std::numeric_limits::max(), .index = 0, .status = static_cast(store::NodeStatus::Leaf), - }}}; + } } }; CHECK_FALSE(octree::storage::decode_index(invalid_index).has_value()); - const mesh::sf::Payload oversized_mesh{ + const mesh::sf::Payload oversized_mesh { .vertex_count = std::numeric_limits::max(), .face_count = 0, .triangles = {}, - .positions = {0}, + .positions = { 0 }, .uvs = {}, .texture = {}, }; CHECK_FALSE(mesh::sf::validate(oversized_mesh).has_value()); } -TEST_CASE("storage hard-links matching payload formats") { +TEST_CASE("storage hard-links matching payload formats") +{ TemporaryDirectory source_directory("hard-link-source"); TemporaryDirectory target_directory("hard-link-target"); - auto source = make_storage( - source_directory.path(), - octree::store_layout::flat(), - ".sfmesh"); - auto target = make_storage( - target_directory.path(), - octree::store_layout::flat(), - ".sfmesh"); + auto source = make_storage(source_directory.path(), octree::store_layout::flat(), ".sfmesh"); + auto target = make_storage(target_directory.path(), octree::store_layout::flat(), ".sfmesh"); const octree::Id root = octree::Id::root(); REQUIRE(source.save(root, triangle_mesh(0.0)).has_value()); @@ -129,24 +120,17 @@ TEST_CASE("storage hard-links matching payload formats") { REQUIRE(target.save_index().has_value()); } -TEST_CASE("storage re-encodes differing payload formats") { +TEST_CASE("storage re-encodes differing payload formats") +{ TemporaryDirectory source_directory("reencode-source"); TemporaryDirectory target_directory("reencode-target"); - auto source = make_storage( - source_directory.path(), - octree::store_layout::flat(), - ".sfmesh"); - auto target = make_storage( - target_directory.path(), - octree::store_layout::flat(), - ".glb"); + auto source = make_storage(source_directory.path(), octree::store_layout::flat(), ".sfmesh"); + auto target = make_storage(target_directory.path(), octree::store_layout::flat(), ".glb"); const octree::Id root = octree::Id::root(); REQUIRE(source.save(root, triangle_mesh(4.0)).has_value()); REQUIRE(target.copy_from(root, source).has_value()); - CHECK_FALSE(std::filesystem::equivalent( - source.path_for(root).value(), - target.path_for(root).value())); + CHECK_FALSE(std::filesystem::equivalent(source.path_for(root).value(), target.path_for(root).value())); const auto loaded = target.load(root); REQUIRE(loaded.has_value()); CHECK(loaded->face_count() == 1); @@ -154,12 +138,10 @@ TEST_CASE("storage re-encodes differing payload formats") { REQUIRE(target.save_index().has_value()); } -TEST_CASE("storage supports enabled overwrites") { +TEST_CASE("storage supports enabled overwrites") +{ TemporaryDirectory directory("overwrite"); - auto storage = make_storage( - directory.path(), - octree::store_layout::flat(), - ".sfmesh"); + auto storage = make_storage(directory.path(), octree::store_layout::flat(), ".sfmesh"); storage.settings().allow_overwrite = true; const octree::Id root = octree::Id::root(); @@ -172,7 +154,8 @@ TEST_CASE("storage supports enabled overwrites") { REQUIRE(storage.save_index().has_value()); } -TEST_CASE("folder opening persists and reopens metadata-driven storage") { +TEST_CASE("folder opening persists and reopens metadata-driven storage") +{ TemporaryDirectory directory("folder-open"); octree::OpenOptions options; options.default_mapping = octree::store_layout::flat(); @@ -209,8 +192,9 @@ TEST_CASE("folder opening persists and reopens metadata-driven storage") { CHECK(metadata->codec_selector == ".sfmesh"); } -TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][open]") { - for (const std::string extension : {".sfmesh", ".glb", ".gltf"}) { +TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][open]") +{ + for (const std::string extension : { ".sfmesh", ".glb", ".gltf" }) { const auto codec = mesh::codec::from_extension(extension); REQUIRE(codec.has_value()); const auto paths = codec.value()->paths(store::NodePath("node")); @@ -224,10 +208,11 @@ TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][op CHECK(unknown.error().operation == store::CodecOperation::Resolve); } -TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store][open]") { +TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store][open]") +{ TemporaryDirectory directory("payload-class"); const std::filesystem::path index_path = directory.path() / "octree.storeindex"; - const store::IndexMetadata index_file{ + const store::IndexMetadata index_file { .index = {}, .layout_id = "flat", .payload_class = "dag.ClusterBatch", @@ -238,15 +223,15 @@ TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store] const auto result = octree::open_index(index_path); REQUIRE_FALSE(result.has_value()); REQUIRE(std::holds_alternative(result.error())); - CHECK(std::get(result.error()).category - == store::CodecErrorCategory::UnsupportedCodec); + CHECK(std::get(result.error()).category == store::CodecErrorCategory::UnsupportedCodec); } -TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") { +TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") +{ TemporaryDirectory directory("open-errors"); const std::filesystem::path index_path = directory.path() / "octree.storeindex"; - store::IndexMetadata index_file{ + store::IndexMetadata index_file { .index = {}, .layout_id = "unknown-layout", .payload_class = std::string(mesh::storage::payload_class), @@ -265,19 +250,16 @@ TEST_CASE("octree opening retains index failures without directory fallback", "[ const auto unknown_codec = octree::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_codec.has_value()); REQUIRE(std::holds_alternative(unknown_codec.error())); - CHECK(std::get(unknown_codec.error()).category - == store::CodecErrorCategory::UnsupportedCodec); + CHECK(std::get(unknown_codec.error()).category == store::CodecErrorCategory::UnsupportedCodec); - const std::array malformed_bytes{0xff, 0x00, 0x01}; + const std::array malformed_bytes { 0xff, 0x00, 0x01 }; REQUIRE(io::write_bytes_to_path(malformed_bytes, index_path).has_value()); const auto malformed = octree::open_folder_indexed(directory.path()); REQUIRE_FALSE(malformed.has_value()); REQUIRE(std::holds_alternative(malformed.error())); - CHECK(std::get(malformed.error()).category - == store::IndexFormatErrorCategory::Malformed); + CHECK(std::get(malformed.error()).category == store::IndexFormatErrorCategory::Malformed); - const auto independent_metadata = - octree::storage::read_store_metadata(directory.path()); + const auto independent_metadata = octree::storage::read_store_metadata(directory.path()); REQUIRE(independent_metadata.has_value()); CHECK(independent_metadata->codec_selector == ".unknown"); } diff --git a/unittests/terrainlib/store_index.cpp b/unittests/terrainlib/store_index.cpp index 454d7948..966ef4d5 100644 --- a/unittests/terrainlib/store_index.cpp +++ b/unittests/terrainlib/store_index.cpp @@ -11,11 +11,8 @@ static_assert(store::HierarchyTraits); static_assert(store::HierarchyTraits); -TEMPLATE_TEST_CASE( - "shared hierarchy index transitions", - "[store][index]", - octree::StoreTraits, - raster_store::StoreTraits) { +TEMPLATE_TEST_CASE("shared hierarchy index transitions", "[store][index]", octree::StoreTraits, raster_store::StoreTraits) +{ using Traits = TestType; using Index = store::Index; @@ -42,40 +39,42 @@ TEMPLATE_TEST_CASE( CHECK(index.empty()); } -TEST_CASE("raster store traits validate tile boundaries", "[store][index][raster]") { +TEST_CASE("raster store traits validate tile boundaries", "[store][index][raster]") +{ using Traits = raster_store::StoreTraits; using Key = Traits::Key; - CHECK(Traits::is_valid(Key{0, {0, 0}})); - CHECK_FALSE(Traits::is_valid(Key{0, {1, 0}})); - CHECK(Traits::is_valid(Key{31, {uint32_t{0x7fffffff}, uint32_t{0x7fffffff}}})); - CHECK_FALSE(Traits::is_valid(Key{31, {uint32_t{0x80000000}, 0}})); - CHECK(Traits::is_valid(Key{32, {0, 0}})); - CHECK(Traits::is_valid(Key{32, {UINT32_MAX, UINT32_MAX}})); - CHECK_FALSE(Traits::is_valid(Key{33, {0, 0}})); + CHECK(Traits::is_valid(Key { 0, { 0, 0 } })); + CHECK_FALSE(Traits::is_valid(Key { 0, { 1, 0 } })); + CHECK(Traits::is_valid(Key { 31, { uint32_t { 0x7fffffff }, uint32_t { 0x7fffffff } } })); + CHECK_FALSE(Traits::is_valid(Key { 31, { uint32_t { 0x80000000 }, 0 } })); + CHECK(Traits::is_valid(Key { 32, { 0, 0 } })); + CHECK(Traits::is_valid(Key { 32, { UINT32_MAX, UINT32_MAX } })); + CHECK_FALSE(Traits::is_valid(Key { 33, { 0, 0 } })); CHECK_FALSE(Traits::parent(Traits::root()).has_value()); - CHECK_FALSE(Traits::children(Key{32, {UINT32_MAX, UINT32_MAX}}).has_value()); + CHECK_FALSE(Traits::children(Key { 32, { UINT32_MAX, UINT32_MAX } }).has_value()); - const Key level_31{31, {uint32_t{0x7fffffff}, uint32_t{0x7fffffff}}}; + const Key level_31 { 31, { uint32_t { 0x7fffffff }, uint32_t { 0x7fffffff } } }; const auto children = Traits::children(level_31); REQUIRE(children.has_value()); - CHECK(children->at(0) == Key{32, {UINT32_MAX - 1, UINT32_MAX - 1}}); - CHECK(children->at(1) == Key{32, {UINT32_MAX, UINT32_MAX - 1}}); - CHECK(children->at(2) == Key{32, {UINT32_MAX - 1, UINT32_MAX}}); - CHECK(children->at(3) == Key{32, {UINT32_MAX, UINT32_MAX}}); + CHECK(children->at(0) == Key { 32, { UINT32_MAX - 1, UINT32_MAX - 1 } }); + CHECK(children->at(1) == Key { 32, { UINT32_MAX, UINT32_MAX - 1 } }); + CHECK(children->at(2) == Key { 32, { UINT32_MAX - 1, UINT32_MAX } }); + CHECK(children->at(3) == Key { 32, { UINT32_MAX, UINT32_MAX } }); } -TEST_CASE("shared index rejects invalid keys", "[store][index][raster]") { +TEST_CASE("shared index rejects invalid keys", "[store][index][raster]") +{ using Traits = raster_store::StoreTraits; using Key = Traits::Key; store::Index index; - const Key invalid{4, {16, 0}}; + const Key invalid { 4, { 16, 0 } }; - CHECK(index.get(invalid) == std::unexpected(store::InvalidKey{invalid})); - CHECK(index.add(invalid) == std::unexpected(store::InvalidKey{invalid})); - CHECK(index.remove(invalid) == std::unexpected(store::InvalidKey{invalid})); - CHECK(index.is_present(invalid) == std::unexpected(store::InvalidKey{invalid})); - CHECK(index.is_absent(invalid) == std::unexpected(store::InvalidKey{invalid})); - CHECK(index.is(store::NodeStatus::Leaf, invalid) == std::unexpected(store::InvalidKey{invalid})); + CHECK(index.get(invalid) == std::unexpected(store::InvalidKey { invalid })); + CHECK(index.add(invalid) == std::unexpected(store::InvalidKey { invalid })); + CHECK(index.remove(invalid) == std::unexpected(store::InvalidKey { invalid })); + CHECK(index.is_present(invalid) == std::unexpected(store::InvalidKey { invalid })); + CHECK(index.is_absent(invalid) == std::unexpected(store::InvalidKey { invalid })); + CHECK(index.is(store::NodeStatus::Leaf, invalid) == std::unexpected(store::InvalidKey { invalid })); } diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp index 4d0db771..a67bc80b 100644 --- a/unittests/terrainlib/store_layout.cpp +++ b/unittests/terrainlib/store_layout.cpp @@ -7,44 +7,44 @@ #include "octree/store_layout/Mappings.h" #include "store/Layout.h" -TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layout]") { +TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layout]") +{ const std::filesystem::path dataset = "dataset"; mesh::codec::SfMesh sfmesh; - SECTION("flat") { + SECTION("flat") + { const store::Layout layout(dataset / "sf-flat", octree::store_layout::flat()); const store::NodePath node_path = layout.node_path(octree::Id::root()); CHECK(node_path.path() == dataset / "sf-flat/0-0"); - CHECK(sfmesh.paths(node_path) == std::vector{dataset / "sf-flat/0-0.sfmesh"}); + CHECK(sfmesh.paths(node_path) == std::vector { dataset / "sf-flat/0-0.sfmesh" }); CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); } - SECTION("level and coordinate directories") { - const store::Layout layout( - dataset / "sf-coordinates", - octree::store_layout::level_and_coordinate_directories()); + SECTION("level and coordinate directories") + { + const store::Layout layout(dataset / "sf-coordinates", octree::store_layout::level_and_coordinate_directories()); const octree::Id child = octree::Id::root().child(2).value(); const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); - CHECK(sfmesh.paths(layout.node_path(child)) - == std::vector{dataset / "sf-coordinates/1/0/1/0.sfmesh"}); - CHECK(sfmesh.paths(layout.node_path(deep)) - == std::vector{dataset / "sf-coordinates/2/3/1/3.sfmesh"}); + CHECK(sfmesh.paths(layout.node_path(child)) == std::vector { dataset / "sf-coordinates/1/0/1/0.sfmesh" }); + CHECK(sfmesh.paths(layout.node_path(deep)) == std::vector { dataset / "sf-coordinates/2/3/1/3.sfmesh" }); CHECK(layout.key_from_node_path(layout.node_path(child)) == child); CHECK(layout.key_from_node_path(layout.node_path(deep)) == deep); } } -TEST_CASE("octree mapping lookup is explicit", "[store][layout]") { +TEST_CASE("octree mapping lookup is explicit", "[store][layout]") +{ REQUIRE(octree::store_layout::from_id("flat").has_value()); CHECK(octree::store_layout::from_id("flat")->id == "flat"); REQUIRE(octree::store_layout::from_id("level_and_coordinate_directories").has_value()); - CHECK(octree::store_layout::from_id("level_and_coordinate_directories")->id - == "level_and_coordinate_directories"); + CHECK(octree::store_layout::from_id("level_and_coordinate_directories")->id == "level_and_coordinate_directories"); CHECK_FALSE(octree::store_layout::from_id("unknown").has_value()); CHECK(octree::store_layout::all().size() == 2); } -TEST_CASE("extensionless octree mappings validate complete paths", "[store][layout]") { +TEST_CASE("extensionless octree mappings validate complete paths", "[store][layout]") +{ const auto flat = octree::store_layout::flat(); CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2.sfmesh")).has_value()); CHECK_FALSE(flat.node_path_to_key(store::NodePath("nested/1-2")).has_value()); @@ -57,13 +57,14 @@ TEST_CASE("extensionless octree mappings validate complete paths", "[store][layo CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("22/0/0/0")).has_value()); } -TEST_CASE("runtime mesh codecs own their filename endings", "[store][layout][codec]") { +TEST_CASE("runtime mesh codecs own their filename endings", "[store][layout][codec]") +{ const store::NodePath path("12/34/56/78"); mesh::codec::SfMesh sfmesh; mesh::codec::Gltf binary(mesh::codec::GltfContainer::Binary); mesh::codec::Gltf json(mesh::codec::GltfContainer::Json); - CHECK(sfmesh.paths(path) == std::vector{"12/34/56/78.sfmesh"}); - CHECK(binary.paths(path) == std::vector{"12/34/56/78.glb"}); - CHECK(json.paths(path) == std::vector{"12/34/56/78.gltf"}); + CHECK(sfmesh.paths(path) == std::vector { "12/34/56/78.sfmesh" }); + CHECK(binary.paths(path) == std::vector { "12/34/56/78.glb" }); + CHECK(json.paths(path) == std::vector { "12/34/56/78.gltf" }); } diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index 6a06b075..bc424fa3 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -21,34 +21,33 @@ namespace { class TemporaryDirectory { public: - explicit TemporaryDirectory(const std::string_view label) { + explicit TemporaryDirectory(const std::string_view label) + { static std::atomic_uint64_t counter = 0; const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - _path = std::filesystem::temp_directory_path() - / ("atb-store-storage-" + std::string(label) + "-" - + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(_path)); + m_path = std::filesystem::temp_directory_path() + / ("atb-store-storage-" + std::string(label) + "-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); + REQUIRE(std::filesystem::create_directories(m_path)); } - ~TemporaryDirectory() { + ~TemporaryDirectory() + { std::error_code error; - std::filesystem::remove_all(_path, error); - } - const std::filesystem::path &path() const { - return _path; + std::filesystem::remove_all(m_path, error); } + const std::filesystem::path& path() const { return m_path; } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -store::NodePath tile_to_path(const radix::tile::Id &key) { - return store::NodePath( - std::to_string(key.zoom_level) + "/" + std::to_string(key.coords.x) - + "/" + std::to_string(key.coords.y)); +store::NodePath tile_to_path(const radix::tile::Id& key) +{ + return store::NodePath(std::to_string(key.zoom_level) + "/" + std::to_string(key.coords.x) + "/" + std::to_string(key.coords.y)); } -std::optional path_to_tile(const store::NodePath &node_path) { - const std::filesystem::path &path = node_path.path(); +std::optional path_to_tile(const store::NodePath& node_path) +{ + const std::filesystem::path& path = node_path.path(); if (path.is_absolute() || std::distance(path.begin(), path.end()) != 3) { return std::nullopt; } @@ -61,29 +60,30 @@ std::optional path_to_tile(const store::NodePath &node_path) { if (!zoom.has_value() || !x.has_value() || !y.has_value()) { return std::nullopt; } - const radix::tile::Id key{zoom.value(), {x.value(), y.value()}}; - return raster_store::StoreTraits::is_valid(key) - ? std::optional(key) - : std::nullopt; + const radix::tile::Id key { zoom.value(), { x.value(), y.value() } }; + return raster_store::StoreTraits::is_valid(key) ? std::optional(key) : std::nullopt; } -template +template store::PathMapping test_mapping(); -template<> -store::PathMapping test_mapping() { +template <> +store::PathMapping test_mapping() +{ return octree::store_layout::flat(); } -template<> -store::PathMapping test_mapping() { - return {"test_tiles", tile_to_path, path_to_tile}; +template <> +store::PathMapping test_mapping() +{ + return { "test_tiles", tile_to_path, path_to_tile }; } -std::expected read_int(const std::filesystem::path &path) { +std::expected read_int(const std::filesystem::path& path) +{ auto bytes = io::read_bytes_from_path(path); if (!bytes || bytes->size() != sizeof(int)) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::Io, "test integer read failed", @@ -94,15 +94,11 @@ std::expected read_int(const std::filesystem::path &path return value; } -std::expected write_int( - const int value, - const std::filesystem::path &path) +std::expected write_int(const int value, const std::filesystem::path& path) { - const auto bytes = std::span( - reinterpret_cast(&value), - sizeof(value)); + const auto bytes = std::span(reinterpret_cast(&value), sizeof(value)); if (!io::write_bytes_to_path(bytes, path)) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Io, "test integer write failed", @@ -113,50 +109,44 @@ std::expected write_int( class IntCodec final : public store::Codec { public: - std::vector paths( - const store::NodePath &node_path) const override + std::vector paths(const store::NodePath& node_path) const override { std::filesystem::path path = node_path.path(); path += ".data"; - return {path}; + return { path }; } - std::expected read( - const store::NodePath &node_path) const override - { - return read_int(paths(node_path).front()); - } + std::expected read(const store::NodePath& node_path) const override { return read_int(paths(node_path).front()); } - std::expected write( - const store::NodePath &node_path, - const int &value) const override + std::expected write(const store::NodePath& node_path, const int& value) const override { return write_int(value, paths(node_path).front()); } }; -template -store::Storage make_storage(const std::filesystem::path &path) { - return store::Storage(store::RawStorage( - store::Layout(path, test_mapping()), - std::make_unique())); +template +store::Storage make_storage(const std::filesystem::path& path) +{ + return store::Storage( + store::RawStorage(store::Layout(path, test_mapping()), std::make_unique())); } class MultiFileCodec final : public store::Codec { public: - std::vector paths(const store::NodePath &node_path) const override { + std::vector paths(const store::NodePath& node_path) const override + { std::filesystem::path first = node_path.path(); std::filesystem::path second = node_path.path(); first += ".data"; second += ".metadata"; - return {first, second}; + return { first, second }; } - std::expected read( - const store::NodePath &node_path) const override { + std::expected read(const store::NodePath& node_path) const override + { const auto value = read_int(paths(node_path).front()); if (!value) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::Io, "multi-file read failed", @@ -165,13 +155,11 @@ class MultiFileCodec final : public store::Codec { return value.value(); } - std::expected write( - const store::NodePath &node_path, - const int &value) const override { + std::expected write(const store::NodePath& node_path, const int& value) const override + { const auto node_paths = paths(node_path); - if (!write_int(value, node_paths[0]) - || !write_int(value + 1, node_paths[1])) { - return std::unexpected(store::CodecError{ + if (!write_int(value, node_paths[0]) || !write_int(value + 1, node_paths[1])) { + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Io, "multi-file write failed", @@ -184,19 +172,21 @@ class MultiFileCodec final : public store::Codec { class ConfigurableCodec final : public store::Codec { public: explicit ConfigurableCodec(std::string extension) - : extension(std::move(extension)) {} + : extension(std::move(extension)) + { + } - std::vector paths( - const store::NodePath &node_path) const override { + std::vector paths(const store::NodePath& node_path) const override + { std::filesystem::path path = node_path.path(); path += extension; - return {path}; + return { path }; } - std::expected read( - const store::NodePath &node_path) const override { + std::expected read(const store::NodePath& node_path) const override + { if (fail_read) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::Domain, "injected decode failure", @@ -204,7 +194,7 @@ class ConfigurableCodec final : public store::Codec { } const auto value = read_int(paths(node_path).front()); if (!value) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Read, store::CodecErrorCategory::Io, "configurable read failed", @@ -213,18 +203,17 @@ class ConfigurableCodec final : public store::Codec { return value.value(); } - std::expected write( - const store::NodePath &node_path, - const int &value) const override { + std::expected write(const store::NodePath& node_path, const int& value) const override + { if (fail_write) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Domain, "injected encode failure", }); } if (!write_int(value, paths(node_path).front())) { - return std::unexpected(store::CodecError{ + return std::unexpected(store::CodecError { store::CodecOperation::Write, store::CodecErrorCategory::Io, "configurable write failed", @@ -240,34 +229,30 @@ class ConfigurableCodec final : public store::Codec { std::atomic_uint32_t index_writes = 0; -std::expected, store::IndexFormatError> -unused_index_read(const std::filesystem::path &path) { - return std::unexpected(store::IndexFormatError{ +std::expected, store::IndexFormatError> unused_index_read(const std::filesystem::path& path) +{ + return std::unexpected(store::IndexFormatError { store::IndexFormatErrorCategory::Open, path, "not used", }); } -std::expected count_index_write( - const std::filesystem::path &, - const store::IndexMetadata &) { +std::expected count_index_write(const std::filesystem::path&, const store::IndexMetadata&) +{ ++index_writes; return {}; } -std::optional> fake_mapping_from_id( - const std::string_view id) { - return id == "flat" - ? std::optional>(octree::store_layout::flat()) - : std::nullopt; +std::optional> fake_mapping_from_id(const std::string_view id) +{ + return id == "flat" ? std::optional>(octree::store_layout::flat()) : std::nullopt; } -store::PathMapping fake_default_mapping() { - return octree::store_layout::flat(); -} +store::PathMapping fake_default_mapping() { return octree::store_layout::flat(); } -store::IndexFormat counting_index_format() { +store::IndexFormat counting_index_format() +{ return { "octree.storeindex", unused_index_read, @@ -278,16 +263,13 @@ store::IndexFormat counting_index_format() { } store::IndexedStorage make_indexed_storage( - const std::filesystem::path &path, - std::unique_ptr> codec = - std::make_unique()) { + const std::filesystem::path& path, std::unique_ptr> codec = std::make_unique()) +{ using Storage = store::Storage; return store::IndexedStorage( - store::RawStorage( - store::Layout(path, octree::store_layout::flat()), - std::move(codec)), - store::Index{}, - Storage::Persistence{ + store::RawStorage(store::Layout(path, octree::store_layout::flat()), std::move(codec)), + store::Index {}, + Storage::Persistence { counting_index_format(), path / "octree.storeindex", "flat", @@ -298,11 +280,8 @@ store::IndexedStorage make_indexed_storage( } // namespace -TEMPLATE_TEST_CASE( - "shared raw storage has no hidden octree dependency", - "[store][storage]", - octree::StoreTraits, - raster_store::StoreTraits) { +TEMPLATE_TEST_CASE("shared raw storage has no hidden octree dependency", "[store][storage]", octree::StoreTraits, raster_store::StoreTraits) +{ using Traits = TestType; TemporaryDirectory directory("traits"); auto storage = make_storage(directory.path()); @@ -317,22 +296,20 @@ TEMPLATE_TEST_CASE( CHECK_FALSE(storage.has(root).value()); } -TEST_CASE("shared storage rejects invalid 2D keys", "[store][storage]") { +TEST_CASE("shared storage rejects invalid 2D keys", "[store][storage]") +{ TemporaryDirectory directory("invalid-key"); auto storage = make_storage(directory.path()); - const radix::tile::Id invalid{3, {8, 0}}; - - CHECK(std::holds_alternative>( - storage.load(invalid).error())); - CHECK(std::holds_alternative>( - storage.save(invalid, 1).error())); - CHECK(std::holds_alternative>( - storage.has(invalid).error())); - CHECK(std::holds_alternative>( - storage.remove(invalid).error())); + const radix::tile::Id invalid { 3, { 8, 0 } }; + + CHECK(std::holds_alternative>(storage.load(invalid).error())); + CHECK(std::holds_alternative>(storage.save(invalid, 1).error())); + CHECK(std::holds_alternative>(storage.has(invalid).error())); + CHECK(std::holds_alternative>(storage.remove(invalid).error())); } -TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") { +TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") +{ TemporaryDirectory directory("overwrite"); auto storage = make_storage(directory.path()); const octree::Id root = octree::Id::root(); @@ -348,11 +325,11 @@ TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") { CHECK(storage.load(root).value() == 2); } -TEST_CASE("raw storage requires and removes every codec path", "[store][storage]") { +TEST_CASE("raw storage requires and removes every codec path", "[store][storage]") +{ TemporaryDirectory directory("multi-file"); store::RawStorage storage( - store::Layout(directory.path(), octree::store_layout::flat()), - std::make_unique()); + store::Layout(directory.path(), octree::store_layout::flat()), std::make_unique()); const octree::Id root = octree::Id::root(); const auto paths = storage.paths(root).value(); REQUIRE(paths.size() == 2); @@ -366,7 +343,8 @@ TEST_CASE("raw storage requires and removes every codec path", "[store][storage] CHECK_FALSE(std::filesystem::exists(paths[1])); } -TEST_CASE("storage moves transfer and finalize dirty index state", "[store][storage]") { +TEST_CASE("storage moves transfer and finalize dirty index state", "[store][storage]") +{ index_writes = 0; TemporaryDirectory first_directory("move-first"); TemporaryDirectory second_directory("move-second"); @@ -393,10 +371,12 @@ TEST_CASE("storage moves transfer and finalize dirty index state", "[store][stor CHECK(index_writes == 3); } -TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][copy]") { +TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][copy]") +{ const octree::Id root = octree::Id::root(); - SECTION("one file") { + SECTION("one file") + { TemporaryDirectory source_directory("copy-one-source"); TemporaryDirectory target_directory("copy-one-target"); auto source = make_indexed_storage(source_directory.path()); @@ -406,18 +386,15 @@ TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][co REQUIRE(target.copy_from(root, source).has_value()); REQUIRE(target.has(root).value()); CHECK(target.load(root).value() == 41); - CHECK(std::filesystem::equivalent( - source.paths(root)->front(), - target.paths(root)->front())); + CHECK(std::filesystem::equivalent(source.paths(root)->front(), target.paths(root)->front())); } - SECTION("multiple files") { + SECTION("multiple files") + { TemporaryDirectory source_directory("copy-many-source"); TemporaryDirectory target_directory("copy-many-target"); - auto source = make_indexed_storage( - source_directory.path(), std::make_unique()); - auto target = make_indexed_storage( - target_directory.path(), std::make_unique()); + auto source = make_indexed_storage(source_directory.path(), std::make_unique()); + auto target = make_indexed_storage(target_directory.path(), std::make_unique()); REQUIRE(source.save(root, 42).has_value()); REQUIRE(target.copy_from(root, source).has_value()); @@ -430,12 +407,12 @@ TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][co } } -TEST_CASE("copy_from re-encodes differing path lists", "[store][storage][copy]") { +TEST_CASE("copy_from re-encodes differing path lists", "[store][storage][copy]") +{ const octree::Id root = octree::Id::root(); TemporaryDirectory source_directory("copy-reencode-source"); TemporaryDirectory target_directory("copy-reencode-target"); - auto source = make_indexed_storage( - source_directory.path(), std::make_unique()); + auto source = make_indexed_storage(source_directory.path(), std::make_unique()); auto target = make_indexed_storage(target_directory.path()); REQUIRE(source.save(root, 43).has_value()); @@ -443,12 +420,11 @@ TEST_CASE("copy_from re-encodes differing path lists", "[store][storage][copy]") CHECK(target.load(root).value() == 43); CHECK(target.paths(root)->size() == 1); CHECK(source.paths(root)->size() == 2); - CHECK_FALSE(std::filesystem::equivalent( - source.paths(root)->front(), - target.paths(root)->front())); + CHECK_FALSE(std::filesystem::equivalent(source.paths(root)->front(), target.paths(root)->front())); } -TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") { +TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") +{ const octree::Id root = octree::Id::root(); TemporaryDirectory source_directory("copy-overwrite-source"); TemporaryDirectory target_directory("copy-overwrite-target"); @@ -465,21 +441,16 @@ TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") { target.settings().allow_overwrite = true; REQUIRE(target.copy_from(root, source).has_value()); CHECK(target.load(root).value() == 44); - CHECK(std::filesystem::equivalent( - source.paths(root)->front(), - target.paths(root)->front())); + CHECK(std::filesystem::equivalent(source.paths(root)->front(), target.paths(root)->front())); } -TEST_CASE( - "failed multi-file overwrite leaves the target node unindexed", - "[store][storage][copy]") { +TEST_CASE("failed multi-file overwrite leaves the target node unindexed", "[store][storage][copy]") +{ const octree::Id root = octree::Id::root(); TemporaryDirectory source_directory("copy-partial-source"); TemporaryDirectory target_directory("copy-partial-target"); - auto source = make_indexed_storage( - source_directory.path(), std::make_unique()); - auto target = make_indexed_storage( - target_directory.path(), std::make_unique()); + auto source = make_indexed_storage(source_directory.path(), std::make_unique()); + auto target = make_indexed_storage(target_directory.path(), std::make_unique()); REQUIRE(source.save(root, 45).has_value()); REQUIRE(target.save(root, 1).has_value()); target.settings().allow_overwrite = true; @@ -494,16 +465,16 @@ TEST_CASE( CHECK(std::holds_alternative(copied.error())); CHECK_FALSE(target.has(root).value()); CHECK_FALSE(target.index().get(root).value().has_value()); - CHECK(std::filesystem::equivalent( - source.paths(root)->front(), - target_paths.front())); + CHECK(std::filesystem::equivalent(source.paths(root)->front(), target_paths.front())); CHECK(std::filesystem::is_directory(target_paths[1])); } -TEST_CASE("copy_from propagates source and codec failures", "[store][storage][copy]") { +TEST_CASE("copy_from propagates source and codec failures", "[store][storage][copy]") +{ const octree::Id root = octree::Id::root(); - SECTION("missing source") { + SECTION("missing source") + { TemporaryDirectory source_directory("copy-missing-source"); TemporaryDirectory target_directory("copy-missing-target"); auto source = make_indexed_storage(source_directory.path()); @@ -513,22 +484,19 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - CHECK(std::holds_alternative>( - copied.error())); + CHECK(std::holds_alternative>(copied.error())); CHECK(target.has(root).value()); CHECK(target.load(root).value() == 7); } - SECTION("decode failure") { + SECTION("decode failure") + { TemporaryDirectory source_directory("copy-decode-source"); TemporaryDirectory target_directory("copy-decode-target"); auto source_codec = std::make_unique(".source"); - ConfigurableCodec *source_codec_ptr = source_codec.get(); - auto source = make_indexed_storage( - source_directory.path(), std::move(source_codec)); - auto target = make_indexed_storage( - target_directory.path(), - std::make_unique(".target")); + ConfigurableCodec* source_codec_ptr = source_codec.get(); + auto source = make_indexed_storage(source_directory.path(), std::move(source_codec)); + auto target = make_indexed_storage(target_directory.path(), std::make_unique(".target")); REQUIRE(source.save(root, 46).has_value()); REQUIRE(target.save(root, 7).has_value()); target.settings().allow_overwrite = true; @@ -537,22 +505,19 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); REQUIRE(std::holds_alternative(copied.error())); - CHECK(std::get(copied.error()).operation - == store::CodecOperation::Read); + CHECK(std::get(copied.error()).operation == store::CodecOperation::Read); CHECK(target.has(root).value()); CHECK(target.load(root).value() == 7); } - SECTION("encode failure") { + SECTION("encode failure") + { TemporaryDirectory source_directory("copy-encode-source"); TemporaryDirectory target_directory("copy-encode-target"); - auto source = make_indexed_storage( - source_directory.path(), - std::make_unique(".source")); + auto source = make_indexed_storage(source_directory.path(), std::make_unique(".source")); auto target_codec = std::make_unique(".target"); - ConfigurableCodec *target_codec_ptr = target_codec.get(); - auto target = make_indexed_storage( - target_directory.path(), std::move(target_codec)); + ConfigurableCodec* target_codec_ptr = target_codec.get(); + auto target = make_indexed_storage(target_directory.path(), std::move(target_codec)); REQUIRE(source.save(root, 47).has_value()); REQUIRE(target.save(root, 7).has_value()); target.settings().allow_overwrite = true; @@ -561,8 +526,7 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); REQUIRE(std::holds_alternative(copied.error())); - CHECK(std::get(copied.error()).operation - == store::CodecOperation::Write); + CHECK(std::get(copied.error()).operation == store::CodecOperation::Write); CHECK_FALSE(target.has(root).value()); } } diff --git a/unittests/terrainlib/store_traverse.cpp b/unittests/terrainlib/store_traverse.cpp index 43992ed1..9eba2bc0 100644 --- a/unittests/terrainlib/store_traverse.cpp +++ b/unittests/terrainlib/store_traverse.cpp @@ -7,11 +7,8 @@ #include "raster_store/StoreTraits.h" #include "store/traverse.h" -TEMPLATE_TEST_CASE( - "shared hierarchy traversal preserves trait child order", - "[store][traverse]", - octree::StoreTraits, - raster_store::StoreTraits) { +TEMPLATE_TEST_CASE("shared hierarchy traversal preserves trait child order", "[store][traverse]", octree::StoreTraits, raster_store::StoreTraits) +{ using Traits = TestType; using Key = typename Traits::Key; @@ -27,42 +24,31 @@ TEMPLATE_TEST_CASE( REQUIRE(index.add(grandchild).has_value()); std::vector depth_first; - REQUIRE(store::traverse( - index, - [&](const Key &key, const store::NodeStatus) { depth_first.push_back(key); }) - .has_value()); - CHECK(depth_first == std::vector{root, first, grandchild, second}); + REQUIRE(store::traverse(index, [&](const Key& key, const store::NodeStatus) { depth_first.push_back(key); }).has_value()); + CHECK(depth_first == std::vector { root, first, grandchild, second }); std::vector breadth_first; REQUIRE(store::traverse( - index, - [&](const Key &key, const store::NodeStatus) { breadth_first.push_back(key); }, - store::AlwaysRefine{}, - root, - store::TraversalOrder::BreadthFirst) - .has_value()); - CHECK(breadth_first == std::vector{root, first, second, grandchild}); + index, + [&](const Key& key, const store::NodeStatus) { breadth_first.push_back(key); }, + store::AlwaysRefine {}, + root, + store::TraversalOrder::BreadthFirst) + .has_value()); + CHECK(breadth_first == std::vector { root, first, second, grandchild }); std::vector explicit_root; - REQUIRE(store::traverse( - index, - [&](const Key &key, const store::NodeStatus) { explicit_root.push_back(key); }, - store::AlwaysRefine{}, - first) - .has_value()); - CHECK(explicit_root == std::vector{first, grandchild}); + REQUIRE(store::traverse(index, [&](const Key& key, const store::NodeStatus) { explicit_root.push_back(key); }, store::AlwaysRefine {}, first).has_value()); + CHECK(explicit_root == std::vector { first, grandchild }); } -TEST_CASE("shared traversal rejects an invalid explicit root", "[store][traverse][raster]") { +TEST_CASE("shared traversal rejects an invalid explicit root", "[store][traverse][raster]") +{ using Traits = raster_store::StoreTraits; using Key = Traits::Key; - const Key invalid{1, {2, 0}}; + const Key invalid { 1, { 2, 0 } }; store::Index index; - const auto result = store::traverse( - index, - [](const Key &, const store::NodeStatus) {}, - store::AlwaysRefine{}, - invalid); - CHECK(result == std::unexpected(store::InvalidKey{invalid})); + const auto result = store::traverse(index, [](const Key&, const store::NodeStatus) { }, store::AlwaysRefine {}, invalid); + CHECK(result == std::unexpected(store::InvalidKey { invalid })); } diff --git a/unittests/tile_downloader/downloader.cpp b/unittests/tile_downloader/downloader.cpp index 083ecd49..a2dcd67c 100644 --- a/unittests/tile_downloader/downloader.cpp +++ b/unittests/tile_downloader/downloader.cpp @@ -11,55 +11,52 @@ namespace { class TemporaryPyramid { public: explicit TemporaryPyramid(std::string_view name) - : _path(std::filesystem::temp_directory_path() / name) { + : m_path(std::filesystem::temp_directory_path() / name) + { std::error_code error; - std::filesystem::remove_all(_path, error); - std::filesystem::create_directories(_path); + std::filesystem::remove_all(m_path, error); + std::filesystem::create_directories(m_path); } - ~TemporaryPyramid() { + ~TemporaryPyramid() + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); } - [[nodiscard]] const std::filesystem::path &path() const { - return _path; - } + [[nodiscard]] const std::filesystem::path& path() const { return m_path; } - [[nodiscard]] std::filesystem::path tile_path(const radix::tile::Id &tile) const { - return google_tile_path(_path, tile, ".jpeg"); - } + [[nodiscard]] std::filesystem::path tile_path(const radix::tile::Id& tile) const { return google_tile_path(m_path, tile, ".jpeg"); } - void create_pending(const radix::tile::Id &tile) const { + void create_pending(const radix::tile::Id& tile) const + { const auto path = tile_path(tile); std::filesystem::create_directories(path.parent_path()); - write_file_children_pending(path, std::vector{'t', 'i', 'l', 'e'}); + write_file_children_pending(path, std::vector { 't', 'i', 'l', 'e' }); } - void create_complete(const radix::tile::Id &tile) const { + void create_complete(const radix::tile::Id& tile) const + { create_pending(tile); mark_tile_children_complete(tile_path(tile)); } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -const TileUrlBuilder missing_file_url({ - "file:///definitely-missing-atb-tile/{zoom}/{x}/{y}.jpeg", - TileYDirection::Down -}); +const TileUrlBuilder missing_file_url({ "file:///definitely-missing-atb-tile/{zoom}/{x}/{y}.jpeg", TileYDirection::Down }); -} +} // namespace TEST_CASE("tile downloader promotes parents after completed children") { const TemporaryPyramid pyramid("atb-downloader-complete-pyramid"); - const radix::tile::Id root{0, {0, 0}}; + const radix::tile::Id root { 0, { 0, 0 } }; const auto children = root.children(); pyramid.create_pending(root); - for (const auto &child : children) { + for (const auto& child : children) { pyramid.create_pending(child); } @@ -69,7 +66,7 @@ TEST_CASE("tile downloader promotes parents after completed children") REQUIRE(std::filesystem::exists(pyramid.tile_path(root))); CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); - for (const auto &child : children) { + for (const auto& child : children) { CHECK(std::filesystem::exists(pyramid.tile_path(child))); CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(child)))); } @@ -83,7 +80,7 @@ TEST_CASE("tile downloader promotes parents after completed children") TEST_CASE("tile downloader leaves ancestors pending after a child failure") { const TemporaryPyramid pyramid("atb-downloader-failed-pyramid"); - const radix::tile::Id root{0, {0, 0}}; + const radix::tile::Id root { 0, { 0, 0 } }; const auto children = root.children(); pyramid.create_pending(root); diff --git a/unittests/tile_downloader/http_client.cpp b/unittests/tile_downloader/http_client.cpp index 61311a8c..7b036cf6 100644 --- a/unittests/tile_downloader/http_client.cpp +++ b/unittests/tile_downloader/http_client.cpp @@ -12,39 +12,39 @@ namespace { class TemporaryFile { public: TemporaryFile() - : _path(std::filesystem::temp_directory_path() / "atb-http-client-test.txt") { - std::ofstream output(_path, std::ios::binary); + : m_path(std::filesystem::temp_directory_path() / "atb-http-client-test.txt") + { + std::ofstream output(m_path, std::ios::binary); output << "response body"; REQUIRE(output); } - ~TemporaryFile() { + ~TemporaryFile() + { std::error_code error; - std::filesystem::remove(_path, error); + std::filesystem::remove(m_path, error); } - [[nodiscard]] std::string url() const { - return "file://" + _path.string(); - } + [[nodiscard]] std::string url() const { return "file://" + m_path.string(); } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; class CallbackError : public std::runtime_error { public: CallbackError() - : std::runtime_error("callback failed") {} + : std::runtime_error("callback failed") + { + } }; -} +} // namespace TEST_CASE("http client propagates response writer exceptions") { const TemporaryFile source; - HttpClient client([](std::vector &, const char *, size_t) { - throw CallbackError(); - }); + HttpClient client([](std::vector&, const char*, size_t) { throw CallbackError(); }); CHECK_THROWS_AS(client.get(source.url()), CallbackError); } @@ -54,9 +54,5 @@ TEST_CASE("http client propagates progress callback exceptions") const TemporaryFile source; HttpClient client; - CHECK_THROWS_AS( - client.get(source.url(), [](double) { - throw CallbackError(); - }), - CallbackError); + CHECK_THROWS_AS(client.get(source.url(), [](double) { throw CallbackError(); }), CallbackError); } diff --git a/unittests/tile_downloader/logger.cpp b/unittests/tile_downloader/logger.cpp index c41ecb4c..5a31d33a 100644 --- a/unittests/tile_downloader/logger.cpp +++ b/unittests/tile_downloader/logger.cpp @@ -28,6 +28,6 @@ TEST_CASE("tile logger session can finish normally") TileLogger logger(0); auto session = logger.start(); - logger.skipped(radix::tile::Id{0, {0, 0}}); + logger.skipped(radix::tile::Id { 0, { 0, 0 } }); session.finish(); } diff --git a/unittests/tile_downloader/write_file.cpp b/unittests/tile_downloader/write_file.cpp index 2773c4ed..07c12ebb 100644 --- a/unittests/tile_downloader/write_file.cpp +++ b/unittests/tile_downloader/write_file.cpp @@ -14,42 +14,38 @@ namespace { class TemporaryOutput { public: explicit TemporaryOutput(std::string_view name) - : _path(std::filesystem::temp_directory_path() / name) { + : m_path(std::filesystem::temp_directory_path() / name) + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); std::filesystem::remove(partial_path(), error); std::filesystem::remove(pending_path(), error); } - ~TemporaryOutput() { + ~TemporaryOutput() + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); std::filesystem::remove(partial_path(), error); std::filesystem::remove(pending_path(), error); } - [[nodiscard]] const std::filesystem::path &path() const { - return _path; - } + [[nodiscard]] const std::filesystem::path& path() const { return m_path; } - [[nodiscard]] std::filesystem::path partial_path() const { - return partial_tile_path(_path); - } + [[nodiscard]] std::filesystem::path partial_path() const { return partial_tile_path(m_path); } - [[nodiscard]] std::filesystem::path pending_path() const { - return children_pending_tile_path(_path); - } + [[nodiscard]] std::filesystem::path pending_path() const { return children_pending_tile_path(m_path); } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -} +} // namespace TEST_CASE("checked file writer persists the complete response") { const TemporaryOutput output("atb-write-file-success.bin"); - const std::vector expected{'t', 'i', 'l', 'e'}; + const std::vector expected { 't', 'i', 'l', 'e' }; write_file_children_pending(output.path(), expected); @@ -59,7 +55,7 @@ TEST_CASE("checked file writer persists the complete response") std::ifstream input(output.pending_path(), std::ios::binary); const auto begin = std::istreambuf_iterator(input); - const std::vector actual(begin, std::istreambuf_iterator{}); + const std::vector actual(begin, std::istreambuf_iterator {}); CHECK(actual == expected); mark_tile_children_complete(output.path()); @@ -71,11 +67,9 @@ TEST_CASE("checked file writer preserves the final path when promotion fails") { const TemporaryOutput output("atb-write-file-promotion-failure"); REQUIRE(std::filesystem::create_directory(output.path())); - write_file_children_pending(output.path(), std::vector{'t', 'i', 'l', 'e'}); + write_file_children_pending(output.path(), std::vector { 't', 'i', 'l', 'e' }); - CHECK_THROWS_AS( - mark_tile_children_complete(output.path()), - std::filesystem::filesystem_error); + CHECK_THROWS_AS(mark_tile_children_complete(output.path()), std::filesystem::filesystem_error); CHECK(std::filesystem::is_directory(output.path())); CHECK(std::filesystem::exists(output.pending_path())); diff --git a/unittests/tilebuilder/dataset_reading.cpp b/unittests/tilebuilder/dataset_reading.cpp index ba2804db..d6f23416 100644 --- a/unittests/tilebuilder/dataset_reading.cpp +++ b/unittests/tilebuilder/dataset_reading.cpp @@ -122,7 +122,7 @@ TEST_CASE("reading") const auto s = std::string("/austria/").length(); const auto l = dataset_name.length() - std::string(".tif").length() - s; const auto debug_file = fmt::format("./heights_{}_srs{}_{}.png", test_name, test_srs, dataset_name.substr(s, l)); - image::debugOut(heights, debug_file); + image::debug_out(heights, debug_file); } const auto heights = reader.read(srs_bounds, 100, 111); @@ -171,7 +171,7 @@ TEST_CASE("reading") const auto ref_reader = DatasetReader(ref_dataset, geodetic_srs, 1); const auto ref_heights = ref_reader.read(ref_bounds, render_width, render_height); if (ALP_UNITTESTS_DEBUG_IMAGES) - image::debugOut(ref_heights, fmt::format("./heights_ref.png")); + image::debug_out(ref_heights, fmt::format("./heights_ref.png")); for (std::string dataset_name : datasets) { const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR + std::string(dataset_name)).value(); @@ -183,12 +183,12 @@ TEST_CASE("reading") const auto l = dataset_name.length() - std::string(".tif").length() - s; if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debugOut(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))); + image::debug_out(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(ref_heights.begin(), ref_heights.end(), heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); const auto path = fmt::format("./diffs_{}_{}.png", test_name, dataset_name.substr(s, l)); - image::debugOut(height_diffs, path); + image::debug_out(height_diffs, path); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(ref_heights.begin(), ref_heights.end(), heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -246,12 +246,12 @@ TEST_CASE("reading") fmt::print("low res time: {}; high res time: {}\n", double(low_res_time) / 1000.0, double(high_res_time) / 1000.0); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debugOut(low_res_heights, fmt::format("./low_res_heights.png")); - image::debugOut(high_res_heights, fmt::format("./high_res_heights.png")); + image::debug_out(low_res_heights, fmt::format("./low_res_heights.png")); + image::debug_out(high_res_heights, fmt::format("./high_res_heights.png")); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debugOut(height_diffs, "./diff_low_res_high_res.png"); + image::debug_out(height_diffs, "./diff_low_res_high_res.png"); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -288,12 +288,12 @@ TEST_CASE("reading") REQUIRE(high_res_heights.height() == render_height); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debugOut(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")); - image::debugOut(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")); + image::debug_out(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")); + image::debug_out(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debugOut(height_diffs, "./ov_with_warping_diff_low_res_high_res.png"); + image::debug_out(height_diffs, "./ov_with_warping_diff_low_res_high_res.png"); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -330,12 +330,12 @@ TEST_CASE("reading") REQUIRE(high_res_heights.height() == render_height); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debugOut(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")); - image::debugOut(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")); + image::debug_out(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")); + image::debug_out(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debugOut(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png"); + image::debug_out(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png"); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { diff --git a/unittests/tilebuilder/image_writer.cpp b/unittests/tilebuilder/image_writer.cpp index aeddf2f2..3667e5e3 100644 --- a/unittests/tilebuilder/image_writer.cpp +++ b/unittests/tilebuilder/image_writer.cpp @@ -38,7 +38,7 @@ TEST_CASE("tile builder writes radix rasters as PNG images") raster.pixel({ 0, 1 }) = { 0, 0, 255 }; raster.pixel({ 1, 1 }) = { 255, 255, 255 }; - image::saveImageAsPng(raster, output_path.string()); + image::save_image_as_png(raster, output_path.string()); const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); REQUIRE(decoded.rows == 2); @@ -53,10 +53,7 @@ TEST_CASE("tile builder writes radix rasters as PNG images") TEST_CASE("tile builder handles image writer edge cases") { - SECTION("empty debug raster is rejected") - { - CHECK_THROWS_AS(image::debugOut(radix::Raster {}, "empty.png"), std::invalid_argument); - } + SECTION("empty debug raster is rejected") { CHECK_THROWS_AS(image::debug_out(radix::Raster {}, "empty.png"), std::invalid_argument); } SECTION("constant debug raster is written as black") { @@ -65,7 +62,7 @@ TEST_CASE("tile builder handles image writer edge cases") radix::Raster raster({ 2, 2 }); std::ranges::fill(raster, 42.F); - image::debugOut(raster, output_path.string()); + image::debug_out(raster, output_path.string()); const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); REQUIRE(decoded.rows == 2); @@ -81,6 +78,6 @@ TEST_CASE("tile builder handles image writer edge cases") std::filesystem::remove_all(missing_directory); const auto output_path = missing_directory / "image.png"; - CHECK_THROWS_AS(image::saveImageAsPng(radix::Raster({ 1, 1 }), output_path.string()), std::runtime_error); + CHECK_THROWS_AS(image::save_image_as_png(radix::Raster({ 1, 1 }), output_path.string()), std::runtime_error); } } From 6ea8a4ad379e07a57b18e3d3e1580bad8af18a85 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:33:00 +0200 Subject: [PATCH 19/43] Fix confirmed raster store review findings --- src/dag_builder/build.cpp | 2 +- src/dag_builder/codec/DagFormat.h | 4 ++-- src/sf_merger/NodeLoader.h | 2 +- src/sf_merger/mask.h | 8 ++++++-- src/sf_merger/merge.h | 4 ++-- src/terrainlib/ProgressIndicator.cpp | 8 +++++--- src/terrainlib/mesh/codec/Gltf.h | 3 ++- src/terrainlib/octree/storage/open_runtime.h | 4 ++-- src/terrainlib/store/RawStorage.h | 3 +++ src/terrainlib/store/Storage.h | 2 +- src/terrainlib/store/cache/Interface.h | 4 ++-- src/terrainlib/store/cache/Lru.h | 4 ++-- src/tile_downloader/HttpClient.h | 4 ++-- src/tile_downloader/TileUrlBuilder.h | 4 ++-- src/tile_downloader/cli.h | 4 ++-- unittests/terrainlib/store_storage.cpp | 11 +++++++++++ 16 files changed, 46 insertions(+), 25 deletions(-) diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index e5bda9cf..5d875666 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -519,7 +519,7 @@ std::unordered_set build_level( } // Initialize debug storage if requested (contains .glb meshes) - std::optional debug_storage; + std::optional debug_storage = std::nullopt; if (ctx.options.write_debug_meshes) { octree::OpenOptions options; options.preferred_extension = ".glb"; diff --git a/src/dag_builder/codec/DagFormat.h b/src/dag_builder/codec/DagFormat.h index a5c73f11..27453f39 100644 --- a/src/dag_builder/codec/DagFormat.h +++ b/src/dag_builder/codec/DagFormat.h @@ -100,7 +100,7 @@ inline NodeMetadata encode_metadata(const dag::NodeMetadata& metadata) result.group_assignment = metadata.group_assignment; result.groups.reserve(metadata.groups.size()); for (const dag::Group& group : metadata.groups) { - v1::Group encoded_group{ + v1::Group encoded_group = { .children = {}, .error = group.error, .bounds = { @@ -149,7 +149,7 @@ inline std::expected decode_metadata(NodeMetadat result.group_assignment = std::move(metadata.group_assignment); result.groups.reserve(metadata.groups.size()); for (v1::Group& group : metadata.groups) { - dag::Group decoded_group{ + dag::Group decoded_group = { .children = {}, .error = group.error, .bounds = { diff --git a/src/sf_merger/NodeLoader.h b/src/sf_merger/NodeLoader.h index 23a2cbd1..269db683 100644 --- a/src/sf_merger/NodeLoader.h +++ b/src/sf_merger/NodeLoader.h @@ -29,7 +29,7 @@ class NodeLoader { // Try to retrieve or reconstruct node mesh by ID // TODO: return const ref instead? - std::optional load_node(const octree::Id &id) const noexcept { + std::optional load_node(const octree::Id &id) const { // Try cache if (auto cached = this->_cache.get(id); cached.has_value()) { return cached.value(); diff --git a/src/sf_merger/mask.h b/src/sf_merger/mask.h index 8ceb8900..fde93152 100644 --- a/src/sf_merger/mask.h +++ b/src/sf_merger/mask.h @@ -165,7 +165,8 @@ uint64_t point_count(const OGRGeometry &geometry) { const OGRwkbGeometryType geometry_type = wkbFlatten(geometry.getGeometryType()); if (geometry_type == wkbPolygon) { const OGRPolygon *polygon = geometry.toPolygon(); - uint64_t count = polygon->getExteriorRing()->getNumPoints(); + const OGRLinearRing *exterior_ring = polygon->getExteriorRing(); + uint64_t count = exterior_ring ? exterior_ring->getNumPoints() : 0; for (int i = 0; i < polygon->getNumInteriorRings(); ++i) { count += polygon->getInteriorRing(i)->getNumPoints(); } @@ -354,7 +355,10 @@ inline std::expected load_referenced_from_data MultipolygonWithHoles2 polygons; for (auto &&feature_layer_pair : dataset->GetFeatures()) { OGRGeometry *geometry = feature_layer_pair.feature->GetGeometryRef(); - DEBUG_ASSERT(geometry); + if (!geometry) { + LOG_ERROR("Mask feature has no geometry"); + return std::unexpected(LoadErrorKind::InvalidGeometry); + } const uint64_t original_point_count = point_count(*geometry); std::unique_ptr simplified = simplify_geometry(*geometry, *tolerance); diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index bef384ac..2da551d2 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -95,7 +95,7 @@ class Merger { if (result.source == merge::Source::Left && left_status == Status::Missing) { auto mesh_opt = this->_left.load_node(id); if (mesh_opt.has_value()) { - const auto write_result = this->_output.write_node(id, mesh_opt.value()); + const auto write_result = this->_output.write_node(id, *mesh_opt); if (!write_result.has_value()) { return std::unexpected(sf::ProcessingError(write_result.error())); } @@ -103,7 +103,7 @@ class Merger { } else if (result.source == merge::Source::Right && right_status == Status::Missing) { auto mesh_opt = this->_right.load_node(id); if (mesh_opt.has_value()) { - const auto write_result = this->_output.write_node(id, mesh_opt.value()); + const auto write_result = this->_output.write_node(id, *mesh_opt); if (!write_result.has_value()) { return std::unexpected(sf::ProcessingError(write_result.error())); } diff --git a/src/terrainlib/ProgressIndicator.cpp b/src/terrainlib/ProgressIndicator.cpp index a9f49272..cd9a4dcf 100644 --- a/src/terrainlib/ProgressIndicator.cpp +++ b/src/terrainlib/ProgressIndicator.cpp @@ -35,11 +35,13 @@ ProgressIndicator::ProgressIndicator(size_t n_steps) } void ProgressIndicator::task_finished() { - ++m_step; - if (m_step > m_n_steps) { + const size_t step = ++m_step; + if (step > m_n_steps) { throw std::runtime_error("Too many steps reported."); } - m_monitor_condition.notify_all(); + if (step == m_n_steps) { + m_monitor_condition.notify_one(); + } } std::jthread ProgressIndicator::start_monitoring() const { diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index dd83de61..67f1082f 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "mesh/SimpleMesh.h" @@ -39,7 +40,7 @@ class Gltf final : public store::Codec { result.error().description(), }); } - return result.value(); + return std::move(result.value()); } std::expected write(const store::NodePath& node_path, const mesh::Simple& mesh) const override diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index 49d2fc4d..0420d53b 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -15,8 +15,8 @@ namespace octree::storage { struct OpenOptions { - std::optional> default_mapping; - std::optional preferred_extension; + std::optional> default_mapping = std::nullopt; + std::optional preferred_extension = std::nullopt; }; template diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index 4b747fe8..a61cfdfa 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -157,6 +157,9 @@ class RawStorage { "matching codec probes produced different actual path counts", })); } + if (source_paths == target_paths) { + return {}; + } const auto prepared = before_modify(); if (!prepared.has_value()) { return prepared; diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index 7df1628d..c04f2e9b 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -296,7 +296,7 @@ class Storage { RawStorage m_raw; std::optional> m_index; - std::optional m_persistence; + std::optional m_persistence = std::nullopt; StorageSettings m_settings; mutable bool m_dirty = false; }; diff --git a/src/terrainlib/store/cache/Interface.h b/src/terrainlib/store/cache/Interface.h index ea42a4f2..7cc6ce73 100644 --- a/src/terrainlib/store/cache/Interface.h +++ b/src/terrainlib/store/cache/Interface.h @@ -12,8 +12,8 @@ class Interface { using Key = typename Traits::Key; virtual ~Interface() = default; - virtual std::optional get(const Key& key) noexcept = 0; - virtual bool put(const Key& key, const NodeData& value) noexcept = 0; + virtual std::optional get(const Key& key) = 0; + virtual bool put(const Key& key, const NodeData& value) = 0; virtual bool remove(const Key& key) noexcept = 0; virtual bool contains(const Key& key) const noexcept = 0; }; diff --git a/src/terrainlib/store/cache/Lru.h b/src/terrainlib/store/cache/Lru.h index 1eb8f0a4..60b33d76 100644 --- a/src/terrainlib/store/cache/Lru.h +++ b/src/terrainlib/store/cache/Lru.h @@ -18,7 +18,7 @@ class Lru final : public Interface { { } - std::optional get(const Key& key) noexcept override + std::optional get(const Key& key) override { const auto iterator = m_values.find(key); if (iterator == m_values.end()) { @@ -28,7 +28,7 @@ class Lru final : public Interface { return iterator->second.first; } - bool put(const Key& key, const NodeData& value) noexcept override + bool put(const Key& key, const NodeData& value) override { const auto iterator = m_values.find(key); if (iterator != m_values.end()) { diff --git a/src/tile_downloader/HttpClient.h b/src/tile_downloader/HttpClient.h index d5ef1ca4..6eaeeb45 100644 --- a/src/tile_downloader/HttpClient.h +++ b/src/tile_downloader/HttpClient.h @@ -40,8 +40,8 @@ class HttpClient { HttpResponse get(const std::string &url, const ProgressFn &on_progress = {}) const { HttpResponse response; - WriteContext write_context{&response.body, &this->_write, {}}; - ProgressContext progress_context{&on_progress, {}}; + WriteContext write_context = {&response.body, &this->_write, {}}; + ProgressContext progress_context = {&on_progress, {}}; curl_easy_setopt(this->_curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(this->_curl, CURLOPT_WRITEFUNCTION, write_cb); diff --git a/src/tile_downloader/TileUrlBuilder.h b/src/tile_downloader/TileUrlBuilder.h index e28f3002..6e53cf84 100644 --- a/src/tile_downloader/TileUrlBuilder.h +++ b/src/tile_downloader/TileUrlBuilder.h @@ -10,12 +10,12 @@ enum class TileDownloadProvider { Basemap, - Gataki + Gataki, }; enum class TileYDirection { Down, - Up + Up, }; struct TileProviderConfig { diff --git a/src/tile_downloader/cli.h b/src/tile_downloader/cli.h index 8e2380f8..6e03207c 100644 --- a/src/tile_downloader/cli.h +++ b/src/tile_downloader/cli.h @@ -11,8 +11,8 @@ namespace cli { struct Args { - std::optional provider; - std::optional url_pattern; + std::optional provider = std::nullopt; + std::optional url_pattern = std::nullopt; unsigned int zoom; unsigned int x; unsigned int y; diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index bc424fa3..ded054b9 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -405,6 +405,17 @@ TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][co CHECK(std::filesystem::equivalent(source_paths[0], target_paths[0])); CHECK(std::filesystem::equivalent(source_paths[1], target_paths[1])); } + + SECTION("same storage") + { + TemporaryDirectory directory("copy-self"); + auto storage = make_indexed_storage(directory.path()); + REQUIRE(storage.save(root, 43).has_value()); + storage.settings().allow_overwrite = true; + + REQUIRE(storage.copy_from(root, storage).has_value()); + CHECK(storage.load(root).value() == 43); + } } TEST_CASE("copy_from re-encodes differing path lists", "[store][storage][copy]") From 3fbb86bfbb6ab84f15ba8d6e79be97d02701a7dd Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:20:35 +0200 Subject: [PATCH 20/43] Serialize shared storage access in parallel builders --- src/dag_builder/build.cpp | 8 +++++++- src/sf_builder/terrainbuilder.cpp | 12 ++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 5d875666..4a6eb91d 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -520,6 +521,7 @@ std::unordered_set build_level( // Initialize debug storage if requested (contains .glb meshes) std::optional debug_storage = std::nullopt; + std::mutex debug_storage_mutex; if (ctx.options.write_debug_meshes) { octree::OpenOptions options; options.preferred_extension = ".glb"; @@ -553,7 +555,11 @@ std::unordered_set build_level( const auto save_result = ctx.output_storage.save(target, *result); if (save_result) { if (debug_storage) { - const auto debug_save_result = debug_storage->save(target, clustering_to_mesh(result->clustering)); + const auto debug_mesh = clustering_to_mesh(result->clustering); + const auto debug_save_result = [&] { + std::scoped_lock lock(debug_storage_mutex); + return debug_storage->save(target, debug_mesh); + }(); if (!debug_save_result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to save debug mesh for node {}: {}", diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 22a4d392..68d830e1 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -207,6 +208,7 @@ std::expected build_all_patches( } mesh::storage::Storage storage = std::move(storage_result.value()); storage.settings().allow_overwrite = overwrite_existing; + std::mutex storage_mutex; const auto dataset_srs = dataset.srs(); const auto dataset_bounds = dataset.bounds3d(true); @@ -291,7 +293,10 @@ std::expected build_all_patches( tbb::task_group_context context; tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; - const auto already_exists = storage.has(node); + const auto already_exists = [&] { + std::scoped_lock lock(storage_mutex); + return storage.has(node); + }(); if (!already_exists.has_value()) { LOG_ERROR( "Failed to inspect node {}: {}", @@ -323,7 +328,10 @@ std::expected build_all_patches( if (mesh_result.has_value()) { const auto mesh = std::move(mesh_result.value()); mesh::validate(mesh); - const auto save_result = storage.save(node, mesh); + const auto save_result = [&] { + std::scoped_lock lock(storage_mutex); + return storage.save(node, mesh); + }(); if (!save_result.has_value()) { LOG_ERROR( "Failed to save mesh for node {}: {}", From b1a2884a13393799a703c15a7c1891fe8475c738 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:32:13 +0200 Subject: [PATCH 21/43] Report directory creation failures without throwing --- src/terrainlib/io/Error.h | 6 +++++- src/terrainlib/io/bytes.cpp | 7 ++++++- src/terrainlib/io/utils.cpp | 12 ++++++++---- src/terrainlib/io/utils.h | 3 ++- unittests/terrainlib/store_storage.cpp | 11 +++++++++++ 5 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/terrainlib/io/Error.h b/src/terrainlib/io/Error.h index b4e7aece..40d1f1c4 100644 --- a/src/terrainlib/io/Error.h +++ b/src/terrainlib/io/Error.h @@ -13,7 +13,8 @@ class Error { WriteBytes, DetermineSize, ReadBytes, - OutOfMemory + OutOfMemory, + CreateDirectories, }; Error() = default; @@ -66,6 +67,9 @@ struct fmt::formatter { case io::Error::Value::OutOfMemory: name = "OutOfMemory"; break; + case io::Error::Value::CreateDirectories: + name = "CreateDirectories"; + break; default: UNREACHABLE(); } diff --git a/src/terrainlib/io/bytes.cpp b/src/terrainlib/io/bytes.cpp index 1412ce33..95aa8d86 100644 --- a/src/terrainlib/io/bytes.cpp +++ b/src/terrainlib/io/bytes.cpp @@ -1,4 +1,5 @@ #include +#include #include "io/bytes.h" #include "io/utils.h" @@ -10,7 +11,11 @@ std::expected write_bytes_to_path(const std::span by LOG_TRACE("Writing bytes to path {}", path); if (make_dirs) { - utils::create_parent_directories(path); + const std::error_code error = utils::create_parent_directories(path); + if (error) { + LOG_ERROR("Failed to create parent directories for {}: {}", path, error.message()); + return std::unexpected(Error::CreateDirectories); + } } std::ofstream file(path, std::ios::binary); diff --git a/src/terrainlib/io/utils.cpp b/src/terrainlib/io/utils.cpp index 85b5c77f..0d8f6298 100644 --- a/src/terrainlib/io/utils.cpp +++ b/src/terrainlib/io/utils.cpp @@ -2,10 +2,14 @@ namespace io::utils { -std::filesystem::path create_parent_directories(const std::filesystem::path &path) { - const std::filesystem::path parent_path = std::filesystem::absolute(path).parent_path(); - std::filesystem::create_directories(parent_path); - return parent_path; +std::error_code create_parent_directories(const std::filesystem::path &path) { + std::error_code error; + const std::filesystem::path parent_path = std::filesystem::absolute(path, error).parent_path(); + if (error) { + return error; + } + std::filesystem::create_directories(parent_path, error); + return error; } } // namespace io::utils diff --git a/src/terrainlib/io/utils.h b/src/terrainlib/io/utils.h index 8b6b4e69..aa369d69 100644 --- a/src/terrainlib/io/utils.h +++ b/src/terrainlib/io/utils.h @@ -1,9 +1,10 @@ #pragma once #include +#include namespace io::utils { -std::filesystem::path create_parent_directories(const std::filesystem::path &path); +[[nodiscard]] std::error_code create_parent_directories(const std::filesystem::path &path); } // namespace io::utils diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index ded054b9..c99b50e2 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -280,6 +280,17 @@ store::IndexedStorage make_indexed_storage( } // namespace +TEST_CASE("byte writes report directory creation errors", "[io][bytes]") +{ + TemporaryDirectory directory("write-parent-error"); + const std::filesystem::path blocker = directory.path() / "blocker"; + REQUIRE(write_int(1, blocker).has_value()); + + const auto result = io::write_bytes_to_path(std::span {}, blocker / "payload"); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error() == io::Error(io::Error::CreateDirectories)); +} + TEMPLATE_TEST_CASE("shared raw storage has no hidden octree dependency", "[store][storage]", octree::StoreTraits, raster_store::StoreTraits) { using Traits = TestType; From d293327ca32166f55a592ba108bb9ed6db807a73 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:17:38 +0200 Subject: [PATCH 22/43] Validate DAG cross-file cluster counts --- src/dag_builder/codec/Dag.h | 21 ++++++++++- .../dag_builder/storage_compatibility.cpp | 37 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index 5894ada6..2cfb2f85 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -22,6 +22,18 @@ inline store::CodecError file_error(const store::CodecOperation operation, const }; } +inline std::expected validate_batch(const dag::ClusterBatch& batch, const store::CodecOperation operation) +{ + if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { + return std::unexpected(store::CodecError { + operation, + operation == store::CodecOperation::Read ? store::CodecErrorCategory::InvalidData : store::CodecErrorCategory::Domain, + "DAG metadata group-assignment count does not match clustering cluster count", + }); + } + return {}; +} + class Dag final : public store::Codec { public: std::vector paths(const store::NodePath& node_path) const override @@ -59,14 +71,21 @@ class Dag final : public store::Codec { metadata.error(), }); } - return dag::ClusterBatch { + dag::ClusterBatch batch { .metadata = std::move(*metadata), .clustering = std::move(*clustering), }; + if (auto valid = validate_batch(batch, store::CodecOperation::Read); !valid) { + return std::unexpected(valid.error()); + } + return batch; } std::expected write(const store::NodePath& node_path, const dag::ClusterBatch& batch) const override { + if (auto valid = validate_batch(batch, store::CodecOperation::Write); !valid) { + return valid; + } auto clustering = dag::format::encode_clustering(batch.clustering); if (!clustering) { return std::unexpected(store::CodecError { diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 00ea1317..925b3810 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -87,6 +87,43 @@ TEST_CASE("runtime DAG envelope codec is reentrant") } } +TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") +{ + TemporaryDirectory output; + dag::codec::Dag codec; + const store::NodePath node_path(output.path() / "node"); + const auto node_paths = codec.paths(node_path); + + SECTION("write") + { + auto batch = sample_batch(); + batch.metadata.group_assignment.clear(); + + const auto result = codec.write(node_path, batch); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().operation == store::CodecOperation::Write); + CHECK(result.error().category == store::CodecErrorCategory::Domain); + CHECK_FALSE(std::filesystem::exists(node_paths[0])); + CHECK_FALSE(std::filesystem::exists(node_paths[1])); + } + + SECTION("read") + { + REQUIRE(codec.write(node_path, sample_batch()).has_value()); + dag::NodeMetadata metadata = sample_batch().metadata; + metadata.group_assignment.clear(); + const auto encoded_metadata = dag::format::encode_metadata(metadata); + REQUIRE(io::envelope::write_to_path(encoded_metadata, node_paths[1]).has_value()); + + const auto result = codec.read(node_path); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().operation == store::CodecOperation::Read); + CHECK(result.error().category == store::CodecErrorCategory::InvalidData); + } +} + TEST_CASE("versioned DAG payload validation is free-standing") { dag::format::NodeMetadata invalid_metadata { From a964ff82b14553a4c4e4c838505593e895313eaa Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:49:35 +0200 Subject: [PATCH 23/43] List all headers in CMake targets --- src/dag_builder/CMakeLists.txt | 58 ++++++- src/dag_convert_debug/CMakeLists.txt | 2 +- src/mesh_convert/CMakeLists.txt | 2 +- src/sf_browser/CMakeLists.txt | 20 +-- src/sf_builder/CMakeLists.txt | 9 +- src/sf_index_browser/CMakeLists.txt | 2 +- src/sf_merger/CMakeLists.txt | 25 ++- src/terrainlib/CMakeLists.txt | 226 ++++++++++++++++++++++----- src/tile_builder/CMakeLists.txt | 19 ++- src/tile_downloader/CMakeLists.txt | 8 +- 10 files changed, 304 insertions(+), 67 deletions(-) diff --git a/src/dag_builder/CMakeLists.txt b/src/dag_builder/CMakeLists.txt index a3ddc1df..f7b8fc8c 100644 --- a/src/dag_builder/CMakeLists.txt +++ b/src/dag_builder/CMakeLists.txt @@ -1,17 +1,63 @@ add_library(dagbuilderlib - atlas/rect/atlas.cpp - atlas/rect/HorizontalStripPlanner.cpp - atlas/rect/PackingPlanner.cpp - merge/clusterings.cpp + ContinuationMode.h + Partitioning.h + TextureSet.h + VertexInCluster.h + VertexInClustering.h + build.h build.cpp + centroids.h + cluster.h + clusterize.h + compact.h + correspondence.h + dag_id.h + dag_node.h + fixed.h + fixed_point.h + meshopt.h + metadata.h + optimize.h + parallel.h + partition.h + simplify.h + slice.h + split.h + storage.h + texture_reuse.h + texture_sizing.h + texturing.h + thread_safe_storage.h + utils.h + validate.h + vertex_lock.h - build.cpp + atlas/BakeSource.h + atlas/bake_cluster_texture.h + atlas/pull_reproject_texture.h + atlas/rect/HorizontalStripPlanner.h atlas/rect/HorizontalStripPlanner.cpp + atlas/rect/PackingPlanner.h atlas/rect/PackingPlanner.cpp + atlas/rect/Plan.h + atlas/rect/Planner.h + atlas/rect/atlas.h atlas/rect/atlas.cpp + atlas/rect/create.h + atlas/reprojection.h + + codec/Dag.h + codec/DagFormat.h + codec/MetadataView.h + codec/from_extension.h + + merge/clusterings.h merge/clusterings.cpp + merge/clusters.h + merge/weld.h ) + target_include_directories(dagbuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(dagbuilderlib PUBLIC terrainlib libassert::assert metis rectpack2D meshoptimizer) if(ALP_BUILD_DAG_BUILDER) add_executable(dag-builder - cli.cpp + cli.h cli.cpp main.cpp ) target_link_libraries(dag-builder PRIVATE dagbuilderlib CLI11::CLI11) diff --git a/src/dag_convert_debug/CMakeLists.txt b/src/dag_convert_debug/CMakeLists.txt index ade99815..d94dc228 100644 --- a/src/dag_convert_debug/CMakeLists.txt +++ b/src/dag_convert_debug/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable(dag-convert-debug - cli.cpp + cli.h cli.cpp main.cpp ) target_link_libraries(dag-convert-debug PRIVATE dagbuilderlib CLI11::CLI11) diff --git a/src/mesh_convert/CMakeLists.txt b/src/mesh_convert/CMakeLists.txt index 8779e0f7..01a98169 100644 --- a/src/mesh_convert/CMakeLists.txt +++ b/src/mesh_convert/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable(mesh-convert - cli.cpp + cli.h cli.cpp main.cpp ) target_link_libraries(mesh-convert PRIVATE terrainlib CLI11::CLI11) diff --git a/src/sf_browser/CMakeLists.txt b/src/sf_browser/CMakeLists.txt index 8ed300c6..d3f379ce 100644 --- a/src/sf_browser/CMakeLists.txt +++ b/src/sf_browser/CMakeLists.txt @@ -2,16 +2,18 @@ add_subdirectory(res) add_executable(sf-browser main.cpp - core/Application.cpp - core/window/Window.cpp - core/shader/Shader.cpp - core/shader/ShaderProgram.cpp - core/shader/Uniform.h - core/shader/GLUniformAbstractions.h - core/Buffer.cpp - core/Camera.cpp + + core/Application.h core/Application.cpp + core/Buffer.h core/Buffer.cpp + core/Camera.h core/Camera.cpp core/geometry/UnitCube.h - core/rendering/OctreeRenderManager.cpp + core/rendering/OctreeRenderManager.h core/rendering/OctreeRenderManager.cpp + core/shader/GLUniformAbstractions.h + core/shader/Shader.h core/shader/Shader.cpp + core/shader/ShaderProgram.h core/shader/ShaderProgram.cpp + core/shader/Uniform.h + core/window/Window.h core/window/Window.cpp ) + target_include_directories(sf-browser PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(sf-browser PRIVATE terrainlib glfw glad_gl_core_46 sf-browser-resources imgui) diff --git a/src/sf_builder/CMakeLists.txt b/src/sf_builder/CMakeLists.txt index cfdabb70..4a32f9cc 100644 --- a/src/sf_builder/CMakeLists.txt +++ b/src/sf_builder/CMakeLists.txt @@ -1,7 +1,12 @@ add_library(sfbuilderlib - terrainbuilder.cpp - mesh_builder.cpp + border.h + mesh_builder.h mesh_builder.cpp + raw_dataset_reader.h + terrainbuilder.h terrainbuilder.cpp + texture_assembler.h + tile_provider.h ) + target_include_directories(sfbuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(sfbuilderlib PUBLIC terrainlib spdlog) diff --git a/src/sf_index_browser/CMakeLists.txt b/src/sf_index_browser/CMakeLists.txt index 22dbfda5..c580b6a5 100644 --- a/src/sf_index_browser/CMakeLists.txt +++ b/src/sf_index_browser/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable(sf-index-browser - cli.cpp + cli.h cli.cpp main.cpp ) target_link_libraries(sf-index-browser PRIVATE terrainlib CLI11::CLI11 ftxui::dom ftxui::component ftxui::screen) diff --git a/src/sf_merger/CMakeLists.txt b/src/sf_merger/CMakeLists.txt index c17cfc9a..2f647ab4 100644 --- a/src/sf_merger/CMakeLists.txt +++ b/src/sf_merger/CMakeLists.txt @@ -1,9 +1,30 @@ -add_library(sfmergerlib INTERFACE) +add_library(sfmergerlib INTERFACE + NodeLoader.h + NodeWriter.h + cut.h + earth.h + mask.h + merge.h + utils.h + + atlas/HorizontalStripLayoutPlanner.h + atlas/Layout.h + atlas/LayoutPlanner.h + atlas/Sheet.h + atlas/build.h + + merge/NodeData.h + merge/Result.h + merge/visitor/Masked.h + merge/visitor/Simple.h + merge/visitor/Visitor.h +) + target_include_directories(sfmergerlib INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(sfmergerlib INTERFACE terrainlib) add_executable(sf-merger + cli.h cli.cpp main.cpp - cli.cpp ) target_link_libraries(sf-merger PRIVATE sfmergerlib CLI11::CLI11) diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 773d223a..59ee256a 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -1,4 +1,46 @@ add_library(terrainlib + Dataset.h Dataset.cpp + ImageKey.h + ProgressIndicator.h ProgressIndicator.cpp + Range.h + SphereProjector.h + VecHash.h + VecRange.h + array_utils.h + build_config.h + enumerate.h + fmt_impls.h + glm_utils.h + hash_utils.h + init.h init.cpp + log.h log.cpp + macros.h + opencv_utils.h + optional_utils.h + pch.h + range_utils.h + srs.h + string_utils.h + tile_path.h + type_utils.h + variant_utils.h + vector_utils.h + + containers/Buffer.h + containers/Cow.h + containers/FixedVector.h + containers/HybridIndexPairMap.h + containers/HybridVector.h + containers/OffsetTable.h + containers/OffsetVector.h + containers/SegmentedBuffer.h + containers/Size.h + containers/StampSet.h + containers/ThreadLocalCache.h + containers/TinyVector.h + containers/UnionFind.h + containers/Vector2D.h + ctb/CTBException.hpp ctb/GlobalGeodetic.hpp ctb/GlobalGeodetic.cpp ctb/GlobalMercator.hpp ctb/GlobalMercator.cpp @@ -6,53 +48,165 @@ add_library(terrainlib ctb/TileCoordinate.hpp ctb/types.hpp - io/bytes.cpp - io/compression.cpp - io/conversion.h - io/utils.cpp + geometry/CoordFrame.h + geometry/PlaneFrame.h + geometry/Rect.h + geometry/geometry.h - mesh/connectivity/adjacency.cpp - mesh/connectivity/boundary.cpp - mesh/connectivity/manifold.cpp - mesh/connectivity/topology.cpp - mesh/connectivity/vertex_index_range.cpp + io/Error.h + io/bytes.h io/bytes.cpp + io/compression.h io/compression.cpp + io/conversion.h + io/envelope.h + io/envelope_file.h + io/utils.h io/utils.cpp + mesh/EncodedMesh.h + mesh/PointWithMeta.h + mesh/SimpleMesh.h + mesh/TriangleContainer.h + mesh/TriangleSoup.h + mesh/VertexMap.h + mesh/View.h + mesh/WindingOrder.h + mesh/bfs.h mesh/bfs.cpp + mesh/boolean.h mesh/boolean.cpp + mesh/bounds.h + mesh/cgal.h + mesh/cleanup.h mesh/cleanup.cpp + mesh/clip.h mesh/clip.cpp + mesh/codec/Gltf.h + mesh/codec/SfMesh.h + mesh/codec/SfMeshFormat.h + mesh/codec/from_extension.h + mesh/combine.h + mesh/connectivity/adjacency.h mesh/connectivity/adjacency.cpp + mesh/connectivity/boundary.h mesh/connectivity/boundary.cpp + mesh/connectivity/connected_components.h + mesh/connectivity/edges.h + mesh/connectivity/manifold.h mesh/connectivity/manifold.cpp + mesh/connectivity/topology.h mesh/connectivity/topology.cpp + mesh/connectivity/triangle_compare.h + mesh/connectivity/vertex_index_range.h mesh/connectivity/vertex_index_range.cpp + mesh/convert.h mesh/convert.cpp + mesh/encode.h + mesh/geometry.h mesh/geometry.cpp + mesh/holes.h mesh/holes.cpp + mesh/igl/convert.h + mesh/igl/cut.h + mesh/igl/cut_to_disk.h mesh/igl/igl.cpp + mesh/igl/manifold.h + mesh/igl/orient.h + mesh/io.h mesh/io.cpp + mesh/io/error.h + mesh/io/gltf.h mesh/io/gltf.cpp + mesh/io/options.h + mesh/io/texture.h mesh/io/texture.cpp + mesh/io/utils.h mesh/io/utils.cpp + mesh/merge.h + mesh/merging/EpsilonVertexDeduplicate.h + mesh/merging/ExactVertexDeduplicate.h + mesh/merging/SphereProjectionVertexDeduplicate.h + mesh/merging/VertexDeduplicate.h + mesh/merging/VertexId.h + mesh/merging/VertexMapping.h + mesh/merging/helpers.h + mesh/merging/mapping.h mesh/merging/mapping.cpp + mesh/normalize.h + mesh/reindex.h + mesh/split.h + mesh/storage.h + mesh/texture_trim.h mesh/texture_trim.cpp + mesh/utils.h + mesh/validate.h mesh/validate.cpp + mesh/validate_cgal.h - mesh/io/gltf.cpp - mesh/io/texture.cpp - mesh/io/utils.cpp + numeric/StrongDouble.h + numeric/glm_math.h + numeric/int_math.h + numeric/number_types.h + numeric/number_utils.h + numeric/quantize.h + numeric/wide_integer.h - mesh/merging/mapping.cpp + octree/Id.h + octree/IdRect.h + octree/OddLevelShifted.h octree/OddLevelShifted.cpp + octree/Space.h octree/Space.cpp + octree/StoreTraits.h + octree/storage/Format.h + octree/storage/IndexFile.h + octree/storage/open.h + octree/storage/open_runtime.h + octree/store_layout/Flat.h + octree/store_layout/LevelAndCoordinateDirectories.h + octree/store_layout/Mappings.h + octree/utils.h - mesh/bfs.cpp - mesh/boolean.cpp - mesh/cleanup.cpp - mesh/clip.cpp - mesh/convert.cpp - mesh/geometry.cpp - mesh/holes.cpp - mesh/io.cpp - mesh/texture_trim.cpp - mesh/validate.cpp + polygon/Polygon.h + polygon/clip.h polygon/clip.cpp + polygon/triangulate.h polygon/triangulate.cpp + polygon/utils.h polygon/utils.cpp - octree/Space.cpp - octree/OddLevelShifted.cpp + range/AnyRange.h + range/Bound.h + range/Range.h + range/RangeBounds.h + range/RangeFrom.h + range/RangeFull.h + range/RangeInclusive.h + range/RangeTo.h + range/RangeToInclusive.h - polygon/triangulate.cpp - polygon/utils.cpp - polygon/clip.cpp + raster_store/StoreTraits.h - texture/dilate_colors.cpp + sf/Error.h + sf/InvalidTopology.h + sf/finalize_storage.h + sf/validate_index.h - uv/unwrap.cpp - uv/atlas.cpp + spatial_lookup/CellBased.h + spatial_lookup/CellBasedStorage.h + spatial_lookup/Grid.h + spatial_lookup/GridStorage.h + spatial_lookup/Hashmap.h + spatial_lookup/HashmapStorage.h + spatial_lookup/NDLoopHelper.h + spatial_lookup/QuantizedHash.h + spatial_lookup/SpatialLookup.h - Dataset.cpp - init.cpp - log.cpp - ProgressIndicator.cpp - tile_path.h + store/Codec.h + store/CodecError.h + store/Index.h + store/IndexFormat.h + store/IndexedStorage.h + store/InvalidKey.h + store/Layout.h + store/NodePath.h + store/NodeStatus.h + store/NodeStatusOrMissing.h + store/OpenError.h + store/PathMapping.h + store/RawStorage.h + store/Storage.h + store/StorageError.h + store/StorageSettings.h + store/Traits.h + store/cache/Dummy.h + store/cache/Interface.h + store/cache/Lru.h + store/codec/Path.h + store/describe_error.h + store/open.h + store/traverse.h + + texture/Mask.h + texture/dilate_colors.h texture/dilate_colors.cpp + + uv/atlas.h uv/atlas.cpp + uv/fit.h + uv/unwrap.h uv/unwrap.cpp ) target_compile_features(terrainlib PUBLIC cxx_std_23) diff --git a/src/tile_builder/CMakeLists.txt b/src/tile_builder/CMakeLists.txt index 576df41c..a05e9edb 100644 --- a/src/tile_builder/CMakeLists.txt +++ b/src/tile_builder/CMakeLists.txt @@ -1,13 +1,16 @@ add_library(tilebuilderlib - alpine_raster.cpp - DatasetReader.cpp - image_writer.cpp - ParallelTileGenerator.cpp - ParallelTiler.cpp - TileHeightsGenerator.cpp - Tiler.cpp - TopDownTiler.cpp + DatasetReader.h DatasetReader.cpp + Exception.h + ParallelTileGenerator.h ParallelTileGenerator.cpp + ParallelTiler.h ParallelTiler.cpp + TileHeightsGenerator.h TileHeightsGenerator.cpp + Tiler.h Tiler.cpp + TopDownTiler.h TopDownTiler.cpp + alpine_raster.h alpine_raster.cpp + depth_first_tile_traverser.h + image_writer.h image_writer.cpp ) + target_include_directories(tilebuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(tilebuilderlib PUBLIC terrainlib ZLIB::ZLIB GDAL::GDAL fmt) diff --git a/src/tile_downloader/CMakeLists.txt b/src/tile_downloader/CMakeLists.txt index c1aebead..ac21b220 100644 --- a/src/tile_downloader/CMakeLists.txt +++ b/src/tile_downloader/CMakeLists.txt @@ -1,5 +1,11 @@ add_executable(tile-downloader + HttpClient.h + TileDownloader.h + TileLogger.h + TileUrlBuilder.h + cli.h cli.cpp main.cpp - cli.cpp + write_file.h ) + target_link_libraries(tile-downloader PRIVATE terrainlib CLI11::CLI11 CURL::libcurl) From d8f9290260ab2b54a54b255733207a4cb4a4b4f6 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:49:30 +0200 Subject: [PATCH 24/43] Compile cgltf implementation outside unity builds --- src/terrainlib/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 59ee256a..10e0b846 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -208,6 +208,7 @@ add_library(terrainlib uv/fit.h uv/unwrap.h uv/unwrap.cpp ) +set_source_files_properties(mesh/io/gltf.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) target_compile_features(terrainlib PUBLIC cxx_std_23) target_compile_definitions(terrainlib PUBLIC SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE) From 6d1f334b12e864994240f9cedbb5e03053c6111a Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:48:50 +0200 Subject: [PATCH 25/43] Simplify raster store structure --- docs/raster-store/architecture.md | 4 +- src/dag_builder/CMakeLists.txt | 2 - src/dag_builder/build.cpp | 23 ++-- src/dag_builder/codec/Dag.h | 29 +++- src/dag_builder/codec/MetadataView.h | 39 ------ src/dag_builder/codec/from_extension.h | 1 - src/dag_builder/main.cpp | 4 +- src/dag_builder/thread_safe_storage.h | 64 --------- src/dag_convert_debug/main.cpp | 6 +- src/sf_builder/terrainbuilder.cpp | 30 ++--- src/sf_builder/terrainbuilder.h | 2 - src/sf_index_browser/main.cpp | 6 +- src/sf_merger/cut.h | 5 +- src/sf_merger/main.cpp | 11 +- src/terrainlib/CMakeLists.txt | 5 +- src/terrainlib/io/envelope_file.h | 6 + src/terrainlib/mesh/codec/Gltf.h | 1 - src/terrainlib/mesh/codec/SfMesh.h | 5 +- src/terrainlib/octree/storage/Format.h | 125 ------------------ src/terrainlib/octree/storage/IndexFile.h | 118 ++++++++++++++++- src/terrainlib/octree/storage/open.h | 29 ---- src/terrainlib/store/Codec.h | 12 ++ src/terrainlib/store/Storage.h | 5 +- src/terrainlib/store/StorageSettings.h | 9 -- src/terrainlib/store/ThreadSafeStorage.h | 66 +++++++++ src/terrainlib/store/codec/Path.h | 17 --- unittests/CMakeLists.txt | 4 + .../dag_builder/storage_compatibility.cpp | 31 +---- unittests/sf_builder/storage_finalization.cpp | 39 ++---- unittests/sf_merger/storage_boundaries.cpp | 32 +---- unittests/temporary_directory.h | 40 ++++++ unittests/terrainlib/store_codec.cpp | 23 +--- unittests/terrainlib/store_compatibility.cpp | 43 ++---- unittests/terrainlib/store_storage.cpp | 23 +--- 34 files changed, 346 insertions(+), 513 deletions(-) delete mode 100644 src/dag_builder/codec/MetadataView.h delete mode 100644 src/dag_builder/thread_safe_storage.h delete mode 100644 src/terrainlib/octree/storage/Format.h delete mode 100644 src/terrainlib/octree/storage/open.h delete mode 100644 src/terrainlib/store/StorageSettings.h create mode 100644 src/terrainlib/store/ThreadSafeStorage.h delete mode 100644 src/terrainlib/store/codec/Path.h create mode 100644 unittests/temporary_directory.h diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md index 9373d754..392c02e9 100644 --- a/docs/raster-store/architecture.md +++ b/docs/raster-store/architecture.md @@ -34,9 +34,9 @@ DTO, and mesh-opening compatibility functions. A legacy 3D mesh dataset is opened through the 3D adapter: ```cpp -#include "octree/storage/open.h" +#include "mesh/storage.h" -auto opened = octree::open_folder_indexed(dataset_path); +auto opened = mesh::storage::open_folder_indexed(dataset_path); if (!opened.has_value()) { return std::unexpected(opened.error()); } diff --git a/src/dag_builder/CMakeLists.txt b/src/dag_builder/CMakeLists.txt index f7b8fc8c..5479a9a2 100644 --- a/src/dag_builder/CMakeLists.txt +++ b/src/dag_builder/CMakeLists.txt @@ -26,7 +26,6 @@ add_library(dagbuilderlib texture_reuse.h texture_sizing.h texturing.h - thread_safe_storage.h utils.h validate.h vertex_lock.h @@ -44,7 +43,6 @@ add_library(dagbuilderlib codec/Dag.h codec/DagFormat.h - codec/MetadataView.h codec/from_extension.h merge/clusterings.h merge/clusterings.cpp diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 4a6eb91d..32fb9eed 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -26,7 +25,7 @@ #include "octree/IdRect.h" #include "octree/OddLevelShifted.h" #include "octree/Space.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" #include "store/traverse.h" #include "ProgressIndicator.h" #include "partition.h" @@ -34,7 +33,7 @@ #include "slice.h" #include "range_utils.h" #include "storage.h" -#include "thread_safe_storage.h" +#include "store/ThreadSafeStorage.h" #include "store/describe_error.h" #include "utils.h" #include "vertex_lock.h" @@ -154,7 +153,7 @@ std::vector load_input_clusters( // Dependencies shared across the whole dag building pipeline. struct BuildContext { const mesh::storage::IndexedStorage &input_storage; - ThreadSafeStorage output_storage; + store::ThreadSafeStorage output_storage; const BuildOptions &options; const octree::Space &space; const octree::OddLevelShifted &shifted_space; @@ -520,12 +519,11 @@ std::unordered_set build_level( } // Initialize debug storage if requested (contains .glb meshes) - std::optional debug_storage = std::nullopt; - std::mutex debug_storage_mutex; + std::optional> debug_storage = std::nullopt; if (ctx.options.write_debug_meshes) { - octree::OpenOptions options; + mesh::storage::OpenOptions options; options.preferred_extension = ".glb"; - auto debug_result = octree::open_folder( + auto debug_result = mesh::storage::open_folder( ctx.output_storage.base_path().string() + "-debug", std::move(options)); if (!debug_result.has_value()) { @@ -533,7 +531,7 @@ std::unordered_set build_level( "Failed to open debug mesh storage: {}", store::describe_error(debug_result.error())); } - debug_storage = std::move(debug_result.value()); + debug_storage.emplace(std::move(debug_result.value())); } tbb::concurrent_vector saved_ids; @@ -556,10 +554,7 @@ std::unordered_set build_level( if (save_result) { if (debug_storage) { const auto debug_mesh = clustering_to_mesh(result->clustering); - const auto debug_save_result = [&] { - std::scoped_lock lock(debug_storage_mutex); - return debug_storage->save(target, debug_mesh); - }(); + const auto debug_save_result = debug_storage->save(target, debug_mesh); if (!debug_save_result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to save debug mesh for node {}: {}", @@ -634,7 +629,7 @@ std::expected build_levels( } } - BuildContext ctx{input_storage, ThreadSafeStorage(std::move(output_storage)), options, space, shifted_space, root_bounds}; + BuildContext ctx{input_storage, store::ThreadSafeStorage(std::move(output_storage)), options, space, shifted_space, root_bounds}; for (uint32_t level = range.end; level-- > range.start;) { prev_level_built = build_level( diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index 2cfb2f85..e96f86e0 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -6,16 +6,14 @@ #include "codec/DagFormat.h" #include "io/envelope_file.h" #include "store/Codec.h" -#include "store/codec/Path.h" namespace dag::codec { inline store::CodecError file_error(const store::CodecOperation operation, const io::envelope::FileError& error) { - const bool missing = std::holds_alternative(error) && std::get(error) == io::Error(io::Error::OpenFile); return { operation, - missing ? store::CodecErrorCategory::FileNotFound + io::envelope::is_file_not_found(error) ? store::CodecErrorCategory::FileNotFound : operation == store::CodecOperation::Read ? store::CodecErrorCategory::InvalidData : store::CodecErrorCategory::Io, io::envelope::describe_error(error), @@ -122,4 +120,29 @@ class Dag final : public store::Codec { } }; +class MetadataView final : public store::Codec { +public: + std::vector paths(const store::NodePath& node_path) const override + { + return { store::codec::append_extension(node_path, ".dagmeta") }; + } + + std::expected read(const store::NodePath& node_path) const override + { + auto payload = io::envelope::read_from_path(paths(node_path).front()); + if (!payload) { + return std::unexpected(file_error(store::CodecOperation::Read, payload.error())); + } + auto metadata = dag::format::decode_metadata(std::move(*payload)); + if (!metadata) { + return std::unexpected(store::CodecError { + store::CodecOperation::Read, + store::CodecErrorCategory::InvalidData, + metadata.error(), + }); + } + return std::move(*metadata); + } +}; + } // namespace dag::codec diff --git a/src/dag_builder/codec/MetadataView.h b/src/dag_builder/codec/MetadataView.h deleted file mode 100644 index f668e507..00000000 --- a/src/dag_builder/codec/MetadataView.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include -#include - -#include "codec/Dag.h" -#include "codec/DagFormat.h" -#include "io/envelope_file.h" -#include "store/Codec.h" -#include "store/codec/Path.h" - -namespace dag::codec { - -class MetadataView final : public store::Codec { -public: - std::vector paths(const store::NodePath& node_path) const override - { - return { store::codec::append_extension(node_path, ".dagmeta") }; - } - - std::expected read(const store::NodePath& node_path) const override - { - auto payload = io::envelope::read_from_path(paths(node_path).front()); - if (!payload) { - return std::unexpected(file_error(store::CodecOperation::Read, payload.error())); - } - auto metadata = dag::format::decode_metadata(std::move(*payload)); - if (!metadata) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::InvalidData, - metadata.error(), - }); - } - return std::move(*metadata); - } -}; - -} // namespace dag::codec diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h index 928d93a5..0023aa97 100644 --- a/src/dag_builder/codec/from_extension.h +++ b/src/dag_builder/codec/from_extension.h @@ -7,7 +7,6 @@ #include #include "codec/Dag.h" -#include "codec/MetadataView.h" #include "dag_node.h" namespace dag::codec { diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index b733c3c6..de077fed 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -5,7 +5,7 @@ #include "cli.h" #include "build.h" #include "log.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" #include "storage.h" #include "store/describe_error.h" #include "sf/Error.h" @@ -17,7 +17,7 @@ int main(int argc, char **argv) { Log::init(args.log_level); try { - auto input_result = octree::open_folder_indexed(args.input_path); + auto input_result = mesh::storage::open_folder_indexed(args.input_path); if (!input_result.has_value()) { LOG_ERROR( "Failed to open input dataset {}: {}", diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h deleted file mode 100644 index 9a4c483d..00000000 --- a/src/dag_builder/thread_safe_storage.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -namespace dag { - -// A mutex-based thread-safe wrapper around a Storage instance. -// TODO: move this locking into Storage itself and remove this wrapper once that lands. -template -class ThreadSafeStorage { - static_assert(!std::is_const_v, "ThreadSafeStorage requires a non-const Storage"); - -public: - using value_type = typename Storage::value_type; - using key_type = typename Storage::key_type; - using load_error = typename Storage::load_error; - using save_error = typename Storage::save_error; - - explicit ThreadSafeStorage(Storage &&storage) - : _storage(std::move(storage)) {} - - ThreadSafeStorage(const ThreadSafeStorage &) = delete; - ThreadSafeStorage &operator=(const ThreadSafeStorage &) = delete; - ThreadSafeStorage(ThreadSafeStorage &&) = delete; - ThreadSafeStorage &operator=(ThreadSafeStorage &&) = delete; - - Storage release() && { - return std::move(this->_storage); - } - - std::expected load(const key_type &id) const { - std::shared_lock lock(this->_mutex); - return this->_storage.load(id); - } - - auto has(const key_type &id) const { - std::shared_lock lock(this->_mutex); - return this->_storage.has(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_storage.base_path(); - } - - std::expected save(const key_type &id, const value_type &value) const { - std::unique_lock lock(this->_mutex); - return this->_storage.save(id, value); - } - - auto save_or_create_index() const { - std::unique_lock lock(this->_mutex); - return this->_storage.save_or_create_index(); - } - -private: - mutable Storage _storage; - mutable std::shared_mutex _mutex; -}; - -} // namespace dag diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index 6f21e90a..772e8827 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -7,7 +7,7 @@ #include "log.h" #include "mesh/io.h" #include "mesh/storage.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" #include "ProgressIndicator.h" #include "storage.h" #include "utils.h" @@ -48,9 +48,9 @@ void export_storage(const cli::Args &args) { } const dag::storage::IndexedStorage input_storage = std::move(input_result.value()); - octree::OpenOptions options; + mesh::storage::OpenOptions options; options.preferred_extension = ".glb"; - auto output_result = octree::open_folder( + auto output_result = mesh::storage::open_folder( args.output_path, std::move(options)); if (!output_result.has_value()) { diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 68d830e1..e4e75b95 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -42,8 +41,8 @@ #include "octree/Id.h" #include "octree/Space.h" -#include "octree/storage/open.h" #include "octree/utils.h" +#include "store/ThreadSafeStorage.h" #include "store/describe_error.h" #include "sf/finalize_storage.h" @@ -174,10 +173,6 @@ T expect(const std::optional &opt, const std::string &msg) { } } -std::expected finalize_storage(mesh::storage::Storage &storage) { - return sf::finalize_storage(storage); -} - std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, @@ -195,9 +190,9 @@ std::expected build_all_patches( LOG_ERROR_AND_EXIT("Output base path {} exists but is not a directory", output_base_path); } - octree::OpenOptions open_options; + mesh::storage::OpenOptions open_options; open_options.preferred_extension = output_format; - auto storage_result = octree::open_folder( + auto storage_result = mesh::storage::open_folder( output_base_path, std::move(open_options)); if (!storage_result.has_value()) { @@ -206,9 +201,9 @@ std::expected build_all_patches( output_base_path, store::describe_error(storage_result.error())); } - mesh::storage::Storage storage = std::move(storage_result.value()); - storage.settings().allow_overwrite = overwrite_existing; - std::mutex storage_mutex; + mesh::storage::Storage raw_storage = std::move(storage_result.value()); + raw_storage.settings().allow_overwrite = overwrite_existing; + store::ThreadSafeStorage storage(std::move(raw_storage)); const auto dataset_srs = dataset.srs(); const auto dataset_bounds = dataset.bounds3d(true); @@ -293,10 +288,7 @@ std::expected build_all_patches( tbb::task_group_context context; tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; - const auto already_exists = [&] { - std::scoped_lock lock(storage_mutex); - return storage.has(node); - }(); + const auto already_exists = storage.has(node); if (!already_exists.has_value()) { LOG_ERROR( "Failed to inspect node {}: {}", @@ -328,10 +320,7 @@ std::expected build_all_patches( if (mesh_result.has_value()) { const auto mesh = std::move(mesh_result.value()); mesh::validate(mesh); - const auto save_result = [&] { - std::scoped_lock lock(storage_mutex); - return storage.save(node, mesh); - }(); + const auto save_result = storage.save(node, mesh); if (!save_result.has_value()) { LOG_ERROR( "Failed to save mesh for node {}: {}", @@ -358,6 +347,7 @@ std::expected build_all_patches( LOG_ERROR_AND_EXIT("Failed to build all terrain patches"); } - return finalize_storage(storage); + auto finalized_storage = std::move(storage).release(); + return sf::finalize_storage(finalized_storage); } } diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index 8ddbeba0..a59b06e9 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -31,8 +31,6 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -std::expected finalize_storage(mesh::storage::Storage &storage); - std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index f1008ff3..faa97119 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -14,7 +14,7 @@ #include "log.h" #include "octree/Id.h" #include "mesh/storage.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" #include "cli.h" #include "store/describe_error.h" #include "store/NodeStatusOrMissing.h" @@ -191,8 +191,8 @@ const octree::Id find_deepest_root( mesh::storage::IndexedStorage open_path_indexed(const std::filesystem::path& path) { auto result = std::filesystem::is_directory(path) - ? octree::open_folder_indexed(path) - : octree::open_index(path); + ? mesh::storage::open_folder_indexed(path) + : mesh::storage::open_index(path); if (!result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to open dataset {}: {}", diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 9a8d56a1..5613ce8d 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -9,7 +9,6 @@ #include "mask.h" #include "octree/Id.h" #include "mesh/storage.h" -#include "octree/storage/open.h" #include "octree/Space.h" #include "utils.h" #include "containers/Cow.h" @@ -155,10 +154,10 @@ inline std::expected cut_dataset( LOG_TRACE("Creating output dataset at {}", output_path); std::filesystem::create_directories(output_path); - octree::OpenOptions options; + mesh::storage::OpenOptions options; options.default_mapping = octree::store_layout::level_and_coordinate_directories(); options.preferred_extension = ".glb"; - auto output_result = octree::open_folder_indexed(output_path, std::move(options)); + auto output_result = mesh::storage::open_folder_indexed(output_path, std::move(options)); if (!output_result.has_value()) { return std::unexpected(sf::ProcessingError(output_result.error())); } diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 9fa72497..855752c7 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -7,6 +7,7 @@ #include "cut.h" #include "log.h" #include "mask.h" +#include "mesh/storage.h" #include "merge.h" #include "optional_utils.h" #include "earth.h" @@ -35,7 +36,7 @@ std::optional load_mask_from_path(const std::filesystem::path& path) { void run(const cli::MergeArgs& args) { LOG_TRACE("Loading base dataset from {}", args.base_path); - auto base_result = octree::open_folder_indexed(args.base_path); + auto base_result = mesh::storage::open_folder_indexed(args.base_path); if (!base_result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to open base dataset {}: {}", @@ -45,7 +46,7 @@ void run(const cli::MergeArgs& args) { mesh::storage::IndexedStorage base_dataset = std::move(base_result.value()); LOG_TRACE("Loading new dataset from {}", args.new_path); - auto new_result = octree::open_folder_indexed(args.new_path); + auto new_result = mesh::storage::open_folder_indexed(args.new_path); if (!new_result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to open new dataset {}: {}", @@ -56,10 +57,10 @@ void run(const cli::MergeArgs& args) { LOG_TRACE("Creating output dataset at {}", args.output_path); std::filesystem::create_directories(args.output_path); - octree::OpenOptions options; + mesh::storage::OpenOptions options; options.default_mapping = base_dataset.layout().mapping(); options.preferred_extension = std::string(base_dataset.codec_selector().value_or(".sfmesh")); - auto output_result = octree::open_folder(args.output_path, std::move(options)); + auto output_result = mesh::storage::open_folder(args.output_path, std::move(options)); if (!output_result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to open output dataset {}: {}", @@ -82,7 +83,7 @@ void run(const cli::MergeArgs& args) { void run(const cli::CutArgs& args) { LOG_TRACE("Loading input dataset from {}", args.input_path); - auto input_result = octree::open_folder_indexed(args.input_path); + auto input_result = mesh::storage::open_folder_indexed(args.input_path); if (!input_result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to open input dataset {}: {}", diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 10e0b846..f61342a3 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -135,9 +135,7 @@ add_library(terrainlib octree/OddLevelShifted.h octree/OddLevelShifted.cpp octree/Space.h octree/Space.cpp octree/StoreTraits.h - octree/storage/Format.h octree/storage/IndexFile.h - octree/storage/open.h octree/storage/open_runtime.h octree/store_layout/Flat.h octree/store_layout/LevelAndCoordinateDirectories.h @@ -191,12 +189,11 @@ add_library(terrainlib store/RawStorage.h store/Storage.h store/StorageError.h - store/StorageSettings.h + store/ThreadSafeStorage.h store/Traits.h store/cache/Dummy.h store/cache/Interface.h store/cache/Lru.h - store/codec/Path.h store/describe_error.h store/open.h store/traverse.h diff --git a/src/terrainlib/io/envelope_file.h b/src/terrainlib/io/envelope_file.h index 20c1b3e2..d70476c0 100644 --- a/src/terrainlib/io/envelope_file.h +++ b/src/terrainlib/io/envelope_file.h @@ -17,6 +17,12 @@ namespace io::envelope { using FileError = std::variant<::io::Error, Error>; +inline bool is_file_not_found(const FileError& error) +{ + const auto* io_error = std::get_if<::io::Error>(&error); + return io_error != nullptr && *io_error == ::io::Error(::io::Error::OpenFile); +} + inline std::string describe_error(const Error& error) { return "envelope error " + std::to_string(static_cast(error.code)); } inline std::string describe_error(const FileError& error) diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index 67f1082f..d9618711 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -9,7 +9,6 @@ #include "mesh/SimpleMesh.h" #include "mesh/io/gltf.h" #include "store/Codec.h" -#include "store/codec/Path.h" namespace mesh::codec { diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h index e9f4ada3..18851c36 100644 --- a/src/terrainlib/mesh/codec/SfMesh.h +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -7,7 +7,6 @@ #include "mesh/SimpleMesh.h" #include "mesh/codec/SfMeshFormat.h" #include "store/Codec.h" -#include "store/codec/Path.h" namespace mesh::codec { @@ -22,11 +21,9 @@ class SfMesh final : public store::Codec { { auto payload = ::io::envelope::read_from_path(paths(node_path).front()); if (!payload) { - const bool missing - = std::holds_alternative<::io::Error>(payload.error()) && std::get<::io::Error>(payload.error()) == ::io::Error(::io::Error::OpenFile); return std::unexpected(store::CodecError { store::CodecOperation::Read, - missing ? store::CodecErrorCategory::FileNotFound : store::CodecErrorCategory::InvalidData, + ::io::envelope::is_file_not_found(payload.error()) ? store::CodecErrorCategory::FileNotFound : store::CodecErrorCategory::InvalidData, ::io::envelope::describe_error(payload.error()), }); } diff --git a/src/terrainlib/octree/storage/Format.h b/src/terrainlib/octree/storage/Format.h deleted file mode 100644 index 40325e8f..00000000 --- a/src/terrainlib/octree/storage/Format.h +++ /dev/null @@ -1,125 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - -#include "io/envelope.h" -#include "octree/StoreTraits.h" -#include "store/Index.h" - -namespace octree::storage { - -inline constexpr std::string_view metadata_file_name = "octree.storemeta"; -inline constexpr std::string_view index_file_name = "octree.storeindex"; - -namespace v1 { - - struct StoreMetadata { - std::string layout_id; - std::string payload_class; - std::string codec_selector; - }; - - struct IndexEntry { - std::uint8_t level; - std::uint64_t index; - std::uint8_t status; - }; - - struct StoreIndex { - std::vector entries; - }; - -} // namespace v1 - -using StoreMetadataSchema = io::envelope::PayloadSchema<"octree.StoreMetadata", io::envelope::Version<1, v1::StoreMetadata>>; -using StoreIndexSchema = io::envelope::PayloadSchema<"octree.StoreIndex", io::envelope::Version<1, v1::StoreIndex>>; - -using StoreMetadata = StoreMetadataSchema::latest_type; -using StoreIndex = StoreIndexSchema::latest_type; - -inline std::expected validate(const StoreMetadata& metadata) -{ - if (metadata.layout_id.empty()) { - return std::unexpected("layout ID is empty"); - } - if (metadata.payload_class.empty()) { - return std::unexpected("payload class is empty"); - } - if (metadata.codec_selector.empty()) { - return std::unexpected("codec selector is empty"); - } - return {}; -} - -inline StoreIndex encode_index(const store::Index& index) -{ - StoreIndex result; - result.entries.reserve(index.size()); - for (const auto& [id, status] : index) { - result.entries.push_back({ - .level = id.level(), - .index = id.index_on_level(), - .status = static_cast(static_cast(status)), - }); - } - return result; -} - -inline std::expected, std::string> decode_index(const StoreIndex& encoded) -{ - store::Index result; - std::unordered_set seen; - for (const v1::IndexEntry& entry : encoded.entries) { - const auto id = Id::try_make(static_cast(entry.level), static_cast(entry.index)); - if (!id) { - return std::unexpected("index contains an invalid octree ID"); - } - if (!seen.insert(*id).second) { - return std::unexpected("index contains a duplicate octree ID"); - } - if (entry.status > static_cast(store::NodeStatus::Virtual)) { - return std::unexpected("index contains an invalid node status"); - } - const auto set = result.set_raw(*id, store::NodeStatus { static_cast(entry.status) }); - if (!set) { - return std::unexpected("index contains an invalid octree ID"); - } - } - - for (const auto& [id, status] : result) { - const auto children = StoreTraits::children(id); - bool has_child = false; - if (children) { - for (const Id& child : *children) { - const auto child_status = result.get(child); - if (!child_status) { - return std::unexpected("index child lookup failed"); - } - has_child = has_child || child_status->has_value(); - } - } - if ((status == store::NodeStatus::Leaf && has_child) || ((status == store::NodeStatus::Inner || status == store::NodeStatus::Virtual) && !has_child)) { - return std::unexpected("index contains inconsistent node topology"); - } - - const auto parent = StoreTraits::parent(id); - if (!parent) { - if (id != StoreTraits::root()) { - return std::unexpected("index contains a non-root node without a parent"); - } - continue; - } - const auto parent_status = result.get(*parent); - if (!parent_status || !parent_status->has_value() || parent_status->value() == store::NodeStatus::Leaf) { - return std::unexpected("index contains a node without a valid indexed parent"); - } - } - return result; -} - -} // namespace octree::storage diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 56d8a301..49ab35af 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -1,20 +1,132 @@ #pragma once +#include #include +#include +#include +#include #include "io/envelope_file.h" #include "octree/StoreTraits.h" -#include "octree/storage/Format.h" #include "octree/store_layout/Mappings.h" +#include "store/Index.h" #include "store/IndexFormat.h" namespace octree::storage { +inline constexpr std::string_view metadata_file_name = "octree.storemeta"; +inline constexpr std::string_view index_file_name = "octree.storeindex"; + +namespace v1 { + + struct StoreMetadata { + std::string layout_id; + std::string payload_class; + std::string codec_selector; + }; + + struct IndexEntry { + std::uint8_t level; + std::uint64_t index; + std::uint8_t status; + }; + + struct StoreIndex { + std::vector entries; + }; + +} // namespace v1 + +using StoreMetadataSchema = io::envelope::PayloadSchema<"octree.StoreMetadata", io::envelope::Version<1, v1::StoreMetadata>>; +using StoreIndexSchema = io::envelope::PayloadSchema<"octree.StoreIndex", io::envelope::Version<1, v1::StoreIndex>>; + +using StoreMetadata = StoreMetadataSchema::latest_type; +using StoreIndex = StoreIndexSchema::latest_type; + +inline std::expected validate(const StoreMetadata& metadata) +{ + if (metadata.layout_id.empty()) { + return std::unexpected("layout ID is empty"); + } + if (metadata.payload_class.empty()) { + return std::unexpected("payload class is empty"); + } + if (metadata.codec_selector.empty()) { + return std::unexpected("codec selector is empty"); + } + return {}; +} + +inline StoreIndex encode_index(const store::Index& index) +{ + StoreIndex result; + result.entries.reserve(index.size()); + for (const auto& [id, status] : index) { + result.entries.push_back({ + .level = id.level(), + .index = id.index_on_level(), + .status = static_cast(static_cast(status)), + }); + } + return result; +} + +inline std::expected, std::string> decode_index(const StoreIndex& encoded) +{ + store::Index result; + std::unordered_set seen; + for (const v1::IndexEntry& entry : encoded.entries) { + const auto id = Id::try_make(static_cast(entry.level), static_cast(entry.index)); + if (!id) { + return std::unexpected("index contains an invalid octree ID"); + } + if (!seen.insert(*id).second) { + return std::unexpected("index contains a duplicate octree ID"); + } + if (entry.status > static_cast(store::NodeStatus::Virtual)) { + return std::unexpected("index contains an invalid node status"); + } + const auto set = result.set_raw(*id, store::NodeStatus { static_cast(entry.status) }); + if (!set) { + return std::unexpected("index contains an invalid octree ID"); + } + } + + for (const auto& [id, status] : result) { + const auto children = StoreTraits::children(id); + bool has_child = false; + if (children) { + for (const Id& child : *children) { + const auto child_status = result.get(child); + if (!child_status) { + return std::unexpected("index child lookup failed"); + } + has_child = has_child || child_status->has_value(); + } + } + if ((status == store::NodeStatus::Leaf && has_child) || ((status == store::NodeStatus::Inner || status == store::NodeStatus::Virtual) && !has_child)) { + return std::unexpected("index contains inconsistent node topology"); + } + + const auto parent = StoreTraits::parent(id); + if (!parent) { + if (id != StoreTraits::root()) { + return std::unexpected("index contains a non-root node without a parent"); + } + continue; + } + const auto parent_status = result.get(*parent); + if (!parent_status || !parent_status->has_value() || parent_status->value() == store::NodeStatus::Leaf) { + return std::unexpected("index contains a node without a valid indexed parent"); + } + } + return result; +} + inline store::IndexFormatError read_error(const std::filesystem::path& path, const io::envelope::FileError& error) { - const bool missing = std::holds_alternative(error) && std::get(error) == io::Error(io::Error::OpenFile); return { - missing ? store::IndexFormatErrorCategory::Open : store::IndexFormatErrorCategory::Malformed, + io::envelope::is_file_not_found(error) ? store::IndexFormatErrorCategory::Open : store::IndexFormatErrorCategory::Malformed, path, io::envelope::describe_error(error), }; diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h deleted file mode 100644 index cfd10db0..00000000 --- a/src/terrainlib/octree/storage/open.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include -#include - -#include "mesh/storage.h" - -namespace octree { - -using OpenOptions = mesh::storage::OpenOptions; - -inline std::expected> open_index( - const std::filesystem::path &path) { - return mesh::storage::open_index(path); -} - -inline std::expected> open_folder( - const std::filesystem::path &path, - OpenOptions options = {}) { - return mesh::storage::open_folder(path, std::move(options)); -} - -inline std::expected> open_folder_indexed( - const std::filesystem::path &path, - OpenOptions options = {}) { - return mesh::storage::open_folder_indexed(path, std::move(options)); -} - -} // namespace octree diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h index d112f69f..6e1921ef 100644 --- a/src/terrainlib/store/Codec.h +++ b/src/terrainlib/store/Codec.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -10,6 +11,17 @@ namespace store { +namespace codec { + +inline std::filesystem::path append_extension(const NodePath& node_path, const std::string_view extension) +{ + std::filesystem::path result = node_path.path(); + result += extension; + return result; +} + +} // namespace codec + template class Codec { public: diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index c04f2e9b..4ed0104a 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -11,10 +11,13 @@ #include "log.h" #include "store/IndexFormat.h" #include "store/RawStorage.h" -#include "store/StorageSettings.h" namespace store { +struct StorageSettings { + bool allow_overwrite = false; +}; + template class Storage { public: diff --git a/src/terrainlib/store/StorageSettings.h b/src/terrainlib/store/StorageSettings.h deleted file mode 100644 index 2a883ee1..00000000 --- a/src/terrainlib/store/StorageSettings.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -namespace store { - -struct StorageSettings { - bool allow_overwrite = false; -}; - -} // namespace store diff --git a/src/terrainlib/store/ThreadSafeStorage.h b/src/terrainlib/store/ThreadSafeStorage.h new file mode 100644 index 00000000..56427038 --- /dev/null +++ b/src/terrainlib/store/ThreadSafeStorage.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace store { + +template +class ThreadSafeStorage { + static_assert(!std::is_const_v, "ThreadSafeStorage requires a non-const Storage"); + +public: + using value_type = typename Storage::value_type; + using key_type = typename Storage::key_type; + using load_error = typename Storage::load_error; + using save_error = typename Storage::save_error; + + explicit ThreadSafeStorage(Storage&& storage) + : m_storage(std::move(storage)) + { + } + + ThreadSafeStorage(const ThreadSafeStorage&) = delete; + ThreadSafeStorage& operator=(const ThreadSafeStorage&) = delete; + ThreadSafeStorage(ThreadSafeStorage&&) = delete; + ThreadSafeStorage& operator=(ThreadSafeStorage&&) = delete; + + Storage release() && { return std::move(m_storage); } + + std::expected load(const key_type& key) const + { + std::shared_lock lock(m_mutex); + return m_storage.load(key); + } + + auto has(const key_type& key) const + { + std::shared_lock lock(m_mutex); + return m_storage.has(key); + } + + std::filesystem::path base_path() const noexcept { return m_storage.base_path(); } + + std::expected save(const key_type& key, const value_type& value) const + { + std::unique_lock lock(m_mutex); + return m_storage.save(key, value); + } + + auto save_or_create_index() const + { + std::unique_lock lock(m_mutex); + return m_storage.save_or_create_index(); + } + +private: + mutable Storage m_storage; + mutable std::shared_mutex m_mutex; +}; + +} // namespace store diff --git a/src/terrainlib/store/codec/Path.h b/src/terrainlib/store/codec/Path.h deleted file mode 100644 index 81529800..00000000 --- a/src/terrainlib/store/codec/Path.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include -#include - -#include "store/NodePath.h" - -namespace store::codec { - -inline std::filesystem::path append_extension(const NodePath& node_path, const std::string_view extension) -{ - std::filesystem::path result = node_path.path(); - result += extension; - return result; -} - -} // namespace store::codec diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index d3338950..341dec90 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -15,6 +15,7 @@ endfunction() add_executable(unittests_terrainlib catch2_helpers.h opencv_helpers.h + temporary_directory.h terrainlib/convert.cpp terrainlib/cow.cpp @@ -100,6 +101,7 @@ if(TARGET sfbuilderlib) add_executable(unittests_sfbuilder_finalization catch2_helpers.h + temporary_directory.h sf_builder/storage_finalization.cpp ) target_link_libraries( @@ -125,6 +127,7 @@ endif() if(TARGET dagbuilderlib) add_executable(unittests_dagbuilder catch2_helpers.h + temporary_directory.h dag_builder/bake_cluster_texture.cpp dag_builder/correspondence.cpp dag_builder/reprojection.cpp @@ -145,6 +148,7 @@ endif() if(TARGET sfmergerlib) add_executable(unittests_sfmerger catch2_helpers.h + temporary_directory.h sf_merger/mask.cpp sf_merger/sphere_projector.cpp sf_merger/storage_boundaries.cpp diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 925b3810..a0491493 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -1,5 +1,3 @@ -#include -#include #include #include #include @@ -10,31 +8,14 @@ #include "codec/Dag.h" #include "dag_node.h" #include "mesh/SimpleMesh.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" #include "storage.h" -#include "thread_safe_storage.h" +#include "store/ThreadSafeStorage.h" +#include "../temporary_directory.h" namespace { -class TemporaryDirectory { -public: - TemporaryDirectory() - { - static std::atomic_uint64_t counter = 0; - const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - m_path = std::filesystem::temp_directory_path() / ("atb-dag-codec-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(m_path)); - } - ~TemporaryDirectory() - { - std::error_code error; - std::filesystem::remove_all(m_path, error); - } - const std::filesystem::path& path() const { return m_path; } - -private: - std::filesystem::path m_path; -}; +using test::TemporaryDirectory; dag::ClusterBatch sample_batch() { @@ -167,7 +148,7 @@ TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[s TemporaryDirectory output; auto storage_result = dag::storage::open_folder_indexed(output.path()); REQUIRE(storage_result.has_value()); - dag::ThreadSafeStorage synchronized(std::move(storage_result.value())); + store::ThreadSafeStorage synchronized(std::move(storage_result.value())); REQUIRE(synchronized.save(octree::Id::root(), batch).has_value()); auto released = std::move(synchronized).release(); @@ -220,7 +201,7 @@ TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf { TemporaryDirectory input_directory; TemporaryDirectory output_directory; - auto input_result = octree::open_folder_indexed(input_directory.path()); + auto input_result = mesh::storage::open_folder_indexed(input_directory.path()); REQUIRE(input_result.has_value()); auto input = std::move(input_result.value()); const octree::Id root = octree::Id::root(); diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp index 8c92a884..339b540b 100644 --- a/unittests/sf_builder/storage_finalization.cpp +++ b/unittests/sf_builder/storage_finalization.cpp @@ -1,35 +1,14 @@ -#include -#include #include #include -#include "octree/storage/open.h" -#include "terrainbuilder.h" +#include "mesh/storage.h" +#include "sf/finalize_storage.h" +#include "../temporary_directory.h" namespace { -class TemporaryDirectory { -public: - TemporaryDirectory() - { - static std::atomic_uint64_t counter = 0; - const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - m_path = std::filesystem::temp_directory_path() / ("atb-sfbuilder-finalize-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(m_path)); - } - - ~TemporaryDirectory() - { - std::error_code error; - std::filesystem::remove_all(m_path, error); - } - - const std::filesystem::path& path() const { return m_path; } - -private: - std::filesystem::path m_path; -}; +using test::TemporaryDirectory; mesh::Simple sample_mesh() { return mesh::Simple({ { 0, 1, 2 } }, { { 0.0, 0.0, 0.0 }, { 1.0, 0.0, 0.0 }, { 0.0, 1.0, 0.0 } }); } @@ -38,12 +17,12 @@ mesh::Simple sample_mesh() { return mesh::Simple({ { 0, 1, 2 } }, { { 0.0, 0.0, TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuilder][sf]") { TemporaryDirectory directory; - auto storage_result = octree::open_folder(directory.path()); + auto storage_result = mesh::storage::open_folder(directory.path()); REQUIRE(storage_result.has_value()); auto storage = std::move(storage_result.value()); REQUIRE(storage.save(octree::Id::root(), sample_mesh()).has_value()); - CHECK(terrainbuilder::finalize_storage(storage).has_value()); + CHECK(sf::finalize_storage(storage).has_value()); CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storemeta")); } @@ -51,7 +30,7 @@ TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuil TEST_CASE("SF builder finalization retains an invalid written index for diagnosis", "[sfbuilder][sf]") { TemporaryDirectory directory; - auto storage_result = octree::open_folder(directory.path()); + auto storage_result = mesh::storage::open_folder(directory.path()); REQUIRE(storage_result.has_value()); auto storage = std::move(storage_result.value()); const octree::Id root = octree::Id::root(); @@ -59,13 +38,13 @@ TEST_CASE("SF builder finalization retains an invalid written index for diagnosi REQUIRE(storage.save(root, mesh).has_value()); REQUIRE(storage.save(root.child(0).value(), mesh).has_value()); - const auto finalized = terrainbuilder::finalize_storage(storage); + const auto finalized = sf::finalize_storage(storage); REQUIRE_FALSE(finalized.has_value()); REQUIRE(std::holds_alternative(finalized.error())); CHECK(std::get(finalized.error()).key == root); CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); - auto reopened = octree::open_index(directory.path() / "octree.storeindex"); + auto reopened = mesh::storage::open_index(directory.path() / "octree.storeindex"); REQUIRE(reopened.has_value()); CHECK(reopened->index().is(store::NodeStatus::Inner, root).value()); } diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp index f583705f..5bd50cc7 100644 --- a/unittests/sf_merger/storage_boundaries.cpp +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -1,37 +1,15 @@ -#include -#include #include #include #include "cut.h" #include "merge.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" +#include "../temporary_directory.h" namespace { -class TemporaryDirectory { -public: - explicit TemporaryDirectory(const std::string_view label) - { - static std::atomic_uint64_t counter = 0; - const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - m_path = std::filesystem::temp_directory_path() - / ("atb-sfmerger-" + std::string(label) + "-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(m_path)); - } - - ~TemporaryDirectory() - { - std::error_code error; - std::filesystem::remove_all(m_path, error); - } - - const std::filesystem::path& path() const { return m_path; } - -private: - std::filesystem::path m_path; -}; +using test::TemporaryDirectory; mesh::Simple triangle(const double x_offset = 0.0) { @@ -74,14 +52,14 @@ mesh::Simple clipping_box() mesh::storage::IndexedStorage indexed_storage(const std::filesystem::path& path) { - auto result = octree::open_folder_indexed(path); + auto result = mesh::storage::open_folder_indexed(path); REQUIRE(result.has_value()); return std::move(result.value()); } mesh::storage::Storage unindexed_storage(const std::filesystem::path& path) { - auto result = octree::open_folder(path); + auto result = mesh::storage::open_folder(path); REQUIRE(result.has_value()); return std::move(result.value()); } diff --git a/unittests/temporary_directory.h b/unittests/temporary_directory.h new file mode 100644 index 00000000..837e77fb --- /dev/null +++ b/unittests/temporary_directory.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace test { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label = {}) + { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + std::string name = "atb-test"; + if (!label.empty()) { + name += "-" + std::string(label); + } + name += "-" + std::to_string(timestamp) + "-" + std::to_string(counter++); + m_path = std::filesystem::temp_directory_path() / name; + REQUIRE(std::filesystem::create_directories(m_path)); + } + + ~TemporaryDirectory() + { + std::error_code error; + std::filesystem::remove_all(m_path, error); + } + + const std::filesystem::path& path() const { return m_path; } + +private: + std::filesystem::path m_path; +}; + +} // namespace test diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp index bbdfa2dd..d46499eb 100644 --- a/unittests/terrainlib/store_codec.cpp +++ b/unittests/terrainlib/store_codec.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -11,29 +10,11 @@ #include "mesh/codec/Gltf.h" #include "mesh/codec/SfMesh.h" #include "store/Codec.h" +#include "../temporary_directory.h" namespace { -class TemporaryDirectory { -public: - explicit TemporaryDirectory(const std::string_view label) - { - static std::atomic_uint64_t counter = 0; - const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - m_path = std::filesystem::temp_directory_path() - / ("atb-store-codec-" + std::string(label) + "-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(m_path)); - } - ~TemporaryDirectory() - { - std::error_code error; - std::filesystem::remove_all(m_path, error); - } - const std::filesystem::path& path() const { return m_path; } - -private: - std::filesystem::path m_path; -}; +using test::TemporaryDirectory; class MultiFileCodec final : public store::Codec { public: diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 94c565d9..d8fd880e 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -1,6 +1,4 @@ #include -#include -#include #include #include #include @@ -15,33 +13,12 @@ #include "mesh/codec/from_extension.h" #include "mesh/storage.h" #include "octree/storage/IndexFile.h" -#include "octree/storage/open.h" #include "octree/store_layout/Mappings.h" +#include "../temporary_directory.h" namespace { -class TemporaryDirectory { -public: - explicit TemporaryDirectory(const std::string_view label) - { - static std::atomic_uint64_t counter = 0; - const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - m_path = std::filesystem::temp_directory_path() - / ("atb-raster-store-" + std::string(label) + "-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(m_path)); - } - - ~TemporaryDirectory() - { - std::error_code error; - std::filesystem::remove_all(m_path, error); - } - - const std::filesystem::path& path() const { return m_path; } - -private: - std::filesystem::path m_path; -}; +using test::TemporaryDirectory; mesh::Simple triangle_mesh(const double offset) { @@ -157,12 +134,12 @@ TEST_CASE("storage supports enabled overwrites") TEST_CASE("folder opening persists and reopens metadata-driven storage") { TemporaryDirectory directory("folder-open"); - octree::OpenOptions options; + mesh::storage::OpenOptions options; options.default_mapping = octree::store_layout::flat(); options.preferred_extension = ".sfmesh"; { - auto storage_result = octree::open_folder(directory.path(), std::move(options)); + auto storage_result = mesh::storage::open_folder(directory.path(), std::move(options)); REQUIRE(storage_result.has_value()); auto storage = std::move(storage_result.value()); CHECK(storage.is_indexed()); @@ -173,7 +150,7 @@ TEST_CASE("folder opening persists and reopens metadata-driven storage") } { - auto storage_result = octree::open_folder_indexed(directory.path()); + auto storage_result = mesh::storage::open_folder_indexed(directory.path()); REQUIRE(storage_result.has_value()); const auto storage = std::move(storage_result.value()); CHECK(storage.is_indexed()); @@ -181,7 +158,7 @@ TEST_CASE("folder opening persists and reopens metadata-driven storage") CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); } - const auto reopened = octree::open_index(directory.path() / "octree.storeindex"); + const auto reopened = mesh::storage::open_index(directory.path() / "octree.storeindex"); REQUIRE(reopened.has_value()); CHECK(reopened->has(octree::Id::root()).value()); @@ -220,7 +197,7 @@ TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store] }; REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); - const auto result = octree::open_index(index_path); + const auto result = mesh::storage::open_index(index_path); REQUIRE_FALSE(result.has_value()); REQUIRE(std::holds_alternative(result.error())); CHECK(std::get(result.error()).category == store::CodecErrorCategory::UnsupportedCodec); @@ -239,7 +216,7 @@ TEST_CASE("octree opening retains index failures without directory fallback", "[ }; REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); - const auto unknown_layout = octree::open_folder_indexed(directory.path()); + const auto unknown_layout = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_layout.has_value()); REQUIRE(std::holds_alternative(unknown_layout.error())); CHECK(std::get(unknown_layout.error()).id == "unknown-layout"); @@ -247,14 +224,14 @@ TEST_CASE("octree opening retains index failures without directory fallback", "[ index_file.layout_id = "flat"; index_file.codec_selector = ".unknown"; REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); - const auto unknown_codec = octree::open_folder_indexed(directory.path()); + const auto unknown_codec = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_codec.has_value()); REQUIRE(std::holds_alternative(unknown_codec.error())); CHECK(std::get(unknown_codec.error()).category == store::CodecErrorCategory::UnsupportedCodec); const std::array malformed_bytes { 0xff, 0x00, 0x01 }; REQUIRE(io::write_bytes_to_path(malformed_bytes, index_path).has_value()); - const auto malformed = octree::open_folder_indexed(directory.path()); + const auto malformed = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(malformed.has_value()); REQUIRE(std::holds_alternative(malformed.error())); CHECK(std::get(malformed.error()).category == store::IndexFormatErrorCategory::Malformed); diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index c99b50e2..c55a65d2 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -16,29 +15,11 @@ #include "raster_store/StoreTraits.h" #include "store/IndexedStorage.h" #include "string_utils.h" +#include "../temporary_directory.h" namespace { -class TemporaryDirectory { -public: - explicit TemporaryDirectory(const std::string_view label) - { - static std::atomic_uint64_t counter = 0; - const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); - m_path = std::filesystem::temp_directory_path() - / ("atb-store-storage-" + std::string(label) + "-" + std::to_string(timestamp) + "-" + std::to_string(counter++)); - REQUIRE(std::filesystem::create_directories(m_path)); - } - ~TemporaryDirectory() - { - std::error_code error; - std::filesystem::remove_all(m_path, error); - } - const std::filesystem::path& path() const { return m_path; } - -private: - std::filesystem::path m_path; -}; +using test::TemporaryDirectory; store::NodePath tile_to_path(const radix::tile::Id& key) { From ba1ef5b8e73b0c48ae43685b5f52aaec280f1535 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:21:31 +0200 Subject: [PATCH 26/43] Remove redundant NodePath wrapper --- src/dag_builder/codec/Dag.h | 16 +++---- src/dag_convert_debug/main.cpp | 2 +- src/terrainlib/CMakeLists.txt | 1 - src/terrainlib/mesh/codec/Gltf.h | 8 ++-- src/terrainlib/mesh/codec/SfMesh.h | 10 ++--- src/terrainlib/mesh/io.cpp | 7 +-- src/terrainlib/octree/store_layout/Flat.h | 10 ++--- .../LevelAndCoordinateDirectories.h | 12 +++--- src/terrainlib/store/Codec.h | 25 +++++------ src/terrainlib/store/Layout.h | 9 ++-- src/terrainlib/store/NodePath.h | 24 ----------- src/terrainlib/store/PathMapping.h | 7 ++- src/terrainlib/store/RawStorage.h | 2 +- src/terrainlib/store/Storage.h | 6 +-- .../dag_builder/storage_compatibility.cpp | 10 ++--- unittests/terrainlib/store_codec.cpp | 28 +++++------- unittests/terrainlib/store_compatibility.cpp | 2 +- unittests/terrainlib/store_layout.cpp | 20 ++++----- unittests/terrainlib/store_storage.cpp | 43 ++++++++----------- 19 files changed, 94 insertions(+), 148 deletions(-) delete mode 100644 src/terrainlib/store/NodePath.h diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index e96f86e0..38170336 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -34,15 +34,15 @@ inline std::expected validate_batch(const dag::ClusterB class Dag final : public store::Codec { public: - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { return { - store::codec::append_extension(node_path, ".dag"), - store::codec::append_extension(node_path, ".dagmeta"), + add_extension(node_path, ".dag"), + add_extension(node_path, ".dagmeta"), }; } - std::expected read(const store::NodePath& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { const auto node_paths = paths(node_path); auto clustering_payload = io::envelope::read_from_path(node_paths[0]); @@ -79,7 +79,7 @@ class Dag final : public store::Codec { return batch; } - std::expected write(const store::NodePath& node_path, const dag::ClusterBatch& batch) const override + std::expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override { if (auto valid = validate_batch(batch, store::CodecOperation::Write); !valid) { return valid; @@ -122,12 +122,12 @@ class Dag final : public store::Codec { class MetadataView final : public store::Codec { public: - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { - return { store::codec::append_extension(node_path, ".dagmeta") }; + return { add_extension(node_path, ".dagmeta") }; } - std::expected read(const store::NodePath& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { auto payload = io::envelope::read_from_path(paths(node_path).front()); if (!payload) { diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index 772e8827..aa1e4604 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -23,7 +23,7 @@ void export_node(const cli::Args &args) { std::filesystem::path node_path = args.input_path; node_path.replace_extension(); const dag::codec::Dag codec; - const auto load_result = codec.read(store::NodePath(node_path)); + const auto load_result = codec.read(node_path); if (!load_result.has_value()) { LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error().message); return; diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index f61342a3..8abbf76b 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -181,7 +181,6 @@ add_library(terrainlib store/IndexedStorage.h store/InvalidKey.h store/Layout.h - store/NodePath.h store/NodeStatus.h store/NodeStatusOrMissing.h store/OpenError.h diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index d9618711..5927b25c 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -24,12 +24,12 @@ class Gltf final : public store::Codec { { } - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { - return { store::codec::append_extension(node_path, m_container == GltfContainer::Binary ? ".glb" : ".gltf") }; + return { add_extension(node_path, m_container == GltfContainer::Binary ? ".glb" : ".gltf") }; } - std::expected read(const store::NodePath& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { const auto result = mesh::io::gltf::load_from_path(paths(node_path).front()); if (!result.has_value()) { @@ -42,7 +42,7 @@ class Gltf final : public store::Codec { return std::move(result.value()); } - std::expected write(const store::NodePath& node_path, const mesh::Simple& mesh) const override + std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { try { const auto result = mesh::io::gltf::save_to_path(mesh, paths(node_path).front()); diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h index 18851c36..67d95be5 100644 --- a/src/terrainlib/mesh/codec/SfMesh.h +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -12,12 +12,12 @@ namespace mesh::codec { class SfMesh final : public store::Codec { public: - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { - return { store::codec::append_extension(node_path, ".sfmesh") }; + return { add_extension(node_path, ".sfmesh") }; } - std::expected read(const store::NodePath& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { auto payload = ::io::envelope::read_from_path(paths(node_path).front()); if (!payload) { @@ -45,12 +45,12 @@ class SfMesh final : public store::Codec { return std::move(*decoded); } - std::expected write(const store::NodePath& node_path, const mesh::Simple& mesh) const override + std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { return write(node_path, mesh, {}); } - std::expected write(const store::NodePath& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const + std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const { auto payload = mesh::sf::encode_payload(mesh, options); if (!payload) { diff --git a/src/terrainlib/mesh/io.cpp b/src/terrainlib/mesh/io.cpp index d1a9be1d..7be3eee4 100644 --- a/src/terrainlib/mesh/io.cpp +++ b/src/terrainlib/mesh/io.cpp @@ -16,7 +16,7 @@ std::expected load_from_path( std::filesystem::path node_path = path; node_path.replace_extension(); const mesh::codec::SfMesh codec; - auto result = codec.read(store::NodePath(node_path)); + auto result = codec.read(node_path); if (!result) { return std::unexpected(LoadMeshErrorKind::InvalidFormat); } @@ -41,10 +41,7 @@ std::expected save_to_path( std::filesystem::path node_path = path; node_path.replace_extension(); const mesh::codec::SfMesh codec; - auto result = codec.write( - store::NodePath(node_path), - mesh, - mesh::EncodeOptions{.texture_format = options.texture_format}); + auto result = codec.write(node_path, mesh, mesh::EncodeOptions{.texture_format = options.texture_format}); if (!result) { return std::unexpected(SaveMeshErrorKind::WriteFile); } diff --git a/src/terrainlib/octree/store_layout/Flat.h b/src/terrainlib/octree/store_layout/Flat.h index e05f7f14..f7f6944e 100644 --- a/src/terrainlib/octree/store_layout/Flat.h +++ b/src/terrainlib/octree/store_layout/Flat.h @@ -8,21 +8,19 @@ #include #include "octree/Id.h" -#include "store/NodePath.h" #include "string_utils.h" namespace octree::store_layout { -inline store::NodePath flat_key_to_node_path(const Id& id) { return store::NodePath(fmt::format("{}-{}", id.level(), id.index_on_level())); } +inline std::filesystem::path flat_key_to_node_path(const Id& id) { return fmt::format("{}-{}", id.level(), id.index_on_level()); } -inline std::optional flat_node_path_to_key(const store::NodePath& node_path) +inline std::optional flat_node_path_to_key(const std::filesystem::path& node_path) { - const std::filesystem::path& path = node_path.path(); - if (path.empty() || path.is_absolute() || path.has_parent_path() || path.has_extension()) { + if (node_path.empty() || node_path.is_absolute() || node_path.has_parent_path() || node_path.has_extension()) { return std::nullopt; } - const std::string text = path.string(); + const std::string text = node_path.string(); const size_t separator = text.find('-'); if (separator == std::string::npos || separator != text.rfind('-')) { return std::nullopt; diff --git a/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h index 9545adad..2dac849b 100644 --- a/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h +++ b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h @@ -7,25 +7,23 @@ #include #include "octree/Id.h" -#include "store/NodePath.h" #include "string_utils.h" namespace octree::store_layout { -inline store::NodePath level_and_coordinate_key_to_node_path(const Id& id) +inline std::filesystem::path level_and_coordinate_key_to_node_path(const Id& id) { const Id::Coords coordinates = id.coords(); - return store::NodePath(fmt::format("{}/{}/{}/{}", id.level(), coordinates.x, coordinates.y, coordinates.z)); + return fmt::format("{}/{}/{}/{}", id.level(), coordinates.x, coordinates.y, coordinates.z); } -inline std::optional level_and_coordinate_node_path_to_key(const store::NodePath& node_path) +inline std::optional level_and_coordinate_node_path_to_key(const std::filesystem::path& node_path) { - const std::filesystem::path& path = node_path.path(); - if (path.empty() || path.is_absolute() || std::distance(path.begin(), path.end()) != 4) { + if (node_path.empty() || node_path.is_absolute() || std::distance(node_path.begin(), node_path.end()) != 4) { return std::nullopt; } - auto part = path.begin(); + auto part = node_path.begin(); const auto level = from_chars(part->string()); ++part; const auto x = from_chars(part->string()); diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h index 6e1921ef..b45b6971 100644 --- a/src/terrainlib/store/Codec.h +++ b/src/terrainlib/store/Codec.h @@ -7,37 +7,32 @@ #include #include "store/CodecError.h" -#include "store/NodePath.h" namespace store { -namespace codec { - -inline std::filesystem::path append_extension(const NodePath& node_path, const std::string_view extension) -{ - std::filesystem::path result = node_path.path(); - result += extension; - return result; -} - -} // namespace codec - template class Codec { public: virtual ~Codec() = default; - virtual std::vector paths(const NodePath& node_path) const = 0; + virtual std::vector paths(const std::filesystem::path& node_path) const = 0; - virtual std::expected read(const NodePath&) const + virtual std::expected read(const std::filesystem::path&) const { return std::unexpected(CodecError::unsupported_operation(CodecOperation::Read, "read")); } - virtual std::expected write(const NodePath&, const NodeData&) const + virtual std::expected write(const std::filesystem::path&, const NodeData&) const { return std::unexpected(CodecError::unsupported_operation(CodecOperation::Write, "write")); } + +protected: + static std::filesystem::path add_extension(std::filesystem::path path, const std::string_view extension) + { + path += extension; + return path; + } }; } // namespace store diff --git a/src/terrainlib/store/Layout.h b/src/terrainlib/store/Layout.h index 0bbec70d..dee23991 100644 --- a/src/terrainlib/store/Layout.h +++ b/src/terrainlib/store/Layout.h @@ -4,7 +4,6 @@ #include #include -#include "store/NodePath.h" #include "store/PathMapping.h" namespace store { @@ -18,16 +17,16 @@ class Layout { { } - NodePath node_path(const Key& key) const { return NodePath(m_base_path / m_mapping.key_to_node_path(key).path()); } + std::filesystem::path node_path(const Key& key) const { return m_base_path / m_mapping.key_to_node_path(key); } - std::optional key_from_node_path(const NodePath& node_path) const + std::optional key_from_node_path(const std::filesystem::path& node_path) const { std::error_code error; - const std::filesystem::path relative = std::filesystem::relative(node_path.path(), m_base_path, error); + const std::filesystem::path relative = std::filesystem::relative(node_path, m_base_path, error); if (error || relative.empty() || relative.is_absolute()) { return std::nullopt; } - return m_mapping.node_path_to_key(NodePath(relative)); + return m_mapping.node_path_to_key(relative); } const std::filesystem::path& base_path() const { return m_base_path; } diff --git a/src/terrainlib/store/NodePath.h b/src/terrainlib/store/NodePath.h deleted file mode 100644 index 3307ec9a..00000000 --- a/src/terrainlib/store/NodePath.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include - -namespace store { - -class NodePath { -public: - NodePath() = default; - explicit NodePath(std::filesystem::path path) - : m_path(std::move(path)) - { - } - - const std::filesystem::path& path() const { return m_path; } - - bool operator==(const NodePath&) const = default; - -private: - std::filesystem::path m_path; -}; - -} // namespace store diff --git a/src/terrainlib/store/PathMapping.h b/src/terrainlib/store/PathMapping.h index 771f3381..a1461f18 100644 --- a/src/terrainlib/store/PathMapping.h +++ b/src/terrainlib/store/PathMapping.h @@ -1,17 +1,16 @@ #pragma once +#include #include #include -#include "store/NodePath.h" - namespace store { template struct PathMapping { std::string_view id; - NodePath (*key_to_node_path)(const Key&); - std::optional (*node_path_to_key)(const NodePath&); + std::filesystem::path (*key_to_node_path)(const Key&); + std::optional (*node_path_to_key)(const std::filesystem::path&); bool operator==(const PathMapping&) const = default; }; diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index a61cfdfa..fc7bb38d 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -131,7 +131,7 @@ class RawStorage { return std::unexpected(CopyError(MissingSource { key })); } - const NodePath probe("__codec_probe__/node"); + const std::filesystem::path probe("__codec_probe__/node"); if (m_codec->paths(probe) != source.m_codec->paths(probe)) { const auto loaded = source.m_codec->read(source.m_layout.node_path(key)); if (!loaded.has_value()) { diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index 4ed0104a..282f74ba 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -113,7 +113,7 @@ class Storage { if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); return std::unexpected(SaveError(AlreadyExists { - node_paths.empty() ? m_raw.layout().node_path(key).path() : node_paths.front(), + node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front(), })); } @@ -143,7 +143,7 @@ class Storage { if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); return std::unexpected(CopyError(AlreadyExists { - node_paths.empty() ? m_raw.layout().node_path(key).path() : node_paths.front(), + node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front(), })); } const auto prepare_target = [&]() -> std::expected> { @@ -211,7 +211,7 @@ class Storage { if (!node_paths.has_value()) { return std::unexpected(node_paths.error()); } - return node_paths->empty() ? m_raw.layout().node_path(key).path() : node_paths->front(); + return node_paths->empty() ? m_raw.layout().node_path(key) : node_paths->front(); } const std::filesystem::path& base_path() const { return m_raw.layout().base_path(); } diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index a0491493..a0a2e861 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -55,7 +55,7 @@ TEST_CASE("runtime DAG envelope codec is reentrant") std::vector> operations; for (int index = 0; index < 8; ++index) { operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { - const store::NodePath path(output.path() / std::to_string(index) / "node"); + const std::filesystem::path path(output.path() / std::to_string(index) / "node"); if (!codec.write(path, batch).has_value()) { return false; } @@ -72,7 +72,7 @@ TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") { TemporaryDirectory output; dag::codec::Dag codec; - const store::NodePath node_path(output.path() / "node"); + const std::filesystem::path node_path(output.path() / "node"); const auto node_paths = codec.paths(node_path); SECTION("write") @@ -126,12 +126,12 @@ TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[stor { const auto batch_codec = dag::codec::from_extension(".dag"); REQUIRE(batch_codec.has_value()); - CHECK(batch_codec.value()->paths(store::NodePath("node")) == std::vector { "node.dag", "node.dagmeta" }); + CHECK(batch_codec.value()->paths("node") == std::vector { "node.dag", "node.dagmeta" }); const auto metadata_codec = dag::codec::metadata_from_extension(".dag"); REQUIRE(metadata_codec.has_value()); - CHECK(metadata_codec.value()->paths(store::NodePath("node")) == std::vector { "node.dagmeta" }); - const auto write_result = metadata_codec.value()->write(store::NodePath("node"), dag::NodeMetadata {}); + CHECK(metadata_codec.value()->paths("node") == std::vector { "node.dagmeta" }); + const auto write_result = metadata_codec.value()->write("node", dag::NodeMetadata {}); REQUIRE_FALSE(write_result.has_value()); CHECK(write_result.error().operation == store::CodecOperation::Write); CHECK(write_result.error().category == store::CodecErrorCategory::UnsupportedOperation); diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp index d46499eb..45ef42df 100644 --- a/unittests/terrainlib/store_codec.cpp +++ b/unittests/terrainlib/store_codec.cpp @@ -18,25 +18,19 @@ using test::TemporaryDirectory; class MultiFileCodec final : public store::Codec { public: - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { - std::filesystem::path data = node_path.path(); - std::filesystem::path metadata = node_path.path(); - data += ".data"; - metadata += ".metadata"; - return { data, metadata }; + return { add_extension(node_path, ".data"), add_extension(node_path, ".metadata") }; } }; class WriteOnlyCodec final : public store::Codec { public: - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { - std::filesystem::path path = node_path.path(); - path += ".out"; - return { path }; + return { add_extension(node_path, ".out") }; } - std::expected write(const store::NodePath&, const int&) const override + std::expected write(const std::filesystem::path&, const int&) const override { ++writes; return {}; @@ -56,7 +50,7 @@ void check_mesh_codec_reentrancy(const Codec& codec, const std::filesystem::path std::vector> writes; for (int index = 0; index < 4; ++index) { writes.push_back(std::async(std::launch::async, - [&codec, base, index] { return codec.write(store::NodePath(base / std::to_string(index) / "node"), triangle_mesh(index)).has_value(); })); + [&codec, base, index] { return codec.write(base / std::to_string(index) / "node", triangle_mesh(index)).has_value(); })); } for (auto& write : writes) { REQUIRE(write.get()); @@ -65,7 +59,7 @@ void check_mesh_codec_reentrancy(const Codec& codec, const std::filesystem::path std::vector> reads; for (int index = 0; index < 4; ++index) { reads.push_back(std::async(std::launch::async, [&codec, base, index] { - const auto result = codec.read(store::NodePath(base / std::to_string(index) / "node")); + const auto result = codec.read(base / std::to_string(index) / "node"); return result.has_value() && result->face_count() == 1; })); } @@ -79,7 +73,7 @@ void check_mesh_codec_reentrancy(const Codec& codec, const std::filesystem::path TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec]") { MultiFileCodec codec; - const store::NodePath path("12/34/56/78"); + const std::filesystem::path path("12/34/56/78"); CHECK(codec.paths(path) == std::vector { "12/34/56/78.data", @@ -100,9 +94,9 @@ TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") { WriteOnlyCodec codec; - REQUIRE(codec.write(store::NodePath("node"), 42).has_value()); + REQUIRE(codec.write("node", 42).has_value()); CHECK(codec.writes == 1); - const auto read = codec.read(store::NodePath("node")); + const auto read = codec.read("node"); REQUIRE_FALSE(read.has_value()); CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); } @@ -120,7 +114,7 @@ TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") { TemporaryDirectory directory("gltf-error"); mesh::codec::Gltf codec(mesh::codec::GltfContainer::Binary); - const store::NodePath node_path(directory.path() / "blocked"); + const std::filesystem::path node_path(directory.path() / "blocked"); REQUIRE(std::filesystem::create_directory(codec.paths(node_path).front())); const auto result = codec.write(node_path, triangle_mesh(0.0)); diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index d8fd880e..5fe740dc 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -174,7 +174,7 @@ TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][op for (const std::string extension : { ".sfmesh", ".glb", ".gltf" }) { const auto codec = mesh::codec::from_extension(extension); REQUIRE(codec.has_value()); - const auto paths = codec.value()->paths(store::NodePath("node")); + const auto paths = codec.value()->paths("node"); REQUIRE_FALSE(paths.empty()); CHECK(paths.front().extension() == extension); } diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp index a67bc80b..bc4787dc 100644 --- a/unittests/terrainlib/store_layout.cpp +++ b/unittests/terrainlib/store_layout.cpp @@ -15,8 +15,8 @@ TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layo SECTION("flat") { const store::Layout layout(dataset / "sf-flat", octree::store_layout::flat()); - const store::NodePath node_path = layout.node_path(octree::Id::root()); - CHECK(node_path.path() == dataset / "sf-flat/0-0"); + const std::filesystem::path node_path = layout.node_path(octree::Id::root()); + CHECK(node_path == dataset / "sf-flat/0-0"); CHECK(sfmesh.paths(node_path) == std::vector { dataset / "sf-flat/0-0.sfmesh" }); CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); } @@ -46,20 +46,20 @@ TEST_CASE("octree mapping lookup is explicit", "[store][layout]") TEST_CASE("extensionless octree mappings validate complete paths", "[store][layout]") { const auto flat = octree::store_layout::flat(); - CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2.sfmesh")).has_value()); - CHECK_FALSE(flat.node_path_to_key(store::NodePath("nested/1-2")).has_value()); - CHECK_FALSE(flat.node_path_to_key(store::NodePath("1-2-extra")).has_value()); + CHECK_FALSE(flat.node_path_to_key("1-2.sfmesh").has_value()); + CHECK_FALSE(flat.node_path_to_key("nested/1-2").has_value()); + CHECK_FALSE(flat.node_path_to_key("1-2-extra").has_value()); const auto coordinates = octree::store_layout::level_and_coordinate_directories(); - CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0.sfmesh")).has_value()); - CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0")).has_value()); - CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("1/0/0/0/extra")).has_value()); - CHECK_FALSE(coordinates.node_path_to_key(store::NodePath("22/0/0/0")).has_value()); + CHECK_FALSE(coordinates.node_path_to_key("1/0/0/0.sfmesh").has_value()); + CHECK_FALSE(coordinates.node_path_to_key("1/0/0").has_value()); + CHECK_FALSE(coordinates.node_path_to_key("1/0/0/0/extra").has_value()); + CHECK_FALSE(coordinates.node_path_to_key("22/0/0/0").has_value()); } TEST_CASE("runtime mesh codecs own their filename endings", "[store][layout][codec]") { - const store::NodePath path("12/34/56/78"); + const std::filesystem::path path("12/34/56/78"); mesh::codec::SfMesh sfmesh; mesh::codec::Gltf binary(mesh::codec::GltfContainer::Binary); mesh::codec::Gltf json(mesh::codec::GltfContainer::Json); diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index c55a65d2..12daa30e 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -21,18 +21,17 @@ namespace { using test::TemporaryDirectory; -store::NodePath tile_to_path(const radix::tile::Id& key) +std::filesystem::path tile_to_path(const radix::tile::Id& key) { - return store::NodePath(std::to_string(key.zoom_level) + "/" + std::to_string(key.coords.x) + "/" + std::to_string(key.coords.y)); + return std::to_string(key.zoom_level) + "/" + std::to_string(key.coords.x) + "/" + std::to_string(key.coords.y); } -std::optional path_to_tile(const store::NodePath& node_path) +std::optional path_to_tile(const std::filesystem::path& node_path) { - const std::filesystem::path& path = node_path.path(); - if (path.is_absolute() || std::distance(path.begin(), path.end()) != 3) { + if (node_path.is_absolute() || std::distance(node_path.begin(), node_path.end()) != 3) { return std::nullopt; } - auto part = path.begin(); + auto part = node_path.begin(); const auto zoom = from_chars(part->string()); ++part; const auto x = from_chars(part->string()); @@ -90,16 +89,14 @@ std::expected write_int(const int value, const std::fil class IntCodec final : public store::Codec { public: - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { - std::filesystem::path path = node_path.path(); - path += ".data"; - return { path }; + return { add_extension(node_path, ".data") }; } - std::expected read(const store::NodePath& node_path) const override { return read_int(paths(node_path).front()); } + std::expected read(const std::filesystem::path& node_path) const override { return read_int(paths(node_path).front()); } - std::expected write(const store::NodePath& node_path, const int& value) const override + std::expected write(const std::filesystem::path& node_path, const int& value) const override { return write_int(value, paths(node_path).front()); } @@ -114,16 +111,12 @@ store::Storage make_storage(const std::filesystem::path& path) class MultiFileCodec final : public store::Codec { public: - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { - std::filesystem::path first = node_path.path(); - std::filesystem::path second = node_path.path(); - first += ".data"; - second += ".metadata"; - return { first, second }; + return { add_extension(node_path, ".data"), add_extension(node_path, ".metadata") }; } - std::expected read(const store::NodePath& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { const auto value = read_int(paths(node_path).front()); if (!value) { @@ -136,7 +129,7 @@ class MultiFileCodec final : public store::Codec { return value.value(); } - std::expected write(const store::NodePath& node_path, const int& value) const override + std::expected write(const std::filesystem::path& node_path, const int& value) const override { const auto node_paths = paths(node_path); if (!write_int(value, node_paths[0]) || !write_int(value + 1, node_paths[1])) { @@ -157,14 +150,12 @@ class ConfigurableCodec final : public store::Codec { { } - std::vector paths(const store::NodePath& node_path) const override + std::vector paths(const std::filesystem::path& node_path) const override { - std::filesystem::path path = node_path.path(); - path += extension; - return { path }; + return { add_extension(node_path, extension) }; } - std::expected read(const store::NodePath& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { if (fail_read) { return std::unexpected(store::CodecError { @@ -184,7 +175,7 @@ class ConfigurableCodec final : public store::Codec { return value.value(); } - std::expected write(const store::NodePath& node_path, const int& value) const override + std::expected write(const std::filesystem::path& node_path, const int& value) const override { if (fail_write) { return std::unexpected(store::CodecError { From 56d3d7d3adf7c52584600cc38e89838a9a1e6d99 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:28:28 +0200 Subject: [PATCH 27/43] Consolidate store path layout types --- docs/raster-store/refactor-plan.md | 18 +++++++-------- src/terrainlib/CMakeLists.txt | 3 +-- src/terrainlib/octree/storage/IndexFile.h | 2 +- src/terrainlib/octree/storage/open_runtime.h | 4 ++-- src/terrainlib/octree/store_layout/Mappings.h | 10 ++++----- src/terrainlib/store/IndexFormat.h | 6 ++--- src/terrainlib/store/PathMapping.h | 18 --------------- src/terrainlib/store/RawStorage.h | 8 +++---- src/terrainlib/store/Storage.h | 2 +- src/terrainlib/store/open.h | 6 ++--- .../store/{Layout.h => path_layout.h} | 22 +++++++++++++------ unittests/terrainlib/store_compatibility.cpp | 5 +++-- unittests/terrainlib/store_layout.cpp | 6 ++--- unittests/terrainlib/store_storage.cpp | 18 +++++++-------- 14 files changed, 59 insertions(+), 69 deletions(-) delete mode 100644 src/terrainlib/store/PathMapping.h rename src/terrainlib/store/{Layout.h => path_layout.h} (62%) diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md index 63b17e2c..82028b7a 100644 --- a/docs/raster-store/refactor-plan.md +++ b/docs/raster-store/refactor-plan.md @@ -377,14 +377,14 @@ It does not necessarily name a physical file. Replace ```cpp template -struct store::PathMapping { +struct store::path_layout::Mapping { std::string_view id; NodePath (*key_to_node_path)(const Key&); std::optional (*node_path_to_key)(const NodePath&); }; ``` -`store::Layout` owns the base directory and one `PathMapping`. It +`store::path_layout::Resolver` owns the base directory and one `store::path_layout::Mapping`. It does not own a preferred extension. A configured codec expands the extensionless `NodePath` into the physical file or files. @@ -543,10 +543,10 @@ struct store::IndexFormat { const std::filesystem::path& index_path, const IndexMetadata& metadata); - std::optional> + std::optional> (*mapping_from_id)(std::string_view id); - PathMapping + path_layout::Mapping (*default_mapping)(); }; ``` @@ -817,8 +817,8 @@ storage, layout strategies, and static codecs remain temporarily as the working compatibility path until Phase 3 can replace the complete layout-plus-codec path construction in one step. -1. Add `store::NodePath`, `store::PathMapping`, and - `store::Layout`. +1. Add `store::NodePath`, `store::path_layout::Mapping`, and + `store::path_layout::Resolver`. 2. Add the stateful `store::Codec` interface with `paths()`, `read()`, and `write()`, plus `CodecError`. 3. Add the runtime `store::codec::ZppBits`, preserving the existing `.bin` @@ -854,7 +854,7 @@ path. ### Phase 3 — Generalize storage and index lifecycle 1. Cut the production path construction over to the Phase 2 - `store::Layout` and runtime codecs. Move the legacy preferred extension + `store::path_layout::Resolver` and runtime codecs. Move the legacy preferred extension out of layout state and retain it as the 3D format adapter's codec selector. 2. Port legacy layout discovery so it recognizes codec endings through the supplied resolver, strips the accepted ending, and then calls @@ -1062,8 +1062,8 @@ No formatting-only pass or unrelated refactor belongs in these commits. | `octree/IndexMap.*` | `store/Index.h` | | `octree/traverse.h` | `store/traverse.h` | | complete node paths embedded in layouts | extensionless `store/NodePath.h` plus codec endings | -| `octree/disk/Layout.h` | `store/Layout.h` | -| `octree/disk/layout/Strategy.h` | `store/PathMapping.h` | +| `octree/disk/Layout.h` | `store/path_layout.h` | +| `octree/disk/layout/Strategy.h` | `store/path_layout.h` | | `StrategyRegister.h` | explicit dimension-adapter lookup functions | | `strategy/Flat.h` | `octree/store_layout/Flat.h` | | `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 8abbf76b..a3922074 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -180,11 +180,10 @@ add_library(terrainlib store/IndexFormat.h store/IndexedStorage.h store/InvalidKey.h - store/Layout.h store/NodeStatus.h store/NodeStatusOrMissing.h store/OpenError.h - store/PathMapping.h + store/path_layout.h store/RawStorage.h store/Storage.h store/StorageError.h diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 49ab35af..9527e94f 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -210,7 +210,7 @@ inline std::expected write_index_file(const std:: return {}; } -inline store::PathMapping default_mapping() { return store_layout::level_and_coordinate_directories(); } +inline store::path_layout::Mapping default_mapping() { return store_layout::level_and_coordinate_directories(); } inline store::IndexFormat index_format() { diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index 0420d53b..eb7d3e98 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -15,7 +15,7 @@ namespace octree::storage { struct OpenOptions { - std::optional> default_mapping = std::nullopt; + std::optional> default_mapping = std::nullopt; std::optional preferred_extension = std::nullopt; }; @@ -87,7 +87,7 @@ std::expected, store::OpenError> open_ return store::Storage(std::move(*indexed)); } - const store::PathMapping mapping = options.default_mapping.value_or(format.default_mapping()); + const store::path_layout::Mapping mapping = options.default_mapping.value_or(format.default_mapping()); std::string codec_selector = options.preferred_extension.value_or(std::move(default_codec_selector)); auto codec = resolve_codec(codec_selector); if (!codec) { diff --git a/src/terrainlib/octree/store_layout/Mappings.h b/src/terrainlib/octree/store_layout/Mappings.h index 2cdab912..ce2bb437 100644 --- a/src/terrainlib/octree/store_layout/Mappings.h +++ b/src/terrainlib/octree/store_layout/Mappings.h @@ -6,13 +6,13 @@ #include "octree/store_layout/Flat.h" #include "octree/store_layout/LevelAndCoordinateDirectories.h" -#include "store/PathMapping.h" +#include "store/path_layout.h" namespace octree::store_layout { -inline store::PathMapping flat() { return { "flat", flat_key_to_node_path, flat_node_path_to_key }; } +inline store::path_layout::Mapping flat() { return { "flat", flat_key_to_node_path, flat_node_path_to_key }; } -inline store::PathMapping level_and_coordinate_directories() +inline store::path_layout::Mapping level_and_coordinate_directories() { return { "level_and_coordinate_directories", @@ -21,9 +21,9 @@ inline store::PathMapping level_and_coordinate_directories() }; } -inline std::array, 2> all() { return { flat(), level_and_coordinate_directories() }; } +inline std::array, 2> all() { return { flat(), level_and_coordinate_directories() }; } -inline std::optional> from_id(const std::string_view id) +inline std::optional> from_id(const std::string_view id) { for (const auto mapping : all()) { if (mapping.id == id) { diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h index 57e2d893..0d001176 100644 --- a/src/terrainlib/store/IndexFormat.h +++ b/src/terrainlib/store/IndexFormat.h @@ -8,7 +8,7 @@ #include #include "store/Index.h" -#include "store/PathMapping.h" +#include "store/path_layout.h" namespace store { @@ -40,8 +40,8 @@ struct IndexFormat { std::expected, IndexFormatError> (*read)(const std::filesystem::path& index_path); std::expected (*write)(const std::filesystem::path& index_path, const IndexMetadata& metadata); - std::optional> (*mapping_from_id)(std::string_view id); - PathMapping (*default_mapping)(); + std::optional> (*mapping_from_id)(std::string_view id); + path_layout::Mapping (*default_mapping)(); }; } // namespace store diff --git a/src/terrainlib/store/PathMapping.h b/src/terrainlib/store/PathMapping.h deleted file mode 100644 index a1461f18..00000000 --- a/src/terrainlib/store/PathMapping.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace store { - -template -struct PathMapping { - std::string_view id; - std::filesystem::path (*key_to_node_path)(const Key&); - std::optional (*node_path_to_key)(const std::filesystem::path&); - - bool operator==(const PathMapping&) const = default; -}; - -} // namespace store diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index fc7bb38d..988dfde3 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -10,7 +10,7 @@ #include #include "store/Codec.h" -#include "store/Layout.h" +#include "store/path_layout.h" #include "store/StorageError.h" #include "store/Traits.h" @@ -22,7 +22,7 @@ class RawStorage { using Key = typename Traits::Key; using value_type = NodeData; - RawStorage(Layout layout, std::unique_ptr> codec) + RawStorage(path_layout::Resolver layout, std::unique_ptr> codec) : m_layout(std::move(layout)) , m_codec(std::move(codec)) { @@ -194,11 +194,11 @@ class RawStorage { return {}; } - const Layout& layout() const { return m_layout; } + const path_layout::Resolver& layout() const { return m_layout; } const Codec& codec() const { return *m_codec; } private: - Layout m_layout; + path_layout::Resolver m_layout; std::unique_ptr> m_codec; }; diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index 282f74ba..ae09a03c 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -215,7 +215,7 @@ class Storage { } const std::filesystem::path& base_path() const { return m_raw.layout().base_path(); } - const Layout& layout() const { return m_raw.layout(); } + const path_layout::Resolver& layout() const { return m_raw.layout(); } const Codec& codec() const { return m_raw.codec(); } std::optional codec_selector() const { diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h index b26a65ef..07d486cc 100644 --- a/src/terrainlib/store/open.h +++ b/src/terrainlib/store/open.h @@ -46,7 +46,7 @@ std::expected, OpenError> const std::filesystem::path base_path = index_path.parent_path(); using Storage = store::Storage; - return IndexedStorage(RawStorage(Layout(base_path, mapping.value()), std::move(codec.value())), + return IndexedStorage(RawStorage(path_layout::Resolver(base_path, mapping.value()), std::move(codec.value())), std::move(metadata.index), typename Storage::Persistence { format, @@ -60,7 +60,7 @@ std::expected, OpenError> template std::expected, OpenError> make_storage(const std::filesystem::path& base_path, const IndexFormat format, - const PathMapping mapping, + const path_layout::Mapping mapping, std::string payload_class, std::string codec_selector, std::unique_ptr> codec) @@ -83,7 +83,7 @@ std::expected, OpenError> make_s std::move(payload_class), std::move(codec_selector), }; - RawStorage raw(Layout(base_path, mapping), std::move(codec)); + RawStorage raw(path_layout::Resolver(base_path, mapping), std::move(codec)); return Storage(std::move(raw), Index {}, std::move(persistence), true); } diff --git a/src/terrainlib/store/Layout.h b/src/terrainlib/store/path_layout.h similarity index 62% rename from src/terrainlib/store/Layout.h rename to src/terrainlib/store/path_layout.h index dee23991..d08423d7 100644 --- a/src/terrainlib/store/Layout.h +++ b/src/terrainlib/store/path_layout.h @@ -2,16 +2,24 @@ #include #include +#include #include -#include "store/PathMapping.h" +namespace store::path_layout { -namespace store { +template +struct Mapping { + std::string_view id; + std::filesystem::path (*key_to_node_path)(const Key&); + std::optional (*node_path_to_key)(const std::filesystem::path&); + + bool operator==(const Mapping&) const = default; +}; template -class Layout { +class Resolver { public: - Layout(std::filesystem::path base_path, PathMapping mapping) + Resolver(std::filesystem::path base_path, Mapping mapping) : m_base_path(std::move(base_path)) , m_mapping(mapping) { @@ -30,11 +38,11 @@ class Layout { } const std::filesystem::path& base_path() const { return m_base_path; } - PathMapping mapping() const { return m_mapping; } + Mapping mapping() const { return m_mapping; } private: std::filesystem::path m_base_path; - PathMapping m_mapping; + Mapping m_mapping; }; -} // namespace store +} // namespace store::path_layout diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 5fe740dc..c2490b28 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -25,12 +25,13 @@ mesh::Simple triangle_mesh(const double offset) return mesh::Simple({ { 0, 1, 2 } }, { { offset, 0.0, 0.0 }, { offset + 1.0, 0.0, 0.0 }, { offset, 1.0, 0.0 } }); } -mesh::storage::IndexedStorage make_storage(const std::filesystem::path& path, const store::PathMapping mapping, const std::string& extension) +mesh::storage::IndexedStorage make_storage( + const std::filesystem::path& path, const store::path_layout::Mapping mapping, const std::string& extension) { auto codec = mesh::codec::from_extension(extension); REQUIRE(codec.has_value()); return mesh::storage::IndexedStorage( - store::RawStorage(store::Layout(path, mapping), std::move(codec.value())), + store::RawStorage(store::path_layout::Resolver(path, mapping), std::move(codec.value())), store::Index {}, mesh::storage::Storage::Persistence { octree::storage::index_format(), diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp index bc4787dc..9a0cc837 100644 --- a/unittests/terrainlib/store_layout.cpp +++ b/unittests/terrainlib/store_layout.cpp @@ -5,7 +5,7 @@ #include "mesh/codec/Gltf.h" #include "mesh/codec/SfMesh.h" #include "octree/store_layout/Mappings.h" -#include "store/Layout.h" +#include "store/path_layout.h" TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layout]") { @@ -14,7 +14,7 @@ TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layo SECTION("flat") { - const store::Layout layout(dataset / "sf-flat", octree::store_layout::flat()); + const store::path_layout::Resolver layout(dataset / "sf-flat", octree::store_layout::flat()); const std::filesystem::path node_path = layout.node_path(octree::Id::root()); CHECK(node_path == dataset / "sf-flat/0-0"); CHECK(sfmesh.paths(node_path) == std::vector { dataset / "sf-flat/0-0.sfmesh" }); @@ -23,7 +23,7 @@ TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layo SECTION("level and coordinate directories") { - const store::Layout layout(dataset / "sf-coordinates", octree::store_layout::level_and_coordinate_directories()); + const store::path_layout::Resolver layout(dataset / "sf-coordinates", octree::store_layout::level_and_coordinate_directories()); const octree::Id child = octree::Id::root().child(2).value(); const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); CHECK(sfmesh.paths(layout.node_path(child)) == std::vector { dataset / "sf-coordinates/1/0/1/0.sfmesh" }); diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index 12daa30e..15349238 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -45,16 +45,16 @@ std::optional path_to_tile(const std::filesystem::path& node_pa } template -store::PathMapping test_mapping(); +store::path_layout::Mapping test_mapping(); template <> -store::PathMapping test_mapping() +store::path_layout::Mapping test_mapping() { return octree::store_layout::flat(); } template <> -store::PathMapping test_mapping() +store::path_layout::Mapping test_mapping() { return { "test_tiles", tile_to_path, path_to_tile }; } @@ -106,7 +106,7 @@ template store::Storage make_storage(const std::filesystem::path& path) { return store::Storage( - store::RawStorage(store::Layout(path, test_mapping()), std::make_unique())); + store::RawStorage(store::path_layout::Resolver(path, test_mapping()), std::make_unique())); } class MultiFileCodec final : public store::Codec { @@ -216,12 +216,12 @@ std::expected count_index_write(const std::filesy return {}; } -std::optional> fake_mapping_from_id(const std::string_view id) +std::optional> fake_mapping_from_id(const std::string_view id) { - return id == "flat" ? std::optional>(octree::store_layout::flat()) : std::nullopt; + return id == "flat" ? std::optional>(octree::store_layout::flat()) : std::nullopt; } -store::PathMapping fake_default_mapping() { return octree::store_layout::flat(); } +store::path_layout::Mapping fake_default_mapping() { return octree::store_layout::flat(); } store::IndexFormat counting_index_format() { @@ -239,7 +239,7 @@ store::IndexedStorage make_indexed_storage( { using Storage = store::Storage; return store::IndexedStorage( - store::RawStorage(store::Layout(path, octree::store_layout::flat()), std::move(codec)), + store::RawStorage(store::path_layout::Resolver(path, octree::store_layout::flat()), std::move(codec)), store::Index {}, Storage::Persistence { counting_index_format(), @@ -312,7 +312,7 @@ TEST_CASE("raw storage requires and removes every codec path", "[store][storage] { TemporaryDirectory directory("multi-file"); store::RawStorage storage( - store::Layout(directory.path(), octree::store_layout::flat()), std::make_unique()); + store::path_layout::Resolver(directory.path(), octree::store_layout::flat()), std::make_unique()); const octree::Id root = octree::Id::root(); const auto paths = storage.paths(root).value(); REQUIRE(paths.size() == 2); From 5206d41870e0f4083e7656f0266bd76395749946 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:08:51 +0200 Subject: [PATCH 28/43] Unify project error reporting --- src/dag_builder/build.cpp | 23 ++-- src/dag_builder/build.h | 6 +- src/dag_builder/codec/Dag.h | 76 +++-------- src/dag_builder/codec/from_extension.h | 16 +-- src/dag_builder/main.cpp | 10 +- src/dag_builder/storage.h | 8 +- src/dag_convert_debug/main.cpp | 13 +- src/sf_builder/main.cpp | 3 +- src/sf_builder/terrainbuilder.cpp | 9 +- src/sf_builder/terrainbuilder.h | 4 +- src/sf_index_browser/main.cpp | 5 +- src/sf_merger/NodeWriter.h | 12 +- src/sf_merger/cut.h | 30 ++--- src/sf_merger/main.cpp | 14 +- src/sf_merger/merge.h | 23 ++-- src/terrainlib/CMakeLists.txt | 9 +- src/terrainlib/Error.cpp | 125 ++++++++++++++++++ src/terrainlib/Error.h | 79 +++++++++++ src/terrainlib/io/Error.h | 89 ------------- src/terrainlib/io/bytes.cpp | 33 ++--- src/terrainlib/io/bytes.h | 6 +- src/terrainlib/io/compression.cpp | 48 ++++--- src/terrainlib/io/compression.h | 28 +--- src/terrainlib/io/conversion.h | 65 ++++----- src/terrainlib/io/envelope.h | 6 +- src/terrainlib/io/envelope.inl | 56 ++++---- src/terrainlib/io/envelope_file.h | 40 +----- src/terrainlib/mesh/codec/Gltf.h | 29 +--- src/terrainlib/mesh/codec/SfMesh.h | 42 ++---- src/terrainlib/mesh/codec/from_extension.h | 8 +- src/terrainlib/mesh/storage.h | 6 +- src/terrainlib/octree/storage/IndexFile.h | 48 ++----- src/terrainlib/octree/storage/open_runtime.h | 45 ++----- src/terrainlib/sf/Error.h | 39 ------ src/terrainlib/sf/InvalidTopology.h | 13 -- src/terrainlib/sf/finalize_storage.h | 7 +- src/terrainlib/sf/validate_index.h | 7 +- src/terrainlib/store/Codec.h | 10 +- src/terrainlib/store/CodecError.h | 37 ------ src/terrainlib/store/Index.h | 50 +++---- src/terrainlib/store/IndexFormat.h | 19 +-- src/terrainlib/store/InvalidKey.h | 12 -- src/terrainlib/store/OpenError.h | 19 --- src/terrainlib/store/RawStorage.h | 87 +++++------- src/terrainlib/store/Storage.h | 80 ++++++----- src/terrainlib/store/StorageError.h | 43 ------ src/terrainlib/store/Traits.h | 11 ++ src/terrainlib/store/describe_error.h | 39 ------ src/terrainlib/store/open.h | 26 ++-- src/terrainlib/store/traverse.h | 10 +- .../dag_builder/storage_compatibility.cpp | 14 +- unittests/sf_builder/storage_finalization.cpp | 4 +- unittests/sf_merger/storage_boundaries.cpp | 8 +- unittests/terrainlib/envelope.cpp | 60 +++++---- unittests/terrainlib/io_conversion.cpp | 15 +-- unittests/terrainlib/sf_validate_index.cpp | 3 +- unittests/terrainlib/store_codec.cpp | 13 +- unittests/terrainlib/store_compatibility.cpp | 16 +-- unittests/terrainlib/store_index.cpp | 12 +- unittests/terrainlib/store_storage.cpp | 111 ++++++---------- unittests/terrainlib/store_traverse.cpp | 3 +- 61 files changed, 714 insertions(+), 1068 deletions(-) create mode 100644 src/terrainlib/Error.cpp create mode 100644 src/terrainlib/Error.h delete mode 100644 src/terrainlib/io/Error.h delete mode 100644 src/terrainlib/sf/Error.h delete mode 100644 src/terrainlib/sf/InvalidTopology.h delete mode 100644 src/terrainlib/store/CodecError.h delete mode 100644 src/terrainlib/store/InvalidKey.h delete mode 100644 src/terrainlib/store/OpenError.h delete mode 100644 src/terrainlib/store/StorageError.h delete mode 100644 src/terrainlib/store/describe_error.h diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 32fb9eed..f2e9d090 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -34,7 +34,6 @@ #include "range_utils.h" #include "storage.h" #include "store/ThreadSafeStorage.h" -#include "store/describe_error.h" #include "utils.h" #include "vertex_lock.h" #include "parallel.h" @@ -52,13 +51,11 @@ std::optional load_and_clusterize_mesh( const auto result = storage.load(id); if (!result) { - const auto *codec_error = std::get_if(&result.error()); - if (codec_error == nullptr - || codec_error->category != store::CodecErrorCategory::FileNotFound) { + if (result.error().code() != ::Error::Code::NotFound) { LOG_ERROR( "Failed to read node {}: {}", id, - store::describe_error(result.error())); + result.error().to_string()); } return std::nullopt; } @@ -452,7 +449,7 @@ LevelWorkplan build_level_workplan( } // Pre-filter input nodes to only those that intersect the target root bounds. -std::expected>, sf::InvalidTopology> gather_relevant_input_leaves( +std::expected>, ::Error> gather_relevant_input_leaves( const store::Index &index, const octree::Space &space, const radix::geometry::Aabb3d &root_bounds) @@ -472,7 +469,7 @@ std::expected>, sf::InvalidTopology> gather_ }, start); if (!traversal.has_value()) { - return std::unexpected(sf::InvalidTopology{traversal.error().key}); + return std::unexpected(traversal.error()); } return result; } @@ -529,7 +526,7 @@ std::unordered_set build_level( if (!debug_result.has_value()) { LOG_ERROR_AND_EXIT( "Failed to open debug mesh storage: {}", - store::describe_error(debug_result.error())); + debug_result.error().to_string()); } debug_storage.emplace(std::move(debug_result.value())); } @@ -559,7 +556,7 @@ std::unordered_set build_level( LOG_ERROR_AND_EXIT( "Failed to save debug mesh for node {}: {}", target, - store::describe_error(debug_save_result.error())); + debug_save_result.error().to_string()); } } saved_ids.push_back(target); @@ -567,7 +564,7 @@ std::unordered_set build_level( LOG_ERROR( "Failed to save node {}: {}", target, - store::describe_error(save_result.error())); + save_result.error().to_string()); } } progress.task_finished(); @@ -586,7 +583,7 @@ std::unordered_set build_level( // Builds the DAG from input_storage into output_storage, restricted to levels within level_range. // Iterates octree levels from finest to coarsest, simplifying and re-clustering geometry at each // level from its children. -std::expected build_levels( +std::expected build_levels( const mesh::storage::IndexedStorage &input_storage, dag::storage::IndexedStorage &output_storage, const BuildOptions &options, @@ -643,7 +640,7 @@ std::expected build_levels( LOG_WARN( "Could not save index after level {}: {}", level, - store::describe_error(result.error())); + result.error().to_string()); } } @@ -652,7 +649,7 @@ std::expected build_levels( } // Builds the complete DAG from input_storage into output_storage. -std::expected build_full( +std::expected build_full( const mesh::storage::IndexedStorage &input_storage, dag::storage::IndexedStorage &output_storage, const BuildOptions &options) { diff --git a/src/dag_builder/build.h b/src/dag_builder/build.h index a2b774a4..8247ef21 100644 --- a/src/dag_builder/build.h +++ b/src/dag_builder/build.h @@ -10,7 +10,7 @@ #include "texturing.h" #include "mesh/storage.h" #include "storage.h" -#include "sf/InvalidTopology.h" +#include "Error.h" namespace dag { @@ -32,12 +32,12 @@ struct BuildOptions { ContinuationMode continuation_mode = ContinuationMode::Error; }; -std::expected build_full( +std::expected build_full( const mesh::storage::IndexedStorage &input_storage, dag::storage::IndexedStorage &output_storage, const BuildOptions &options); -std::expected build_levels( +std::expected build_levels( const mesh::storage::IndexedStorage &input_storage, dag::storage::IndexedStorage &output_storage, const BuildOptions &options, diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index 38170336..a1d57060 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include "codec/DagFormat.h" @@ -9,25 +11,15 @@ namespace dag::codec { -inline store::CodecError file_error(const store::CodecOperation operation, const io::envelope::FileError& error) +inline ::Error file_error(std::string operation, ::Error error) { - return { - operation, - io::envelope::is_file_not_found(error) ? store::CodecErrorCategory::FileNotFound - : operation == store::CodecOperation::Read ? store::CodecErrorCategory::InvalidData - : store::CodecErrorCategory::Io, - io::envelope::describe_error(error), - }; + return std::move(error).with_context(std::move(operation)); } -inline std::expected validate_batch(const dag::ClusterBatch& batch, const store::CodecOperation operation) +inline std::expected validate_batch(const dag::ClusterBatch& batch, const ::Error::Code code) { if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { - return std::unexpected(store::CodecError { - operation, - operation == store::CodecOperation::Read ? store::CodecErrorCategory::InvalidData : store::CodecErrorCategory::Domain, - "DAG metadata group-assignment count does not match clustering cluster count", - }); + return std::unexpected(::Error::make(code, "DAG metadata group-assignment count does not match clustering cluster count")); } return {}; } @@ -42,79 +34,59 @@ class Dag final : public store::Codec { }; } - std::expected read(const std::filesystem::path& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { const auto node_paths = paths(node_path); auto clustering_payload = io::envelope::read_from_path(node_paths[0]); if (!clustering_payload) { - return std::unexpected(file_error(store::CodecOperation::Read, clustering_payload.error())); + return std::unexpected(file_error("read DAG clustering", std::move(clustering_payload).error())); } auto metadata_payload = io::envelope::read_from_path(node_paths[1]); if (!metadata_payload) { - return std::unexpected(file_error(store::CodecOperation::Read, metadata_payload.error())); + return std::unexpected(file_error("read DAG metadata", std::move(metadata_payload).error())); } auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); if (!clustering) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::InvalidData, - clustering.error(), - }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, clustering.error())); } auto metadata = dag::format::decode_metadata(std::move(*metadata_payload)); if (!metadata) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::InvalidData, - metadata.error(), - }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, metadata.error())); } dag::ClusterBatch batch { .metadata = std::move(*metadata), .clustering = std::move(*clustering), }; - if (auto valid = validate_batch(batch, store::CodecOperation::Read); !valid) { + if (auto valid = validate_batch(batch, ::Error::Code::CorruptData); !valid) { return std::unexpected(valid.error()); } return batch; } - std::expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override + std::expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override { - if (auto valid = validate_batch(batch, store::CodecOperation::Write); !valid) { + if (auto valid = validate_batch(batch, ::Error::Code::InvalidInput); !valid) { return valid; } auto clustering = dag::format::encode_clustering(batch.clustering); if (!clustering) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - clustering.error(), - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, clustering.error())); } const dag::format::NodeMetadata metadata = dag::format::encode_metadata(batch.metadata); if (auto valid = dag::format::validate(*clustering); !valid) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - valid.error(), - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, valid.error())); } if (auto valid = dag::format::validate(metadata); !valid) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - valid.error(), - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, valid.error())); } const auto node_paths = paths(node_path); auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); if (!data_written) { - return std::unexpected(file_error(store::CodecOperation::Write, data_written.error())); + return std::unexpected(file_error("write DAG clustering", std::move(data_written).error())); } auto metadata_written = io::envelope::write_to_path(metadata, node_paths[1]); if (!metadata_written) { - return std::unexpected(file_error(store::CodecOperation::Write, metadata_written.error())); + return std::unexpected(file_error("write DAG metadata", std::move(metadata_written).error())); } return {}; } @@ -127,19 +99,15 @@ class MetadataView final : public store::Codec { return { add_extension(node_path, ".dagmeta") }; } - std::expected read(const std::filesystem::path& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { auto payload = io::envelope::read_from_path(paths(node_path).front()); if (!payload) { - return std::unexpected(file_error(store::CodecOperation::Read, payload.error())); + return std::unexpected(file_error("read DAG metadata", std::move(payload).error())); } auto metadata = dag::format::decode_metadata(std::move(*payload)); if (!metadata) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::InvalidData, - metadata.error(), - }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, metadata.error())); } return std::move(*metadata); } diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h index 0023aa97..d2bf0953 100644 --- a/src/dag_builder/codec/from_extension.h +++ b/src/dag_builder/codec/from_extension.h @@ -11,28 +11,20 @@ namespace dag::codec { -inline std::expected>, store::CodecError> from_extension(const std::string_view extension) +inline std::expected>, ::Error> from_extension(const std::string_view extension) { if (extension == ".dag") { return std::make_unique(); } - return std::unexpected(store::CodecError { - store::CodecOperation::Resolve, - store::CodecErrorCategory::UnsupportedCodec, - "unsupported DAG codec selector: " + std::string(extension), - }); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported DAG codec selector: " + std::string(extension))); } -inline std::expected>, store::CodecError> metadata_from_extension(const std::string_view extension) +inline std::expected>, ::Error> metadata_from_extension(const std::string_view extension) { if (extension == ".dag") { return std::make_unique(); } - return std::unexpected(store::CodecError { - store::CodecOperation::Resolve, - store::CodecErrorCategory::UnsupportedCodec, - "unsupported DAG metadata codec selector: " + std::string(extension), - }); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported DAG metadata codec selector: " + std::string(extension))); } } // namespace dag::codec diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index de077fed..1c892b6b 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -7,8 +7,6 @@ #include "log.h" #include "mesh/storage.h" #include "storage.h" -#include "store/describe_error.h" -#include "sf/Error.h" #include "ContinuationMode.h" int main(int argc, char **argv) { @@ -22,7 +20,7 @@ int main(int argc, char **argv) { LOG_ERROR( "Failed to open input dataset {}: {}", args.input_path, - store::describe_error(input_result.error())); + input_result.error().to_string()); return EXIT_FAILURE; } auto output_result = dag::storage::open_folder_indexed(args.output_path); @@ -30,7 +28,7 @@ int main(int argc, char **argv) { LOG_ERROR( "Failed to open output dataset {}: {}", args.output_path, - store::describe_error(output_result.error())); + output_result.error().to_string()); return EXIT_FAILURE; } const mesh::storage::IndexedStorage input_storage = std::move(input_result.value()); @@ -65,7 +63,7 @@ int main(int argc, char **argv) { options, args.level_range); if (!build_result.has_value()) { - LOG_ERROR("Invalid Structura Fundamentalis input: {}", sf::describe_error(build_result.error())); + LOG_ERROR("Invalid Structura Fundamentalis input: {}", build_result.error().to_string()); return EXIT_FAILURE; } const auto index_result = output_storage.save_index(); @@ -73,7 +71,7 @@ int main(int argc, char **argv) { LOG_ERROR( "Failed to save output index in {}: {}", args.output_path, - store::describe_error(index_result.error())); + index_result.error().to_string()); return EXIT_FAILURE; } diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 77250463..849ff2b1 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -20,7 +20,7 @@ using MetadataStorage = store::Storage; using IndexedMetadataStorage = store::IndexedStorage; -inline std::expected> open_index( +inline std::expected open_index( const std::filesystem::path &path) { return store::open_index( path, @@ -29,7 +29,7 @@ inline std::expected> open_index( dag::codec::from_extension); } -inline std::expected> open_folder( +inline std::expected open_folder( const std::filesystem::path &path, OpenOptions options = {}) { return octree::storage::open_folder( @@ -40,7 +40,7 @@ inline std::expected> open_folder( std::move(options)); } -inline std::expected> +inline std::expected open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { return octree::storage::open_folder_indexed( path, @@ -50,7 +50,7 @@ open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) std::move(options)); } -inline std::expected> +inline std::expected open_metadata_indexed(const std::filesystem::path &path) { const std::filesystem::path index_path = path.filename() == octree::storage::index_file_name diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index aa1e4604..d6f1dad4 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -11,7 +11,6 @@ #include "ProgressIndicator.h" #include "storage.h" #include "utils.h" -#include "store/describe_error.h" namespace { @@ -25,7 +24,7 @@ void export_node(const cli::Args &args) { const dag::codec::Dag codec; const auto load_result = codec.read(node_path); if (!load_result.has_value()) { - LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error().message); + LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error().to_string()); return; } @@ -43,7 +42,7 @@ void export_storage(const cli::Args &args) { LOG_ERROR( "Failed to open input storage {}: {}", args.input_path, - store::describe_error(input_result.error())); + input_result.error().to_string()); return; } const dag::storage::IndexedStorage input_storage = std::move(input_result.value()); @@ -57,7 +56,7 @@ void export_storage(const cli::Args &args) { LOG_ERROR( "Failed to open output storage {}: {}", args.output_path, - store::describe_error(output_result.error())); + output_result.error().to_string()); return; } mesh::storage::Storage output_storage = std::move(output_result.value()); @@ -79,7 +78,7 @@ void export_storage(const cli::Args &args) { LOG_ERROR( "Failed to load node {}: {}", id, - store::describe_error(load_result.error())); + load_result.error().to_string()); progress.task_finished(); continue; } @@ -91,7 +90,7 @@ void export_storage(const cli::Args &args) { LOG_ERROR( "Failed to save mesh for node {}: {}", id, - store::describe_error(save_result.error())); + save_result.error().to_string()); progress.task_finished(); continue; } @@ -107,7 +106,7 @@ void export_storage(const cli::Args &args) { LOG_ERROR( "Failed to save index for {}: {}", args.output_path, - store::describe_error(index_result.error())); + index_result.error().to_string()); } LOG_INFO("Exported {} debug meshes to {}", exported_count, args.output_path); diff --git a/src/sf_builder/main.cpp b/src/sf_builder/main.cpp index e95c3464..c592fdf1 100644 --- a/src/sf_builder/main.cpp +++ b/src/sf_builder/main.cpp @@ -18,7 +18,6 @@ #include "terrainbuilder.h" #include "octree/Space.h" #include "tile_provider.h" -#include "sf/Error.h" #include "ctb/GlobalGeodetic.hpp" #include "ctb/GlobalMercator.hpp" @@ -330,7 +329,7 @@ int run(std::span args) { if (!build_result.has_value()) { LOG_ERROR( "Failed to finalize Structura Fundamentalis output: {}", - sf::describe_error(build_result.error())); + build_result.error().to_string()); return 1; } } diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index e4e75b95..b052229e 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -43,7 +43,6 @@ #include "octree/Space.h" #include "octree/utils.h" #include "store/ThreadSafeStorage.h" -#include "store/describe_error.h" #include "sf/finalize_storage.h" namespace terrainbuilder { @@ -173,7 +172,7 @@ T expect(const std::optional &opt, const std::string &msg) { } } -std::expected build_all_patches( +std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, @@ -199,7 +198,7 @@ std::expected build_all_patches( LOG_ERROR_AND_EXIT( "Failed to open output dataset {}: {}", output_base_path, - store::describe_error(storage_result.error())); + storage_result.error().to_string()); } mesh::storage::Storage raw_storage = std::move(storage_result.value()); raw_storage.settings().allow_overwrite = overwrite_existing; @@ -293,7 +292,7 @@ std::expected build_all_patches( LOG_ERROR( "Failed to inspect node {}: {}", node, - store::describe_error(already_exists.error())); + already_exists.error().to_string()); progress.task_finished(); context.cancel_group_execution(); return; @@ -325,7 +324,7 @@ std::expected build_all_patches( LOG_ERROR( "Failed to save mesh for node {}: {}", node, - store::describe_error(save_result.error())); + save_result.error().to_string()); progress.task_finished(); context.cancel_group_execution(); return; diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index a59b06e9..1ad13fec 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -10,7 +10,7 @@ #include "mesh/SimpleMesh.h" #include "tile_provider.h" #include "mesh/storage.h" -#include "sf/Error.h" +#include "Error.h" namespace terrainbuilder { @@ -31,7 +31,7 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -std::expected build_all_patches( +std::expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index faa97119..ed6fd9a5 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -16,7 +16,6 @@ #include "mesh/storage.h" #include "mesh/storage.h" #include "cli.h" -#include "store/describe_error.h" #include "store/NodeStatusOrMissing.h" namespace { @@ -87,7 +86,7 @@ class TreeView { const auto &mesh = result.value(); entry = MeshNode{parent.id, mesh.vertex_count(), mesh.face_count()}; } else { - entry = ErrorNode{parent.id, store::describe_error(result.error())}; + entry = ErrorNode{parent.id, result.error().to_string()}; } it++; it = this->_view.emplace(it, entry); @@ -197,7 +196,7 @@ mesh::storage::IndexedStorage open_path_indexed(const std::filesystem::path& pat LOG_ERROR_AND_EXIT( "Failed to open dataset {}: {}", path, - store::describe_error(result.error())); + result.error().to_string()); } return std::move(result.value()); } diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 6a29a556..4f06f145 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -16,12 +16,12 @@ class NodeWriter { public: NodeWriter(mesh::storage::Storage &storage) : _storage(storage) {} - std::expected> has_node( + std::expected has_node( const octree::Id &id) { return this->_storage.has(id); } - std::expected> write_node( + std::expected write_node( const octree::Id &id, const SimpleMesh &mesh) { mesh::validate(mesh); @@ -31,7 +31,7 @@ class NodeWriter { } const auto path_result = this->_storage.path_for(id); if (!path_result.has_value()) { - return std::unexpected(store::SaveError(path_result.error())); + return std::unexpected(path_result.error()); } auto p = path_result.value(); // change extension to .png @@ -42,10 +42,10 @@ class NodeWriter { return {}; } - std::expected> copy_subtree_to_output( + std::expected copy_subtree_to_output( const octree::Id &id, const NodeLoader &loader) { - std::optional> error; + std::optional<::Error> error; const auto traversal = store::traverse( loader.storage().index(), [&](const octree::Id &child_id, const store::NodeStatus &status) { @@ -65,7 +65,7 @@ class NodeWriter { [&](const octree::Id &) { return !error.has_value(); }, id); if (!traversal.has_value()) { - return std::unexpected(store::CopyError(traversal.error())); + return std::unexpected(traversal.error()); } if (error.has_value()) { return std::unexpected(std::move(error.value())); diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 5613ce8d..d74537e0 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -12,8 +12,7 @@ #include "octree/Space.h" #include "utils.h" #include "containers/Cow.h" -#include "store/describe_error.h" -#include "sf/Error.h" +#include "Error.h" #include "sf/finalize_storage.h" #include "sf/validate_index.h" @@ -24,12 +23,12 @@ struct Context { const bool keep_inside; }; -std::expected cut_node( +std::expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask); -inline std::expected cut_leaf_node( +inline std::expected cut_leaf_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -45,7 +44,7 @@ inline std::expected cut_leaf_node( LOG_TRACE("Mesh was fully inside the mask"); const auto copy_result = ctx.output.copy_from(id, ctx.input); if (!copy_result.has_value()) { - return std::unexpected(sf::ProcessingError(copy_result.error())); + return std::unexpected(copy_result.error()); } } else { const SimpleMesh &clipped_mesh = clipped; @@ -54,7 +53,7 @@ inline std::expected cut_leaf_node( mesh.vertex_count(), mesh.face_count(), clipped_mesh.vertex_count(), clipped_mesh.face_count()); const auto save_result = ctx.output.save(id, clipped_mesh); if (!save_result.has_value()) { - return std::unexpected(sf::ProcessingError(save_result.error())); + return std::unexpected(save_result.error()); } } else { LOG_TRACE("Mesh was fully outside the mask"); @@ -63,7 +62,7 @@ inline std::expected cut_leaf_node( return {}; } -inline std::expected cut_virtual_node( +inline std::expected cut_virtual_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -86,7 +85,7 @@ inline std::expected cut_virtual_node( return {}; } -inline std::expected cut_node( +inline std::expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -123,14 +122,14 @@ inline std::expected cut_node( return {}; } -inline std::expected cut_dataset( +inline std::expected cut_dataset( const mesh::storage::IndexedStorage &input, const MeshMask& mask, mesh::storage::Storage &output, const bool keep_inside) { const auto validation = sf::validate_index(input.index()); if (!validation.has_value()) { - return std::unexpected(sf::ProcessingError(validation.error())); + return std::unexpected(validation.error()); } Context ctx(input, output, octree::Space::earth(), keep_inside); const auto cut_result = cut_node(ctx, octree::Id::root(), mask); @@ -139,14 +138,12 @@ inline std::expected cut_dataset( } const auto finalization = sf::finalize_storage(output); if (!finalization.has_value()) { - return std::unexpected(std::visit( - [](const auto &error) -> sf::ProcessingError { return error; }, - finalization.error())); + return std::unexpected(finalization.error()); } return {}; } -inline std::expected cut_dataset( +inline std::expected cut_dataset( const mesh::storage::IndexedStorage &input_dataset, const MeshMask& mask, const std::filesystem::path &output_path, @@ -159,12 +156,11 @@ inline std::expected cut_dataset( options.preferred_extension = ".glb"; auto output_result = mesh::storage::open_folder_indexed(output_path, std::move(options)); if (!output_result.has_value()) { - return std::unexpected(sf::ProcessingError(output_result.error())); + return std::unexpected(output_result.error()); } mesh::storage::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { - return std::unexpected(sf::ProcessingError(store::SaveError( - store::AlreadyExists{output_path}))); + return std::unexpected(::Error::make(::Error::Code::AlreadyExists, "output dataset already contains nodes: " + output_path.string())); } return cut_dataset(input_dataset, mask, output_dataset, keep_inside); diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 855752c7..217d39dd 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -11,8 +11,6 @@ #include "merge.h" #include "optional_utils.h" #include "earth.h" -#include "store/describe_error.h" -#include "sf/Error.h" std::optional load_mask_from_path(const std::filesystem::path& path) { if (std::filesystem::exists(path)) { @@ -41,7 +39,7 @@ void run(const cli::MergeArgs& args) { LOG_ERROR_AND_EXIT( "Failed to open base dataset {}: {}", args.base_path, - store::describe_error(base_result.error())); + base_result.error().to_string()); } mesh::storage::IndexedStorage base_dataset = std::move(base_result.value()); @@ -51,7 +49,7 @@ void run(const cli::MergeArgs& args) { LOG_ERROR_AND_EXIT( "Failed to open new dataset {}: {}", args.new_path, - store::describe_error(new_result.error())); + new_result.error().to_string()); } mesh::storage::IndexedStorage new_dataset = std::move(new_result.value()); @@ -65,7 +63,7 @@ void run(const cli::MergeArgs& args) { LOG_ERROR_AND_EXIT( "Failed to open output dataset {}: {}", args.output_path, - store::describe_error(output_result.error())); + output_result.error().to_string()); } mesh::storage::Storage output_dataset = std::move(output_result.value()); @@ -77,7 +75,7 @@ void run(const cli::MergeArgs& args) { output_dataset, mask); if (!merge_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to merge datasets: {}", sf::describe_error(merge_result.error())); + LOG_ERROR_AND_EXIT("Failed to merge datasets: {}", merge_result.error().to_string()); } } @@ -88,7 +86,7 @@ void run(const cli::CutArgs& args) { LOG_ERROR_AND_EXIT( "Failed to open input dataset {}: {}", args.input_path, - store::describe_error(input_result.error())); + input_result.error().to_string()); } const mesh::storage::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); @@ -98,7 +96,7 @@ void run(const cli::CutArgs& args) { args.output_path, args.keep_inside); if (!cut_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to cut dataset: {}", sf::describe_error(cut_result.error())); + LOG_ERROR_AND_EXIT("Failed to cut dataset: {}", cut_result.error().to_string()); } } diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index 2da551d2..47b97f2f 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -16,8 +16,7 @@ #include "mesh/storage.h" #include "store/NodeStatusOrMissing.h" #include "store/cache/Dummy.h" -#include "store/describe_error.h" -#include "sf/Error.h" +#include "Error.h" #include "sf/finalize_storage.h" #include "sf/validate_index.h" @@ -38,7 +37,7 @@ class Merger { using Status = store::NodeStatusOrMissing; using Context = Visitor::Context; using Result = merge::Result; - using Expected = std::expected; + using Expected = std::expected; Merger( Visitor& visitor, @@ -68,7 +67,7 @@ class Merger { LOG_DEBUG("[{}] Start merging (left = {}, right = {})", id, left_status, right_status); const auto has_result = this->_output.has_node(id); if (!has_result.has_value()) { - return std::unexpected(sf::ProcessingError(has_result.error())); + return std::unexpected(has_result.error()); } if (has_result.value()) { LOG_DEBUG("[{}] Already merged, skipping...", id); @@ -97,7 +96,7 @@ class Merger { if (mesh_opt.has_value()) { const auto write_result = this->_output.write_node(id, *mesh_opt); if (!write_result.has_value()) { - return std::unexpected(sf::ProcessingError(write_result.error())); + return std::unexpected(write_result.error()); } } } else if (result.source == merge::Source::Right && right_status == Status::Missing) { @@ -105,7 +104,7 @@ class Merger { if (mesh_opt.has_value()) { const auto write_result = this->_output.write_node(id, *mesh_opt); if (!write_result.has_value()) { - return std::unexpected(sf::ProcessingError(write_result.error())); + return std::unexpected(write_result.error()); } } } else { @@ -113,7 +112,7 @@ class Merger { id, result.source == merge::Source::Left ? this->_left : this->_right); if (!copy_result.has_value()) { - return std::unexpected(sf::ProcessingError(copy_result.error())); + return std::unexpected(copy_result.error()); } } return {}; @@ -121,7 +120,7 @@ class Merger { LOG_DEBUG("[{}] was merged", id); const auto write_result = this->_output.write_node(id, result.mesh); if (!write_result.has_value()) { - return std::unexpected(sf::ProcessingError(write_result.error())); + return std::unexpected(write_result.error()); } return {}; } else if constexpr (std::is_same_v) { @@ -182,7 +181,7 @@ class Merger { NodeWriter _output; }; -inline std::expected merge_datasets( +inline std::expected merge_datasets( const mesh::storage::IndexedStorage &left_dataset, const mesh::storage::IndexedStorage &right_dataset, mesh::storage::Storage &output_dataset, @@ -190,7 +189,7 @@ inline std::expected merge_datasets( for (const mesh::storage::IndexedStorage *input : {&left_dataset, &right_dataset}) { const auto validation = sf::validate_index(input->index()); if (!validation.has_value()) { - return std::unexpected(sf::ProcessingError(validation.error())); + return std::unexpected(validation.error()); } } @@ -223,9 +222,7 @@ inline std::expected merge_datasets( const auto finalization = sf::finalize_storage(output_dataset); if (!finalization.has_value()) { - return std::unexpected(std::visit( - [](const auto &error) -> sf::ProcessingError { return error; }, - finalization.error())); + return std::unexpected(finalization.error()); } return {}; } diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index a3922074..882fb477 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(terrainlib + Error.h Error.cpp Dataset.h Dataset.cpp ImageKey.h ProgressIndicator.h ProgressIndicator.cpp @@ -53,7 +54,6 @@ add_library(terrainlib geometry/Rect.h geometry/geometry.h - io/Error.h io/bytes.h io/bytes.cpp io/compression.h io/compression.cpp io/conversion.h @@ -159,8 +159,6 @@ add_library(terrainlib raster_store/StoreTraits.h - sf/Error.h - sf/InvalidTopology.h sf/finalize_storage.h sf/validate_index.h @@ -175,24 +173,19 @@ add_library(terrainlib spatial_lookup/SpatialLookup.h store/Codec.h - store/CodecError.h store/Index.h store/IndexFormat.h store/IndexedStorage.h - store/InvalidKey.h store/NodeStatus.h store/NodeStatusOrMissing.h - store/OpenError.h store/path_layout.h store/RawStorage.h store/Storage.h - store/StorageError.h store/ThreadSafeStorage.h store/Traits.h store/cache/Dummy.h store/cache/Interface.h store/cache/Lru.h - store/describe_error.h store/open.h store/traverse.h diff --git a/src/terrainlib/Error.cpp b/src/terrainlib/Error.cpp new file mode 100644 index 00000000..ad4892bf --- /dev/null +++ b/src/terrainlib/Error.cpp @@ -0,0 +1,125 @@ +#include "Error.h" + +#include + +#include + +namespace { + +std::string_view code_name(const Error::Code code) +{ + switch (code) { + case Error::Code::InvalidInput: + return "InvalidInput"; + case Error::Code::NotFound: + return "NotFound"; + case Error::Code::AlreadyExists: + return "AlreadyExists"; + case Error::Code::Unsupported: + return "Unsupported"; + case Error::Code::CorruptData: + return "CorruptData"; + case Error::Code::Io: + return "Io"; + case Error::Code::ResourceExhausted: + return "ResourceExhausted"; + case Error::Code::Internal: + return "Internal"; + } + return "Unknown"; +} + +std::string describe_system_error(const std::error_code& cause) +{ + return fmt::format("{} ({}:{})", cause.message(), cause.category().name(), cause.value()); +} + +} // namespace + +Error::Error(const Code code, Frame frame) + : m_code(code) +{ + m_frames.push_back(std::move(frame)); +} + +Error Error::make(const Code code, std::string message, const std::source_location location) +{ + return Error(code, Frame { std::move(message), location, std::nullopt, std::nullopt }); +} + +Error Error::make(const Code code, + const std::string_view operation, + const std::filesystem::path& path, + const std::source_location location) +{ + return make(code, fmt::format("{} \"{}\"", operation, path.string()), location); +} + +Error Error::make(const Code code, + const std::string_view operation, + const std::error_code& cause, + const std::source_location location) +{ + return make(code, fmt::format("{}: {}", operation, describe_system_error(cause)), location); +} + +Error Error::make(const Code code, + const std::string_view operation, + const std::filesystem::path& path, + const std::error_code& cause, + const std::source_location location) +{ + return make(code, fmt::format("{} \"{}\": {}", operation, path.string(), describe_system_error(cause)), location); +} + +Error Error::make(const Code code, + const std::string_view operation, + const std::filesystem::path& source, + const std::filesystem::path& destination, + const std::error_code& cause, + const std::source_location location) +{ + return make(code, + fmt::format("{} \"{}\" -> \"{}\": {}", operation, source.string(), destination.string(), describe_system_error(cause)), + location); +} + +Error Error::with_context(std::string message, const std::source_location location) && +{ + m_frames.push_back(Frame { + std::move(message), + location, + std::nullopt, + std::nullopt, + }); + return std::move(*this); +} + +Error Error::reclassified(const Code code, std::string message, const std::source_location location) && +{ + m_frames.push_back(Frame { + std::move(message), + location, + m_code, + code, + }); + m_code = code; + return std::move(*this); +} + +std::string Error::to_string() const +{ + std::string result = fmt::format("[{}]", code_name(m_code)); + for (auto frame = m_frames.rbegin(); frame != m_frames.rend(); ++frame) { + result += fmt::format("\n{}{}\n at {}:{} ({})", + frame == m_frames.rbegin() ? "" : "caused by: ", + frame->message, + frame->location.file_name(), + frame->location.line(), + frame->location.function_name()); + if (frame->original_code.has_value() && frame->new_code.has_value()) { + result += fmt::format("\n reclassified {} -> {}", code_name(frame->original_code.value()), code_name(frame->new_code.value())); + } + } + return result; +} diff --git a/src/terrainlib/Error.h b/src/terrainlib/Error.h new file mode 100644 index 00000000..e9ad15da --- /dev/null +++ b/src/terrainlib/Error.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +class Error { +public: + enum class Code { + /// A caller supplied an invalid value, such as a malformed hierarchy key or incompatible dimensions. + InvalidInput, + /// A requested file, node, or key does not exist. + NotFound, + /// Creating or publishing an object failed because the destination already exists. + AlreadyExists, + /// A requested codec, layout, format version, algorithm, or data type is not supported. + Unsupported, + /// Persisted or external data violates its format or invariants, such as a checksum mismatch or invalid topology. + CorruptData, + /// A filesystem or device operation failed, excluding failures classified more specifically above. + Io, + /// An explicit size, storage, or other recoverable resource limit was exceeded. + ResourceExhausted, + /// An internal invariant or an otherwise valid operation failed unexpectedly. + Internal, + }; + + struct Frame { + std::string message; + std::source_location location; + std::optional original_code; + std::optional new_code; + }; + + static Error make(Code code, + std::string message, + std::source_location location = std::source_location::current()); + static Error make(Code code, + std::string_view operation, + const std::filesystem::path& path, + std::source_location location = std::source_location::current()); + static Error make(Code code, + std::string_view operation, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + static Error make(Code code, + std::string_view operation, + const std::filesystem::path& path, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + static Error make(Code code, + std::string_view operation, + const std::filesystem::path& source, + const std::filesystem::path& destination, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + + [[nodiscard]] Error with_context( + std::string message, + std::source_location location = std::source_location::current()) &&; + [[nodiscard]] Error reclassified(Code code, + std::string message, + std::source_location location = std::source_location::current()) &&; + + Code code() const { return m_code; } + std::span frames() const { return m_frames; } + std::string to_string() const; + +private: + Error(Code code, Frame frame); + + Code m_code; + std::vector m_frames; +}; diff --git a/src/terrainlib/io/Error.h b/src/terrainlib/io/Error.h deleted file mode 100644 index 40d1f1c4..00000000 --- a/src/terrainlib/io/Error.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once - -#include - -#include "log.h" - -namespace io { - -class Error { -public: - enum Value : uint8_t { - OpenFile, - WriteBytes, - DetermineSize, - ReadBytes, - OutOfMemory, - CreateDirectories, - }; - - Error() = default; - constexpr Error(Value value) : _value(value) {} - - constexpr operator Value() const { - return this->_value; - } - explicit operator bool() const = delete; - constexpr bool operator==(Error other) const { - return this->_value == other._value; - } - constexpr bool operator!=(Error other) const { - return !(*this == other); - } - - std::string to_string() const; - -private: - Value _value; -}; - -} // namespace io - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const io::Error &error, FormatContext &ctx) const { - const char *name = "Unknown"; - - switch (error) { - case io::Error::Value::OpenFile: - name = "OpenFile"; - break; - case io::Error::Value::WriteBytes: - name = "WriteBytes"; - break; - case io::Error::Value::DetermineSize: - name = "DetermineSize"; - break; - case io::Error::Value::ReadBytes: - name = "ReadBytes"; - break; - case io::Error::Value::OutOfMemory: - name = "OutOfMemory"; - break; - case io::Error::Value::CreateDirectories: - name = "CreateDirectories"; - break; - default: - UNREACHABLE(); - } - - return fmt::format_to(ctx.out(), "{}", name); - } -}; - -#include -#include -inline std::string io::Error::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const io::Error &status) { - fmt::print(os, "{}", status); - return os; -} diff --git a/src/terrainlib/io/bytes.cpp b/src/terrainlib/io/bytes.cpp index 95aa8d86..f3fedf1c 100644 --- a/src/terrainlib/io/bytes.cpp +++ b/src/terrainlib/io/bytes.cpp @@ -7,45 +7,49 @@ namespace io { -std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs) { +std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path& path, bool make_dirs) +{ LOG_TRACE("Writing bytes to path {}", path); if (make_dirs) { const std::error_code error = utils::create_parent_directories(path); if (error) { - LOG_ERROR("Failed to create parent directories for {}: {}", path, error.message()); - return std::unexpected(Error::CreateDirectories); + return std::unexpected(::Error::make(::Error::Code::Io, "create parent directories for", path, error)); } } - + std::ofstream file(path, std::ios::binary); if (!file.is_open()) { - LOG_DEBUG("Failed to open file for writing {}", path); - return std::unexpected(Error::OpenFile); + return std::unexpected(::Error::make(::Error::Code::Io, "open file for writing", path)); } file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); if (!file.good()) { - LOG_ERROR("Failed to write bytes to file {}", path); - return std::unexpected(Error::WriteBytes); + return std::unexpected(::Error::make(::Error::Code::Io, "write bytes to", path)); } return {}; } -std::expected, Error> read_bytes_from_path(const std::filesystem::path& path) { +std::expected, ::Error> read_bytes_from_path(const std::filesystem::path& path) +{ LOG_TRACE("Reading bytes from path {}", path); std::ifstream file(path, std::ios::binary | std::ios::ate); if (!file.is_open()) { - LOG_DEBUG("Failed to open file for reading {}", path); - return std::unexpected(Error::OpenFile); + std::error_code existence_error; + const bool exists = std::filesystem::exists(path, existence_error); + if (!existence_error && !exists) { + return std::unexpected(::Error::make(::Error::Code::NotFound, "open file for reading", path)); + } + return std::unexpected(existence_error + ? ::Error::make(::Error::Code::Io, "check existence of", path, existence_error) + : ::Error::make(::Error::Code::Io, "open file for reading", path)); } const std::streamsize size = file.tellg(); if (size < 0) { - LOG_ERROR("Failed to determine size for file {}", path); - return std::unexpected(Error::DetermineSize); + return std::unexpected(::Error::make(::Error::Code::Io, "determine size of", path)); } std::vector buffer(static_cast(size)); @@ -53,8 +57,7 @@ std::expected, Error> read_bytes_from_path(const std::files file.read(reinterpret_cast(buffer.data()), size); if (!file.good()) { - LOG_ERROR("Failed to read bytes from file {}", path); - return std::unexpected(Error::ReadBytes); + return std::unexpected(::Error::make(::Error::Code::Io, "read bytes from", path)); } return buffer; diff --git a/src/terrainlib/io/bytes.h b/src/terrainlib/io/bytes.h index be3852dc..ebb6f2c9 100644 --- a/src/terrainlib/io/bytes.h +++ b/src/terrainlib/io/bytes.h @@ -6,11 +6,11 @@ #include -#include "io/Error.h" +#include "Error.h" namespace io { -std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs = true); -std::expected, Error> read_bytes_from_path(const std::filesystem::path &path); +std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path& path, bool make_dirs = true); +std::expected, ::Error> read_bytes_from_path(const std::filesystem::path& path); } diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index 35fbb870..460ff976 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -47,7 +47,7 @@ namespace { return checksum; } - std::expected validate_algorithms(const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) + std::expected validate_algorithms(const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) { switch (checksum_algorithm) { case ChecksumAlgorithm::None: @@ -55,7 +55,8 @@ namespace { case ChecksumAlgorithm::Crc32c: break; default: - return std::unexpected(Error { ErrorCode::UnsupportedChecksumAlgorithm }); + return std::unexpected(::Error::make(::Error::Code::Unsupported, + "unsupported checksum algorithm " + std::to_string(static_cast(checksum_algorithm)))); } switch (compression_algorithm) { @@ -64,7 +65,8 @@ namespace { case CompressionAlgorithm::ZstdDefaultCompressionWithChecksum: break; default: - return std::unexpected(Error { ErrorCode::UnsupportedCompressionAlgorithm }); + return std::unexpected(::Error::make(::Error::Code::Unsupported, + "unsupported compression algorithm " + std::to_string(static_cast(compression_algorithm)))); } const bool no_compression = compression_algorithm == CompressionAlgorithm::None @@ -73,7 +75,7 @@ namespace { || compression_algorithm == CompressionAlgorithm::ZstdDefaultCompressionWithChecksum) && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib || checksum_algorithm == ChecksumAlgorithm::Crc32c); if (!no_compression && !zstd_with_checksum) { - return std::unexpected(Error { ErrorCode::InvalidAlgorithmCombination }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "invalid compression and checksum algorithm combination")); } return {}; @@ -81,14 +83,14 @@ namespace { } // namespace -std::expected compress_with_checksum( +std::expected compress_with_checksum( const Bytes& uncompressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) { if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { return std::unexpected(validation.error()); } if (uncompressed_data.size() > default_max_decompressed_size) { - return std::unexpected(Error { ErrorCode::SizeLimitExceeded }); + return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "uncompressed data exceeds the compression size limit")); } const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c ? crc32c_checksum(uncompressed_data) : std::string {}; @@ -99,13 +101,13 @@ std::expected compress_with_checksum( CompressionContext context { ZSTD_createCCtx(), &ZSTD_freeCCtx }; if (!context) { - return std::unexpected(Error { ErrorCode::CompressionFailed }); + return std::unexpected(::Error::make(::Error::Code::Internal, "failed to create the Zstandard compression context")); } const int compression_level = compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum ? ZSTD_maxCLevel() : 0; if (ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_compressionLevel, compression_level)) || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { - return std::unexpected(Error { ErrorCode::CompressionFailed }); + return std::unexpected(::Error::make(::Error::Code::Internal, "failed to configure the Zstandard compression context")); } const std::size_t capacity = ZSTD_compressBound(uncompressed_data.size()); @@ -113,14 +115,15 @@ std::expected compress_with_checksum( const std::size_t compressed_size = ZSTD_compress2(context.get(), compressed_data.data(), compressed_data.size(), uncompressed_data.data(), uncompressed_data.size()); if (ZSTD_isError(compressed_size)) { - return std::unexpected(Error { ErrorCode::CompressionFailed }); + return std::unexpected(::Error::make(::Error::Code::Internal, + std::string("Zstandard compression failed: ") + ZSTD_getErrorName(compressed_size))); } compressed_data.resize(compressed_size); return CompressedData { std::move(compressed_data), checksum }; } -std::expected checked_decompress(const Bytes& compressed_data, +std::expected checked_decompress(const Bytes& compressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm, const std::string_view checksum, @@ -130,12 +133,12 @@ std::expected checked_decompress(const Bytes& compressed_data, return std::unexpected(validation.error()); } if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { - return std::unexpected(Error { ErrorCode::InvalidAlgorithmCombination }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "a checksum was supplied for an algorithm that does not accept one")); } if (checksum_algorithm == ChecksumAlgorithm::Crc32c && (checksum.size() != 8 || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { return (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f'); }))) { - return std::unexpected(Error { ErrorCode::ChecksumMismatch }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid CRC32C checksum encoding")); } const std::size_t effective_max_decompressed_size = std::min(max_decompressed_size, default_max_decompressed_size); @@ -143,28 +146,28 @@ std::expected checked_decompress(const Bytes& compressed_data, Bytes uncompressed_data; if (compression_algorithm == CompressionAlgorithm::None) { if (compressed_data.size() > effective_max_decompressed_size) { - return std::unexpected(Error { ErrorCode::SizeLimitExceeded }); + return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit")); } uncompressed_data = compressed_data; } else { ZSTD_frameHeader frame_header {}; const std::size_t frame_header_result = ZSTD_getFrameHeader(&frame_header, compressed_data.data(), compressed_data.size()); if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { - return std::unexpected(Error { ErrorCode::DecompressionFailed }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid or incomplete Zstandard frame header")); } if (frame_header.checksumFlag == 0) { - return std::unexpected(Error { ErrorCode::ChecksumMismatch }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "Zstandard frame does not contain the required checksum")); } const unsigned long long frame_content_size = ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); if (frame_content_size == ZSTD_CONTENTSIZE_ERROR || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN && frame_content_size > std::numeric_limits::max())) { - return std::unexpected(Error { ErrorCode::DecompressionFailed }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid Zstandard frame content size")); } const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { - return std::unexpected(Error { ErrorCode::SizeLimitExceeded }); + return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit")); } const std::size_t allocation_size = content_size_is_known ? static_cast(frame_content_size) : effective_max_decompressed_size; @@ -173,22 +176,23 @@ std::expected checked_decompress(const Bytes& compressed_data, = ZSTD_decompress(uncompressed_data.data(), uncompressed_data.size(), compressed_data.data(), compressed_data.size()); if (ZSTD_isError(decompressed_size)) { if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { - return std::unexpected(Error { ErrorCode::ChecksumMismatch }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "Zstandard checksum mismatch")); } if (!content_size_is_known && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { - return std::unexpected(Error { ErrorCode::SizeLimitExceeded }); + return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit")); } - return std::unexpected(Error { ErrorCode::DecompressionFailed }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, + std::string("Zstandard decompression failed: ") + ZSTD_getErrorName(decompressed_size))); } if (content_size_is_known && decompressed_size != uncompressed_data.size()) { - return std::unexpected(Error { ErrorCode::DecompressionFailed }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "Zstandard decompressed size does not match its frame header")); } uncompressed_data.resize(decompressed_size); } if (checksum_algorithm == ChecksumAlgorithm::Crc32c && crc32c_checksum(uncompressed_data) != checksum) { - return std::unexpected(Error { ErrorCode::ChecksumMismatch }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "CRC32C checksum mismatch")); } return uncompressed_data; } diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h index dc61a4f9..f4c6a5ad 100644 --- a/src/terrainlib/io/compression.h +++ b/src/terrainlib/io/compression.h @@ -9,6 +9,8 @@ #include +#include "Error.h" + namespace io::envelope { using Bytes = std::vector; @@ -27,37 +29,15 @@ enum class CompressionAlgorithm : std::uint8_t { ZstdDefaultCompressionWithChecksum, }; -enum class ErrorCode : std::uint8_t { - SerializationFailed, - DeserializationFailed, - InvalidMagic, - WrongClassName, - UnsupportedClassVersion, - UnsupportedChecksumAlgorithm, - UnsupportedCompressionAlgorithm, - InvalidAlgorithmCombination, - ChecksumMismatch, - CompressionFailed, - DecompressionFailed, - SizeLimitExceeded, -}; - -struct Error { - ErrorCode code; - std::errc serialization_error {}; - - constexpr bool operator==(const Error&) const = default; -}; - struct CompressedData { Bytes compressed_data; std::string checksum; }; -std::expected compress_with_checksum( +std::expected compress_with_checksum( const Bytes& uncompressed_data, CompressionAlgorithm compression_algorithm, ChecksumAlgorithm checksum_algorithm); -std::expected checked_decompress(const Bytes& compressed_data, +std::expected checked_decompress(const Bytes& compressed_data, CompressionAlgorithm compression_algorithm, ChecksumAlgorithm checksum_algorithm, std::string_view checksum, diff --git a/src/terrainlib/io/conversion.h b/src/terrainlib/io/conversion.h index 7ddd6a14..49000c7b 100644 --- a/src/terrainlib/io/conversion.h +++ b/src/terrainlib/io/conversion.h @@ -11,28 +11,18 @@ #include #include -namespace io::conversion { - -enum class ErrorCode { - EmptyInput, - UnsupportedDimensions, - UnsupportedPixelType, - PixelTypeMismatch, - DimensionsOutOfRange, - AllocationFailed, - OpenCvFailure, -}; +#include "Error.h" -struct Error { - ErrorCode code; - int expected_cv_type { -1 }; - int actual_cv_type { -1 }; - - constexpr bool operator==(const Error&) const = default; -}; +namespace io::conversion { namespace detail { + inline ::Error allocation_error( + const std::source_location location = std::source_location::current()) noexcept + { + return ::Error::make(::Error::Code::ResourceExhausted, "allocate raster conversion output", location); + } + template inline constexpr int cv_depth = -1; @@ -143,27 +133,26 @@ namespace detail { } // namespace detail template -std::expected, Error> to_raster(const cv::Mat& source) +std::expected, ::Error> to_raster(const cv::Mat& source) { if constexpr (!detail::Traits::supported) { - return std::unexpected(Error { ErrorCode::UnsupportedPixelType }); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported raster pixel type")); } else { if (source.empty()) { - return std::unexpected(Error { ErrorCode::EmptyInput }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "cannot convert an empty OpenCV matrix to a raster")); } if (source.dims != 2) { - return std::unexpected(Error { ErrorCode::UnsupportedDimensions }); + return std::unexpected(::Error::make(::Error::Code::Unsupported, + "unsupported OpenCV matrix dimension count " + std::to_string(source.dims))); } if (source.type() != detail::cv_type) { - return std::unexpected(Error { - ErrorCode::PixelTypeMismatch, - detail::cv_type, - source.type(), - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, + "OpenCV matrix type " + std::to_string(source.type()) + " does not match expected type " + + std::to_string(detail::cv_type))); } if (source.cols < 0 || source.rows < 0 || static_cast(source.cols) > std::numeric_limits::max() || static_cast(source.rows) > std::numeric_limits::max()) { - return std::unexpected(Error { ErrorCode::DimensionsOutOfRange }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "OpenCV matrix dimensions are outside the raster range")); } try { @@ -174,25 +163,25 @@ std::expected, Error> to_raster(const cv::Mat& source) detail::copy_mat_to_raster(source, destination); return destination; } catch (const std::bad_alloc&) { - return std::unexpected(Error { ErrorCode::AllocationFailed }); - } catch (const cv::Exception&) { - return std::unexpected(Error { ErrorCode::OpenCvFailure }); + return std::unexpected(detail::allocation_error()); + } catch (const cv::Exception& error) { + return std::unexpected(::Error::make(::Error::Code::Internal, "OpenCV raster conversion failed: " + error.msg)); } } } template -std::expected to_mat(const radix::Raster& source) +std::expected to_mat(const radix::Raster& source) { if constexpr (!detail::Traits::supported) { - return std::unexpected(Error { ErrorCode::UnsupportedPixelType }); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported raster pixel type")); } else { if (source.width() == 0 || source.height() == 0) { - return std::unexpected(Error { ErrorCode::EmptyInput }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "cannot convert an empty raster to an OpenCV matrix")); } if (source.width() > static_cast(std::numeric_limits::max()) || source.height() > static_cast(std::numeric_limits::max())) { - return std::unexpected(Error { ErrorCode::DimensionsOutOfRange }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "raster dimensions are outside the OpenCV matrix range")); } try { @@ -200,9 +189,9 @@ std::expected to_mat(const radix::Raster& source) detail::copy_raster_to_mat(source, destination); return destination; } catch (const std::bad_alloc&) { - return std::unexpected(Error { ErrorCode::AllocationFailed }); - } catch (const cv::Exception&) { - return std::unexpected(Error { ErrorCode::OpenCvFailure }); + return std::unexpected(detail::allocation_error()); + } catch (const cv::Exception& error) { + return std::unexpected(::Error::make(::Error::Code::Internal, "OpenCV raster conversion failed: " + error.msg)); } } } diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h index bf9a6561..2cce906e 100644 --- a/src/terrainlib/io/envelope.h +++ b/src/terrainlib/io/envelope.h @@ -115,17 +115,17 @@ struct PayloadSchema { }; template -std::expected serialize(const typename Schema::template payload_type& payload, +std::expected serialize(const typename Schema::template payload_type& payload, CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template -std::expected serialize(const typename Schema::latest_type& payload, +std::expected serialize(const typename Schema::latest_type& payload, CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template -std::expected deserialize( +std::expected deserialize( std::span bytes, std::size_t max_decompressed_size = default_max_decompressed_size); } // namespace io::envelope diff --git a/src/terrainlib/io/envelope.inl b/src/terrainlib/io/envelope.inl index 4fc57314..0831ab77 100644 --- a/src/terrainlib/io/envelope.inl +++ b/src/terrainlib/io/envelope.inl @@ -8,28 +8,28 @@ namespace io::envelope { namespace detail { template -std::expected serialize_to_bytes(const Value &value) +std::expected serialize_to_bytes(const Value& value) { Bytes bytes; zpp::bits::out output(bytes, zpp::bits::alloc_limit()); const zpp::bits::errc result = output(value); if (zpp::bits::failure(result)) { - return std::unexpected(Error{ErrorCode::SerializationFailed, result.code}); + const auto cause = std::make_error_code(result.code); + const auto code = result.code == std::errc::not_enough_memory ? ::Error::Code::ResourceExhausted : ::Error::Code::Internal; + return std::unexpected(::Error::make(code, "serialize envelope data", cause)); } return bytes; } template -std::expected deserialize_from_bytes(const std::span bytes) +std::expected deserialize_from_bytes(const std::span bytes) { Value value{}; zpp::bits::in input(bytes, zpp::bits::alloc_limit()); const zpp::bits::errc result = input(value); if (zpp::bits::failure(result) || input.position() != bytes.size()) { - return std::unexpected(Error{ - ErrorCode::DeserializationFailed, - zpp::bits::failure(result) ? result.code : std::errc::bad_message, - }); + const auto cause = std::make_error_code(zpp::bits::failure(result) ? result.code : std::errc::bad_message); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "deserialize envelope data", cause)); } return value; } @@ -46,7 +46,7 @@ typename Schema::latest_type convert_to_latest(Current current) } template -std::expected deserialize_version( +std::expected deserialize_version( const std::uint32_t class_version, const std::span payload_bytes) { @@ -54,7 +54,7 @@ std::expected deserialize_version( if (class_version == CurrentVersion::number) { auto payload = deserialize_from_bytes(payload_bytes); if (!payload) { - return std::unexpected(payload.error()); + return std::unexpected(std::move(payload.error()).with_context("deserializing versioned envelope payload")); } return convert_to_latest(std::move(*payload)); } @@ -62,14 +62,15 @@ std::expected deserialize_version( if constexpr (Index + 1 < Schema::version_count) { return deserialize_version(class_version, payload_bytes); } else { - return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + return std::unexpected(::Error::make(::Error::Code::Unsupported, + "unsupported envelope class version " + std::to_string(class_version))); } } } // namespace detail template -std::expected serialize( +std::expected serialize( const typename Schema::template payload_type &payload, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) @@ -79,12 +80,12 @@ std::expected serialize( auto payload_bytes = detail::serialize_to_bytes(payload); if (!payload_bytes) { - return std::unexpected(payload_bytes.error()); + return std::unexpected(std::move(payload_bytes.error()).with_context("serializing envelope payload")); } auto compressed = compress_with_checksum(*payload_bytes, compression_algorithm, checksum_algorithm); if (!compressed) { - return std::unexpected(compressed.error()); + return std::unexpected(std::move(compressed.error()).with_context("compressing envelope payload")); } const Envelope envelope{ @@ -101,7 +102,7 @@ std::expected serialize( } template -std::expected serialize( +std::expected serialize( const typename Schema::latest_type &payload, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) @@ -113,29 +114,31 @@ std::expected serialize( } template -std::expected deserialize( +std::expected deserialize( const std::span bytes, const std::size_t max_decompressed_size) { auto envelope = detail::deserialize_from_bytes(bytes); if (!envelope) { - return std::unexpected(envelope.error()); + return std::unexpected(std::move(envelope.error()).with_context("reading envelope header")); } if (envelope->magic != magic) { - return std::unexpected(Error{ErrorCode::InvalidMagic}); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid envelope magic")); } if (envelope->class_name != Schema::class_name) { - return std::unexpected(Error{ErrorCode::WrongClassName}); + return std::unexpected(::Error::make(::Error::Code::CorruptData, + "unexpected envelope class \"" + envelope->class_name + "\", expected \"" + std::string(Schema::class_name) + "\"")); } if (!Schema::supports_version_number(envelope->class_version)) { - return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + return std::unexpected(::Error::make(::Error::Code::Unsupported, + "unsupported envelope class version " + std::to_string(envelope->class_version))); } const std::size_t effective_max_decompressed_size = std::min(max_decompressed_size, default_max_decompressed_size); if (envelope->uncompressed_size > std::numeric_limits::max() || envelope->uncompressed_size > effective_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "envelope payload exceeds the configured size limit")); } auto payload_bytes = checked_decompress( @@ -145,13 +148,18 @@ std::expected deserialize( envelope->checksum, static_cast(envelope->uncompressed_size)); if (!payload_bytes) { - if (payload_bytes.error().code == ErrorCode::SizeLimitExceeded) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + if (payload_bytes.error().code() == ::Error::Code::ResourceExhausted) { + return std::unexpected(std::move(payload_bytes.error()).reclassified( + ::Error::Code::CorruptData, "decompressed payload exceeds the size declared by the envelope")); } - return std::unexpected(payload_bytes.error()); + if (payload_bytes.error().code() == ::Error::Code::InvalidInput) { + return std::unexpected(std::move(payload_bytes.error()).reclassified( + ::Error::Code::CorruptData, "envelope declares an invalid compression and checksum combination")); + } + return std::unexpected(std::move(payload_bytes.error()).with_context("decompressing envelope payload")); } if (payload_bytes->size() != envelope->uncompressed_size) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "decompressed payload size does not match the envelope declaration")); } return detail::deserialize_version(envelope->class_version, *payload_bytes); diff --git a/src/terrainlib/io/envelope_file.h b/src/terrainlib/io/envelope_file.h index d70476c0..dead8c3a 100644 --- a/src/terrainlib/io/envelope_file.h +++ b/src/terrainlib/io/envelope_file.h @@ -1,12 +1,10 @@ #pragma once -#include #include #include #include #include -#include -#include +#include #include @@ -15,48 +13,24 @@ namespace io::envelope { -using FileError = std::variant<::io::Error, Error>; - -inline bool is_file_not_found(const FileError& error) -{ - const auto* io_error = std::get_if<::io::Error>(&error); - return io_error != nullptr && *io_error == ::io::Error(::io::Error::OpenFile); -} - -inline std::string describe_error(const Error& error) { return "envelope error " + std::to_string(static_cast(error.code)); } - -inline std::string describe_error(const FileError& error) -{ - return std::visit( - [](const auto& value) -> std::string { - using Value = std::remove_cvref_t; - if constexpr (std::same_as) { - return value.to_string(); - } else { - return describe_error(value); - } - }, - error); -} - template -std::expected read_from_path( +std::expected read_from_path( const std::filesystem::path& path, const std::size_t max_decompressed_size = default_max_decompressed_size) { auto file_bytes = ::io::read_bytes_from_path(path); if (!file_bytes) { - return std::unexpected(FileError { file_bytes.error() }); + return std::unexpected(std::move(file_bytes.error()).with_context("reading envelope file \"" + path.string() + "\"")); } const auto bytes = std::as_bytes(std::span { *file_bytes }); auto result = deserialize(bytes, max_decompressed_size); if (!result) { - return std::unexpected(FileError { result.error() }); + return std::unexpected(std::move(result.error()).with_context("decoding envelope file \"" + path.string() + "\"")); } return std::move(*result); } template -std::expected write_to_path(const typename Schema::latest_type& payload, +std::expected write_to_path(const typename Schema::latest_type& payload, const std::filesystem::path& path, const bool make_dirs = true, const CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, @@ -64,7 +38,7 @@ std::expected write_to_path(const typename Schema::latest_type& { auto serialized = serialize(payload, compression_algorithm, checksum_algorithm); if (!serialized) { - return std::unexpected(FileError { serialized.error() }); + return std::unexpected(std::move(serialized.error()).with_context("encoding envelope file \"" + path.string() + "\"")); } const auto bytes = std::span { reinterpret_cast(serialized->data()), @@ -72,7 +46,7 @@ std::expected write_to_path(const typename Schema::latest_type& }; auto result = ::io::write_bytes_to_path(bytes, path, make_dirs); if (!result) { - return std::unexpected(FileError { result.error() }); + return std::unexpected(std::move(result.error()).with_context("writing envelope file \"" + path.string() + "\"")); } return {}; } diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index 5927b25c..3b603ff8 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -29,43 +29,28 @@ class Gltf final : public store::Codec { return { add_extension(node_path, m_container == GltfContainer::Binary ? ".glb" : ".gltf") }; } - std::expected read(const std::filesystem::path& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { const auto result = mesh::io::gltf::load_from_path(paths(node_path).front()); if (!result.has_value()) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - result.error() == mesh::io::LoadMeshErrorKind::FileNotFound ? store::CodecErrorCategory::FileNotFound : store::CodecErrorCategory::InvalidData, - result.error().description(), - }); + return std::unexpected(::Error::make(result.error() == mesh::io::LoadMeshErrorKind::FileNotFound ? ::Error::Code::NotFound : ::Error::Code::CorruptData, + result.error().description())); } return std::move(result.value()); } - std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override + std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { try { const auto result = mesh::io::gltf::save_to_path(mesh, paths(node_path).front()); if (!result.has_value()) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - result.error().description(), - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, result.error().description())); } return {}; } catch (const std::exception& error) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - error.what(), - }); + return std::unexpected(::Error::make(::Error::Code::Internal, error.what())); } catch (...) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - "unknown glTF writer exception", - }); + return std::unexpected(::Error::make(::Error::Code::Internal, "unknown glTF writer exception")); } } diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h index 67d95be5..74ad6ee2 100644 --- a/src/terrainlib/mesh/codec/SfMesh.h +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -17,63 +17,39 @@ class SfMesh final : public store::Codec { return { add_extension(node_path, ".sfmesh") }; } - std::expected read(const std::filesystem::path& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { auto payload = ::io::envelope::read_from_path(paths(node_path).front()); if (!payload) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - ::io::envelope::is_file_not_found(payload.error()) ? store::CodecErrorCategory::FileNotFound : store::CodecErrorCategory::InvalidData, - ::io::envelope::describe_error(payload.error()), - }); + return std::unexpected(std::move(payload).error().with_context("read SF mesh")); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::InvalidData, - valid.error(), - }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, valid.error())); } auto decoded = mesh::sf::decode_payload(std::move(*payload)); if (!decoded) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::InvalidData, - "could not decode SF mesh payload", - }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "could not decode SF mesh payload")); } return std::move(*decoded); } - std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override + std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { return write(node_path, mesh, {}); } - std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const + std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const { auto payload = mesh::sf::encode_payload(mesh, options); if (!payload) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - "could not encode SF mesh payload", - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "could not encode SF mesh payload")); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - valid.error(), - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, valid.error())); } auto result = ::io::envelope::write_to_path(*payload, paths(node_path).front()); if (!result) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Io, - ::io::envelope::describe_error(result.error()), - }); + return std::unexpected(std::move(result).error().with_context("write SF mesh")); } return {}; } diff --git a/src/terrainlib/mesh/codec/from_extension.h b/src/terrainlib/mesh/codec/from_extension.h index ed3dca40..d8e5ae7c 100644 --- a/src/terrainlib/mesh/codec/from_extension.h +++ b/src/terrainlib/mesh/codec/from_extension.h @@ -12,7 +12,7 @@ namespace mesh::codec { -inline std::expected>, store::CodecError> from_extension(const std::string_view extension) +inline std::expected>, ::Error> from_extension(const std::string_view extension) { if (extension == ".sfmesh") { return std::make_unique(); @@ -23,11 +23,7 @@ inline std::expected>, store::CodecEr if (extension == ".gltf") { return std::make_unique(GltfContainer::Json); } - return std::unexpected(store::CodecError { - store::CodecOperation::Resolve, - store::CodecErrorCategory::UnsupportedCodec, - "unsupported mesh codec selector: " + std::string(extension), - }); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported mesh codec selector: " + std::string(extension))); } } // namespace mesh::codec diff --git a/src/terrainlib/mesh/storage.h b/src/terrainlib/mesh/storage.h index ebae84ed..13536bff 100644 --- a/src/terrainlib/mesh/storage.h +++ b/src/terrainlib/mesh/storage.h @@ -18,17 +18,17 @@ using Storage = store::Storage; using IndexedStorage = store::IndexedStorage; using OpenOptions = octree::storage::OpenOptions; -inline std::expected> open_index(const std::filesystem::path& path) +inline std::expected open_index(const std::filesystem::path& path) { return store::open_index(path, octree::storage::index_format(), payload_class, mesh::codec::from_extension); } -inline std::expected> open_folder(const std::filesystem::path& path, OpenOptions options = {}) +inline std::expected open_folder(const std::filesystem::path& path, OpenOptions options = {}) { return octree::storage::open_folder(path, std::string(payload_class), ".sfmesh", mesh::codec::from_extension, std::move(options)); } -inline std::expected> open_folder_indexed(const std::filesystem::path& path, OpenOptions options = {}) +inline std::expected open_folder_indexed(const std::filesystem::path& path, OpenOptions options = {}) { return octree::storage::open_folder_indexed(path, std::string(payload_class), ".sfmesh", mesh::codec::from_extension, std::move(options)); } diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 9527e94f..0dde3a39 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -123,33 +123,25 @@ inline std::expected, std::string> decode_index(const return result; } -inline store::IndexFormatError read_error(const std::filesystem::path& path, const io::envelope::FileError& error) +inline ::Error read_error(const std::filesystem::path& path, ::Error error) { - return { - io::envelope::is_file_not_found(error) ? store::IndexFormatErrorCategory::Open : store::IndexFormatErrorCategory::Malformed, - path, - io::envelope::describe_error(error), - }; + return std::move(error).with_context("read octree store file " + path.string()); } -inline std::expected read_store_metadata(const std::filesystem::path& base_path) +inline std::expected read_store_metadata(const std::filesystem::path& base_path) { const std::filesystem::path path = base_path / metadata_file_name; auto metadata = io::envelope::read_from_path(path); if (!metadata) { - return std::unexpected(read_error(path, metadata.error())); + return std::unexpected(read_error(path, std::move(metadata).error())); } if (auto valid = validate(*metadata); !valid) { - return std::unexpected(store::IndexFormatError { - store::IndexFormatErrorCategory::Malformed, - path, - valid.error(), - }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid store metadata in " + path.string() + ": " + valid.error())); } return std::move(*metadata); } -inline std::expected, store::IndexFormatError> read_index_file(const std::filesystem::path& path) +inline std::expected, ::Error> read_index_file(const std::filesystem::path& path) { auto metadata = read_store_metadata(path.parent_path()); if (!metadata) { @@ -158,15 +150,11 @@ inline std::expected, store::IndexFormatError> auto encoded_index = io::envelope::read_from_path(path); if (!encoded_index) { - return std::unexpected(read_error(path, encoded_index.error())); + return std::unexpected(read_error(path, std::move(encoded_index).error())); } auto index = decode_index(*encoded_index); if (!index) { - return std::unexpected(store::IndexFormatError { - store::IndexFormatErrorCategory::Malformed, - path, - index.error(), - }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid store index in " + path.string() + ": " + index.error())); } return store::IndexMetadata { std::move(*index), @@ -176,7 +164,7 @@ inline std::expected, store::IndexFormatError> }; } -inline std::expected write_index_file(const std::filesystem::path& path, const store::IndexMetadata& metadata) +inline std::expected write_index_file(const std::filesystem::path& path, const store::IndexMetadata& metadata) { const StoreMetadata store_metadata { .layout_id = metadata.layout_id, @@ -184,28 +172,16 @@ inline std::expected write_index_file(const std:: .codec_selector = metadata.codec_selector, }; if (auto valid = validate(store_metadata); !valid) { - return std::unexpected(store::IndexFormatError { - store::IndexFormatErrorCategory::Write, - path.parent_path() / metadata_file_name, - valid.error(), - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "invalid store metadata: " + valid.error())); } const std::filesystem::path metadata_path = path.parent_path() / metadata_file_name; if (auto written = io::envelope::write_to_path(store_metadata, metadata_path); !written) { - return std::unexpected(store::IndexFormatError { - store::IndexFormatErrorCategory::Write, - metadata_path, - io::envelope::describe_error(written.error()), - }); + return std::unexpected(std::move(written).error().with_context("write octree store metadata " + metadata_path.string())); } const StoreIndex encoded_index = encode_index(metadata.index); if (auto written = io::envelope::write_to_path(encoded_index, path); !written) { - return std::unexpected(store::IndexFormatError { - store::IndexFormatErrorCategory::Write, - path, - io::envelope::describe_error(written.error()), - }); + return std::unexpected(std::move(written).error().with_context("write octree store index " + path.string())); } return {}; } diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index eb7d3e98..86406012 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -20,7 +20,7 @@ struct OpenOptions { }; template -std::expected, store::OpenError> open_folder(const std::filesystem::path& base_path, +std::expected, ::Error> open_folder(const std::filesystem::path& base_path, std::string payload_class, std::string default_codec_selector, CodecResolver resolve_codec, @@ -29,27 +29,16 @@ std::expected, store::OpenError> open_ std::error_code error; const bool base_exists = std::filesystem::exists(base_path, error); if (error) { - return std::unexpected(store::OpenError(store::FilesystemError { - base_path, - "exists", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "check existence of", base_path, error)); } if (base_exists && !std::filesystem::is_directory(base_path, error)) { - return std::unexpected(store::OpenError(store::FilesystemError { - base_path, - "is_directory", - error ? error : std::make_error_code(std::errc::not_a_directory), - })); + return std::unexpected(::Error::make( + ::Error::Code::Io, "check directory", base_path, error ? error : std::make_error_code(std::errc::not_a_directory))); } if (!base_exists) { std::filesystem::create_directories(base_path, error); if (error) { - return std::unexpected(store::OpenError(store::FilesystemError { - base_path, - "create_directories", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "create directories", base_path, error)); } } @@ -58,26 +47,16 @@ std::expected, store::OpenError> open_ const std::filesystem::path metadata_path = base_path / metadata_file_name; const bool index_exists = std::filesystem::exists(index_path, error); if (error) { - return std::unexpected(store::OpenError(store::FilesystemError { - index_path, - "exists", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "check existence of", index_path, error)); } const bool metadata_exists = std::filesystem::exists(metadata_path, error); if (error) { - return std::unexpected(store::OpenError(store::FilesystemError { - metadata_path, - "exists", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "check existence of", metadata_path, error)); } if (index_exists != metadata_exists) { - return std::unexpected(store::OpenError(store::IndexFormatError { - store::IndexFormatErrorCategory::Open, - index_exists ? metadata_path : index_path, - "octree dataset metadata and index must either both exist or both be absent", - })); + return std::unexpected(::Error::make(::Error::Code::CorruptData, + "octree dataset metadata and index must either both exist or both be absent: " + + (index_exists ? metadata_path : index_path).string())); } if (index_exists) { auto indexed = store::open_index(index_path, format, payload_class, resolve_codec); @@ -91,14 +70,14 @@ std::expected, store::OpenError> open_ std::string codec_selector = options.preferred_extension.value_or(std::move(default_codec_selector)); auto codec = resolve_codec(codec_selector); if (!codec) { - return std::unexpected(store::OpenError(codec.error())); + return std::unexpected(codec.error()); } return store::make_storage(base_path, format, mapping, std::move(payload_class), std::move(codec_selector), std::move(*codec)); } template -std::expected, store::OpenError> open_folder_indexed(const std::filesystem::path& base_path, +std::expected, ::Error> open_folder_indexed(const std::filesystem::path& base_path, std::string payload_class, std::string default_codec_selector, CodecResolver resolve_codec, diff --git a/src/terrainlib/sf/Error.h b/src/terrainlib/sf/Error.h deleted file mode 100644 index 337a58c3..00000000 --- a/src/terrainlib/sf/Error.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "octree/Id.h" -#include "sf/InvalidTopology.h" -#include "store/OpenError.h" -#include "store/describe_error.h" - -namespace sf { - -using FinalizeError = std::variant; - -using ProcessingError = std::variant, - store::FileOperationError, - store::SaveError, - store::CopyError, - store::IndexFormatError>; - -inline std::string describe_error(const InvalidTopology& error) { return "Structura Fundamentalis topology contains Inner node " + error.key.to_string(); } - -template -std::string describe_error(const std::variant& error) -{ - return std::visit( - [](const auto& value) -> std::string { - if constexpr (std::is_same_v, InvalidTopology>) { - return describe_error(value); - } else { - return store::describe_error(value); - } - }, - error); -} - -} // namespace sf diff --git a/src/terrainlib/sf/InvalidTopology.h b/src/terrainlib/sf/InvalidTopology.h deleted file mode 100644 index a6010691..00000000 --- a/src/terrainlib/sf/InvalidTopology.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include "octree/Id.h" - -namespace sf { - -struct InvalidTopology { - octree::Id key; - - bool operator==(const InvalidTopology&) const = default; -}; - -} // namespace sf diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h index c2e6386e..0da0329e 100644 --- a/src/terrainlib/sf/finalize_storage.h +++ b/src/terrainlib/sf/finalize_storage.h @@ -5,23 +5,22 @@ #include #include "mesh/storage.h" -#include "sf/Error.h" #include "sf/validate_index.h" namespace sf { -inline std::expected finalize_storage(mesh::storage::Storage& storage) +inline std::expected finalize_storage(mesh::storage::Storage& storage) { const auto save_result = storage.save_or_create_index(); if (!save_result.has_value()) { - return std::unexpected(FinalizeError(save_result.error())); + return std::unexpected(save_result.error()); } const auto index = storage.index(); DEBUG_ASSERT(index.has_value()); const auto validation = validate_index(index->get()); if (!validation.has_value()) { - return std::unexpected(FinalizeError(validation.error())); + return std::unexpected(validation.error()); } return {}; } diff --git a/src/terrainlib/sf/validate_index.h b/src/terrainlib/sf/validate_index.h index da04a239..b9507b9c 100644 --- a/src/terrainlib/sf/validate_index.h +++ b/src/terrainlib/sf/validate_index.h @@ -3,16 +3,17 @@ #include #include "octree/StoreTraits.h" -#include "sf/InvalidTopology.h" +#include "Error.h" #include "store/Index.h" namespace sf { -inline std::expected validate_index(const store::Index& index) +inline std::expected validate_index(const store::Index& index) { for (const auto& [key, status] : index) { if (status == store::NodeStatus::Inner) { - return std::unexpected(InvalidTopology { key }); + return std::unexpected(::Error::make( + ::Error::Code::CorruptData, "Structura Fundamentalis topology contains Inner node " + key.to_string())); } } return {}; diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h index b45b6971..e7ff4c0c 100644 --- a/src/terrainlib/store/Codec.h +++ b/src/terrainlib/store/Codec.h @@ -6,7 +6,7 @@ #include -#include "store/CodecError.h" +#include "Error.h" namespace store { @@ -17,14 +17,14 @@ class Codec { virtual std::vector paths(const std::filesystem::path& node_path) const = 0; - virtual std::expected read(const std::filesystem::path&) const + virtual std::expected read(const std::filesystem::path&) const { - return std::unexpected(CodecError::unsupported_operation(CodecOperation::Read, "read")); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "codec does not support reading")); } - virtual std::expected write(const std::filesystem::path&, const NodeData&) const + virtual std::expected write(const std::filesystem::path&, const NodeData&) const { - return std::unexpected(CodecError::unsupported_operation(CodecOperation::Write, "write")); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "codec does not support writing")); } protected: diff --git a/src/terrainlib/store/CodecError.h b/src/terrainlib/store/CodecError.h deleted file mode 100644 index 835069bd..00000000 --- a/src/terrainlib/store/CodecError.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace store { - -enum class CodecOperation { - Read, - Write, - Resolve, -}; - -enum class CodecErrorCategory { - UnsupportedOperation, - UnsupportedCodec, - FileNotFound, - InvalidData, - Io, - Domain, -}; - -struct CodecError { - CodecOperation operation; - CodecErrorCategory category; - std::string message; - - static CodecError unsupported_operation(const CodecOperation operation, const std::string_view name) - { - return { operation, CodecErrorCategory::UnsupportedOperation, std::string(name) }; - } - - bool operator==(const CodecError&) const = default; -}; - -} // namespace store diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h index bb7998f5..3f000bbb 100644 --- a/src/terrainlib/store/Index.h +++ b/src/terrainlib/store/Index.h @@ -5,8 +5,8 @@ #include #include +#include "Error.h" #include "log.h" -#include "store/InvalidKey.h" #include "store/NodeStatus.h" #include "store/Traits.h" #include @@ -17,15 +17,14 @@ template class Index { public: using Key = typename Traits::Key; - using Error = InvalidKey; using Container = std::unordered_map; using iterator = typename Container::iterator; using const_iterator = typename Container::const_iterator; - std::expected, Error> get(const Key& key) const + std::expected, ::Error> get(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(Error { key }); + return std::unexpected(invalid_key_error(key)); } const NodeStatus* status = get_raw_unchecked(key); if (status == nullptr) { @@ -34,10 +33,10 @@ class Index { return *status; } - std::expected add(const Key& key) + std::expected add(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(Error { key }); + return std::unexpected(invalid_key_error(key)); } NodeStatus* status = get_raw_unchecked(key); @@ -63,7 +62,7 @@ class Index { } if (!Traits::is_valid(parent.value())) { - return std::unexpected(Error { parent.value() }); + return std::unexpected(invalid_key_error(parent.value())); } NodeStatus* parent_status = get_raw_unchecked(parent.value()); if (parent_status == nullptr) { @@ -89,10 +88,10 @@ class Index { return true; } - std::expected remove(const Key& key) + std::expected remove(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(Error { key }); + return std::unexpected(invalid_key_error(key)); } NodeStatus* status = get_raw_unchecked(key); @@ -112,14 +111,14 @@ class Index { UNREACHABLE(); } - std::expected is_present(const Key& key) const + std::expected is_present(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(Error { key }); + return std::unexpected(invalid_key_error(key)); } return m_index.contains(key); } - std::expected is_absent(const Key& key) const + std::expected is_absent(const Key& key) const { const auto present = is_present(key); if (!present.has_value()) { @@ -127,7 +126,7 @@ class Index { } return !present.value(); } - std::expected is(const NodeStatus status, const Key& key) const + std::expected is(const NodeStatus status, const Key& key) const { const auto result = get(key); if (!result.has_value()) { @@ -136,32 +135,32 @@ class Index { return result.value() == status; } - std::expected get_raw(const Key& key) + std::expected get_raw(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(Error { key }); + return std::unexpected(invalid_key_error(key)); } return get_raw_unchecked(key); } - std::expected get_raw(const Key& key) const + std::expected get_raw(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(Error { key }); + return std::unexpected(invalid_key_error(key)); } return get_raw_unchecked(key); } - std::expected set_raw(const Key& key, const NodeStatus status) + std::expected set_raw(const Key& key, const NodeStatus status) { if (!Traits::is_valid(key)) { - return std::unexpected(Error { key }); + return std::unexpected(invalid_key_error(key)); } set_raw_unchecked(key, status); return {}; } - std::expected remove_raw(const Key& key) + std::expected remove_raw(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(Error { key }); + return std::unexpected(invalid_key_error(key)); } remove_raw_unchecked(key); return {}; @@ -177,6 +176,11 @@ class Index { const_iterator cend() const { return m_index.cend(); } private: + static ::Error invalid_key_error(const Key& key) + { + return ::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + key_to_string(key)); + } + NodeStatus* get_raw_unchecked(const Key& key) { const auto iterator = m_index.find(key); @@ -190,14 +194,14 @@ class Index { void set_raw_unchecked(const Key& key, const NodeStatus status) { m_index[key] = status; } void remove_raw_unchecked(const Key& key) { m_index.erase(key); } - std::expected update_parent_after_remove(const Key& key) + std::expected update_parent_after_remove(const Key& key) { const auto parent = Traits::parent(key); if (!parent.has_value()) { return true; } if (!Traits::is_valid(parent.value())) { - return std::unexpected(Error { parent.value() }); + return std::unexpected(invalid_key_error(parent.value())); } NodeStatus* parent_status = get_raw_unchecked(parent.value()); diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h index 0d001176..df8c79e6 100644 --- a/src/terrainlib/store/IndexFormat.h +++ b/src/terrainlib/store/IndexFormat.h @@ -7,25 +7,12 @@ #include +#include "Error.h" #include "store/Index.h" #include "store/path_layout.h" namespace store { -enum class IndexFormatErrorCategory { - Open, - Write, - Malformed, -}; - -struct IndexFormatError { - IndexFormatErrorCategory category; - std::filesystem::path path; - std::string message; - - bool operator==(const IndexFormatError&) const = default; -}; - template struct IndexMetadata { Index index; @@ -38,8 +25,8 @@ template struct IndexFormat { std::string_view index_filename; - std::expected, IndexFormatError> (*read)(const std::filesystem::path& index_path); - std::expected (*write)(const std::filesystem::path& index_path, const IndexMetadata& metadata); + std::expected, ::Error> (*read)(const std::filesystem::path& index_path); + std::expected (*write)(const std::filesystem::path& index_path, const IndexMetadata& metadata); std::optional> (*mapping_from_id)(std::string_view id); path_layout::Mapping (*default_mapping)(); }; diff --git a/src/terrainlib/store/InvalidKey.h b/src/terrainlib/store/InvalidKey.h deleted file mode 100644 index 6b23ee9c..00000000 --- a/src/terrainlib/store/InvalidKey.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -namespace store { - -template -struct InvalidKey { - Key key; - - bool operator==(const InvalidKey&) const = default; -}; - -} // namespace store diff --git a/src/terrainlib/store/OpenError.h b/src/terrainlib/store/OpenError.h deleted file mode 100644 index 29b21bc2..00000000 --- a/src/terrainlib/store/OpenError.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include - -#include "store/IndexFormat.h" -#include "store/StorageError.h" - -namespace store { - -struct UnknownLayout { - std::string id; - bool operator==(const UnknownLayout&) const = default; -}; - -template -using OpenError = std::variant>; - -} // namespace store diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index 988dfde3..ffe0a982 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -11,7 +11,6 @@ #include "store/Codec.h" #include "store/path_layout.h" -#include "store/StorageError.h" #include "store/Traits.h" namespace store { @@ -33,43 +32,43 @@ class RawStorage { RawStorage(RawStorage&&) noexcept = default; RawStorage& operator=(RawStorage&&) noexcept = default; - std::expected> load(const Key& key) const + std::expected load(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(LoadError(InvalidKey { key })); + return std::unexpected(invalid_key_error(key)); } const auto result = m_codec->read(m_layout.node_path(key)); if (!result.has_value()) { - return std::unexpected(LoadError(result.error())); + return std::unexpected(result.error()); } return std::move(*result); } - std::expected> save(const Key& key, const NodeData& data) const + std::expected save(const Key& key, const NodeData& data) const { if (!Traits::is_valid(key)) { - return std::unexpected(SaveError(InvalidKey { key })); + return std::unexpected(invalid_key_error(key)); } const auto result = m_codec->write(m_layout.node_path(key), data); if (!result.has_value()) { - return std::unexpected(SaveError(result.error())); + return std::unexpected(result.error()); } return {}; } - std::expected, InvalidKey> paths(const Key& key) const + std::expected, ::Error> paths(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(InvalidKey { key }); + return std::unexpected(invalid_key_error(key)); } return m_codec->paths(m_layout.node_path(key)); } - std::expected> has(const Key& key) const + std::expected has(const Key& key) const { const auto node_paths = paths(key); if (!node_paths.has_value()) { - return std::unexpected(FileOperationError(node_paths.error())); + return std::unexpected(node_paths.error()); } if (node_paths->empty()) { return false; @@ -78,11 +77,7 @@ class RawStorage { std::error_code error; const bool exists = std::filesystem::exists(path, error); if (error) { - return std::unexpected(FileOperationError(FilesystemError { - path, - "exists", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "check existence of", path, error)); } if (!exists) { return false; @@ -91,59 +86,55 @@ class RawStorage { return true; } - std::expected> remove(const Key& key) const + std::expected remove(const Key& key) const { const auto node_paths = paths(key); if (!node_paths.has_value()) { - return std::unexpected(FileOperationError(node_paths.error())); + return std::unexpected(node_paths.error()); } bool removed = false; for (const auto& path : node_paths.value()) { std::error_code error; removed = std::filesystem::remove(path, error) || removed; if (error) { - return std::unexpected(FileOperationError(FilesystemError { - path, - "remove", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "remove", path, error)); } } return removed; } - std::expected> copy_from(const Key& key, const RawStorage& source) const + std::expected copy_from(const Key& key, const RawStorage& source) const { - return copy_from(key, source, []() -> std::expected> { return {}; }); + return copy_from(key, source, []() -> std::expected { return {}; }); } template - std::expected> copy_from(const Key& key, const RawStorage& source, BeforeModify&& before_modify) const + std::expected copy_from(const Key& key, const RawStorage& source, BeforeModify&& before_modify) const { if (!Traits::is_valid(key)) { - return std::unexpected(CopyError(InvalidKey { key })); + return std::unexpected(invalid_key_error(key)); } const auto source_exists = source.has(key); if (!source_exists.has_value()) { - return std::unexpected(std::visit([](const auto& error) -> CopyError { return error; }, source_exists.error())); + return std::unexpected(source_exists.error()); } if (!source_exists.value()) { - return std::unexpected(CopyError(MissingSource { key })); + return std::unexpected(::Error::make(::Error::Code::NotFound, "source node " + key_to_string(key) + " is missing")); } const std::filesystem::path probe("__codec_probe__/node"); if (m_codec->paths(probe) != source.m_codec->paths(probe)) { - const auto loaded = source.m_codec->read(source.m_layout.node_path(key)); + auto loaded = source.m_codec->read(source.m_layout.node_path(key)); if (!loaded.has_value()) { - return std::unexpected(CopyError(loaded.error())); + return std::unexpected(std::move(loaded).error().with_context("read source node while copying")); } const auto prepared = before_modify(); if (!prepared.has_value()) { return prepared; } - const auto written = m_codec->write(m_layout.node_path(key), loaded.value()); + auto written = m_codec->write(m_layout.node_path(key), loaded.value()); if (!written.has_value()) { - return std::unexpected(CopyError(written.error())); + return std::unexpected(std::move(written).error().with_context("write target node while copying")); } return {}; } @@ -151,11 +142,7 @@ class RawStorage { const auto source_paths = source.m_codec->paths(source.m_layout.node_path(key)); const auto target_paths = m_codec->paths(m_layout.node_path(key)); if (source_paths.size() != target_paths.size()) { - return std::unexpected(CopyError(CodecError { - CodecOperation::Write, - CodecErrorCategory::Domain, - "matching codec probes produced different actual path counts", - })); + return std::unexpected(::Error::make(::Error::Code::Internal, "matching codec probes produced different actual path counts")); } if (source_paths == target_paths) { return {}; @@ -168,27 +155,16 @@ class RawStorage { std::error_code error; std::filesystem::remove(target_paths[index], error); if (error) { - return std::unexpected(CopyError(FilesystemError { - target_paths[index], - "remove", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "remove", target_paths[index], error)); } std::filesystem::create_directories(target_paths[index].parent_path(), error); if (error) { - return std::unexpected(CopyError(FilesystemError { - target_paths[index].parent_path(), - "create_directories", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "create directories", target_paths[index].parent_path(), error)); } std::filesystem::create_hard_link(source_paths[index], target_paths[index], error); if (error) { - return std::unexpected(CopyError(FilesystemError { - target_paths[index], - "create_hard_link", - error, - })); + return std::unexpected(::Error::make( + ::Error::Code::Io, "create hard link", source_paths[index], target_paths[index], error)); } } return {}; @@ -198,6 +174,11 @@ class RawStorage { const Codec& codec() const { return *m_codec; } private: + static ::Error invalid_key_error(const Key& key) + { + return ::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + key_to_string(key)); + } + path_layout::Resolver m_layout; std::unique_ptr> m_codec; }; diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index ae09a03c..9bb73927 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -24,8 +24,8 @@ class Storage { using Key = typename Traits::Key; using key_type = Key; using value_type = NodeData; - using load_error = LoadError; - using save_error = SaveError; + using load_error = ::Error; + using save_error = ::Error; struct Persistence { IndexFormat format; @@ -80,41 +80,38 @@ class Storage { return *this; } - std::expected> load(const Key& key) const + std::expected load(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(LoadError(InvalidKey { key })); + return std::unexpected(invalid_key_error(key)); } if (m_index.has_value()) { const auto status = m_index->get(key); if (!status.has_value()) { - return std::unexpected(LoadError(status.error())); + return std::unexpected(status.error()); } if (!status->has_value() || status->value() == NodeStatus::Virtual) { - return std::unexpected(LoadError(CodecError { - CodecOperation::Read, - CodecErrorCategory::FileNotFound, - "node is not physically present in the index", - })); + return std::unexpected( + ::Error::make(::Error::Code::NotFound, "node " + key_to_string(key) + " is not physically present in the index")); } } return m_raw.load(key); } - std::expected> save(const Key& key, const NodeData& data) + std::expected save(const Key& key, const NodeData& data) { if (!Traits::is_valid(key)) { - return std::unexpected(SaveError(InvalidKey { key })); + return std::unexpected(invalid_key_error(key)); } const auto exists = has(key); if (!exists.has_value()) { - return std::unexpected(std::visit([](const auto& error) -> SaveError { return error; }, exists.error())); + return std::unexpected(exists.error()); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); - return std::unexpected(SaveError(AlreadyExists { - node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front(), - })); + return std::unexpected(::Error::make(::Error::Code::AlreadyExists, + "save node to", + node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front())); } const auto result = m_raw.save(key, data); @@ -124,35 +121,35 @@ class Storage { if (m_index.has_value()) { const auto added = m_index->add(key); if (!added.has_value()) { - return std::unexpected(SaveError(added.error())); + return std::unexpected(added.error()); } m_dirty = m_dirty || added.value(); } return {}; } - std::expected> copy_from(const Key& key, const Storage& source) + std::expected copy_from(const Key& key, const Storage& source) { if (!Traits::is_valid(key)) { - return std::unexpected(CopyError(InvalidKey { key })); + return std::unexpected(invalid_key_error(key)); } const auto exists = has(key); if (!exists.has_value()) { - return std::unexpected(std::visit([](const auto& error) -> CopyError { return error; }, exists.error())); + return std::unexpected(exists.error()); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); - return std::unexpected(CopyError(AlreadyExists { - node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front(), - })); + return std::unexpected(::Error::make(::Error::Code::AlreadyExists, + "copy node to", + node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front())); } - const auto prepare_target = [&]() -> std::expected> { + const auto prepare_target = [&]() -> std::expected { if (!exists.value() || !m_index.has_value()) { return {}; } const auto removed = m_index->remove(key); if (!removed.has_value()) { - return std::unexpected(CopyError(removed.error())); + return std::unexpected(removed.error()); } m_dirty = m_dirty || removed.value(); return {}; @@ -165,14 +162,14 @@ class Storage { if (m_index.has_value()) { const auto added = m_index->add(key); if (!added.has_value()) { - return std::unexpected(CopyError(added.error())); + return std::unexpected(added.error()); } m_dirty = m_dirty || added.value(); } return {}; } - std::expected> remove(const Key& key) + std::expected remove(const Key& key) { const auto removed = m_raw.remove(key); if (!removed.has_value()) { @@ -181,31 +178,31 @@ class Storage { if (m_index.has_value()) { const auto index_removed = m_index->remove(key); if (!index_removed.has_value()) { - return std::unexpected(FileOperationError(index_removed.error())); + return std::unexpected(index_removed.error()); } m_dirty = m_dirty || index_removed.value(); } return removed.value(); } - std::expected> has(const Key& key) const + std::expected has(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(FileOperationError(InvalidKey { key })); + return std::unexpected(invalid_key_error(key)); } if (!m_index.has_value()) { return m_raw.has(key); } const auto status = m_index->get(key); if (!status.has_value()) { - return std::unexpected(FileOperationError(status.error())); + return std::unexpected(status.error()); } return status->has_value() && status->value() != NodeStatus::Virtual; } - std::expected, InvalidKey> paths(const Key& key) const { return m_raw.paths(key); } + std::expected, ::Error> paths(const Key& key) const { return m_raw.paths(key); } - std::expected> path_for(const Key& key) const + std::expected path_for(const Key& key) const { const auto node_paths = paths(key); if (!node_paths.has_value()) { @@ -242,17 +239,13 @@ class Storage { return std::cref(m_index.value()); } - std::expected save_index() const + std::expected save_index() const { if (!m_dirty) { return {}; } if (!m_index.has_value() || !m_persistence.has_value()) { - return std::unexpected(IndexFormatError { - IndexFormatErrorCategory::Write, - {}, - "indexed storage has no persistence configuration", - }); + return std::unexpected(::Error::make(::Error::Code::Internal, "indexed storage has no persistence configuration")); } const IndexMetadata metadata { m_index.value(), @@ -267,7 +260,7 @@ class Storage { return result; } - std::expected save_or_create_index() + std::expected save_or_create_index() { if (!m_index.has_value()) { m_index.emplace(); @@ -286,6 +279,11 @@ class Storage { void set_index_dirty(const bool dirty = true) const { m_dirty = dirty; } private: + static ::Error invalid_key_error(const Key& key) + { + return ::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + key_to_string(key)); + } + void finalize_displaced_state() noexcept { if (!m_dirty || !m_index.has_value()) { @@ -293,7 +291,7 @@ class Storage { } const auto result = save_index(); if (!result.has_value()) { - LOG_ERROR("Failed to automatically save index {}: {}", result.error().path, result.error().message); + LOG_ERROR("Failed to automatically save index: {}", result.error().to_string()); } } diff --git a/src/terrainlib/store/StorageError.h b/src/terrainlib/store/StorageError.h deleted file mode 100644 index a641456c..00000000 --- a/src/terrainlib/store/StorageError.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "store/CodecError.h" -#include "store/InvalidKey.h" - -namespace store { - -struct AlreadyExists { - std::filesystem::path path; - bool operator==(const AlreadyExists&) const = default; -}; - -struct FilesystemError { - std::filesystem::path path; - std::string operation; - std::error_code error; - bool operator==(const FilesystemError&) const = default; -}; - -template -struct MissingSource { - Key key; - bool operator==(const MissingSource&) const = default; -}; - -template -using LoadError = std::variant, CodecError>; - -template -using SaveError = std::variant, CodecError, AlreadyExists, FilesystemError>; - -template -using FileOperationError = std::variant, FilesystemError>; - -template -using CopyError = std::variant, MissingSource, AlreadyExists, FilesystemError, CodecError>; - -} // namespace store diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h index d0b8e7f1..ef0efa11 100644 --- a/src/terrainlib/store/Traits.h +++ b/src/terrainlib/store/Traits.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace store { @@ -14,4 +15,14 @@ concept HierarchyTraits = requires(typename Traits::Key key) { { Traits::is_valid(key) } -> std::same_as; }; +template +std::string key_to_string(const Key& key) +{ + if constexpr (requires { key.to_string(); }) { + return key.to_string(); + } else { + return to_string(key); + } +} + } // namespace store diff --git a/src/terrainlib/store/describe_error.h b/src/terrainlib/store/describe_error.h deleted file mode 100644 index f280feed..00000000 --- a/src/terrainlib/store/describe_error.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include -#include - -#include "store/IndexFormat.h" -#include "store/OpenError.h" - -namespace store { - -inline std::string describe_error(const CodecError& error) { return error.message; } - -inline std::string describe_error(const IndexFormatError& error) { return error.path.empty() ? error.message : error.path.string() + ": " + error.message; } - -inline std::string describe_error(const FilesystemError& error) { return error.operation + " " + error.path.string() + ": " + error.error.message(); } - -inline std::string describe_error(const UnknownLayout& error) { return "unknown layout: " + error.id; } - -inline std::string describe_error(const AlreadyExists& error) { return "path already exists: " + error.path.string(); } - -template -std::string describe_error(const InvalidKey&) -{ - return "invalid hierarchy key"; -} - -template -std::string describe_error(const MissingSource&) -{ - return "source node is missing"; -} - -template -std::string describe_error(const std::variant& error) -{ - return std::visit([](const auto& value) { return describe_error(value); }, error); -} - -} // namespace store diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h index 07d486cc..0734ac3e 100644 --- a/src/terrainlib/store/open.h +++ b/src/terrainlib/store/open.h @@ -2,46 +2,42 @@ #include #include +#include #include #include #include "store/IndexedStorage.h" -#include "store/OpenError.h" namespace store { template -std::expected, OpenError> open_index( +std::expected, ::Error> open_index( const std::filesystem::path& index_path, const IndexFormat format, const std::string_view expected_payload_class, CodecResolver&& resolve_codec) { using Key = typename Traits::Key; const auto metadata_result = format.read(index_path); if (!metadata_result.has_value()) { - return std::unexpected(OpenError(metadata_result.error())); + return std::unexpected(metadata_result.error()); } IndexMetadata metadata = std::move(metadata_result.value()); if (metadata.payload_class != expected_payload_class) { - return std::unexpected(OpenError(CodecError { - CodecOperation::Resolve, - CodecErrorCategory::UnsupportedCodec, - "unexpected payload class: " + metadata.payload_class, - })); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "unexpected payload class: " + metadata.payload_class)); } for (const auto& [key, status] : metadata.index) { static_cast(status); if (!Traits::is_valid(key)) { - return std::unexpected(OpenError(InvalidKey { key })); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "index contains invalid hierarchy key " + key_to_string(key))); } } const auto mapping = format.mapping_from_id(metadata.layout_id); if (!mapping.has_value()) { - return std::unexpected(OpenError(UnknownLayout { metadata.layout_id })); + return std::unexpected(::Error::make(::Error::Code::Unsupported, "unknown storage layout: " + metadata.layout_id)); } auto codec = resolve_codec(metadata.codec_selector); if (!codec.has_value()) { - return std::unexpected(OpenError(codec.error())); + return std::unexpected(codec.error()); } const std::filesystem::path base_path = index_path.parent_path(); @@ -58,7 +54,7 @@ std::expected, OpenError> } template -std::expected, OpenError> make_storage(const std::filesystem::path& base_path, +std::expected, ::Error> make_storage(const std::filesystem::path& base_path, const IndexFormat format, const path_layout::Mapping mapping, std::string payload_class, @@ -69,11 +65,7 @@ std::expected, OpenError> make_s std::error_code error; std::filesystem::create_directories(base_path, error); if (error) { - return std::unexpected(OpenError(FilesystemError { - base_path, - "create_directories", - error, - })); + return std::unexpected(::Error::make(::Error::Code::Io, "create directories", base_path, error)); } typename Storage::Persistence persistence { diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h index 811988ad..c03cb49b 100644 --- a/src/terrainlib/store/traverse.h +++ b/src/terrainlib/store/traverse.h @@ -22,17 +22,15 @@ struct AlwaysRefine { }; template -std::expected> traverse(const Index& index, +std::expected traverse(const Index& index, VisitFn&& visit, RefineFn&& refine = {}, const typename Traits::Key& root = Traits::root(), const TraversalOrder order = TraversalOrder::DepthFirst) { using Key = typename Traits::Key; - using Error = InvalidKey; - if (!Traits::is_valid(root)) { - return std::unexpected(Error { root }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + key_to_string(root))); } const auto root_status = index.get(root); if (!root_status.has_value()) { @@ -43,8 +41,8 @@ std::expected> traverse(const Index(const Key&)> depth_first; - depth_first = [&](const Key& current) -> std::expected { + std::function(const Key&)> depth_first; + depth_first = [&](const Key& current) -> std::expected { const auto status = index.get(current); if (!status.has_value()) { return std::unexpected(status.error()); diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index a0a2e861..dac5a7bc 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -83,8 +83,7 @@ TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") const auto result = codec.write(node_path, batch); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().operation == store::CodecOperation::Write); - CHECK(result.error().category == store::CodecErrorCategory::Domain); + CHECK(result.error().code() == ::Error::Code::InvalidInput); CHECK_FALSE(std::filesystem::exists(node_paths[0])); CHECK_FALSE(std::filesystem::exists(node_paths[1])); } @@ -100,8 +99,7 @@ TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") const auto result = codec.read(node_path); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().operation == store::CodecOperation::Read); - CHECK(result.error().category == store::CodecErrorCategory::InvalidData); + CHECK(result.error().code() == ::Error::Code::CorruptData); } } @@ -133,12 +131,11 @@ TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[stor CHECK(metadata_codec.value()->paths("node") == std::vector { "node.dagmeta" }); const auto write_result = metadata_codec.value()->write("node", dag::NodeMetadata {}); REQUIRE_FALSE(write_result.has_value()); - CHECK(write_result.error().operation == store::CodecOperation::Write); - CHECK(write_result.error().category == store::CodecErrorCategory::UnsupportedOperation); + CHECK(write_result.error().code() == ::Error::Code::Unsupported); const auto unknown = dag::codec::from_extension(".unknown"); REQUIRE_FALSE(unknown.has_value()); - CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); + CHECK(unknown.error().code() == ::Error::Code::Unsupported); } TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") @@ -221,6 +218,7 @@ TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf const auto built = dag::build_full(input, output, options); REQUIRE_FALSE(built.has_value()); - CHECK(built.error().key == root); + CHECK(built.error().code() == ::Error::Code::CorruptData); + CHECK(built.error().to_string().contains(root.to_string())); CHECK(output.index().empty()); } diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp index 339b540b..db713283 100644 --- a/unittests/sf_builder/storage_finalization.cpp +++ b/unittests/sf_builder/storage_finalization.cpp @@ -40,8 +40,8 @@ TEST_CASE("SF builder finalization retains an invalid written index for diagnosi const auto finalized = sf::finalize_storage(storage); REQUIRE_FALSE(finalized.has_value()); - REQUIRE(std::holds_alternative(finalized.error())); - CHECK(std::get(finalized.error()).key == root); + CHECK(finalized.error().code() == ::Error::Code::CorruptData); + CHECK(finalized.error().to_string().contains(root.to_string())); CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); auto reopened = mesh::storage::open_index(directory.path() / "octree.storeindex"); diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp index 5bd50cc7..6c34aca7 100644 --- a/unittests/sf_merger/storage_boundaries.cpp +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -80,8 +80,8 @@ TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][valida const auto merged = merge_datasets(left, right, output); REQUIRE_FALSE(merged.has_value()); - REQUIRE(std::holds_alternative(merged.error())); - CHECK(std::get(merged.error()).key == root); + CHECK(merged.error().code() == ::Error::Code::CorruptData); + CHECK(merged.error().to_string().contains(root.to_string())); CHECK_FALSE(std::filesystem::exists(output_directory.path() / "octree.storeindex")); } @@ -99,8 +99,8 @@ TEST_CASE("SF merge retains an invalid output index for diagnosis", "[sfmerger][ const auto merged = merge_datasets(left, right, output); REQUIRE_FALSE(merged.has_value()); - REQUIRE(std::holds_alternative(merged.error())); - CHECK(std::get(merged.error()).key == root); + CHECK(merged.error().code() == ::Error::Code::CorruptData); + CHECK(merged.error().to_string().contains(root.to_string())); CHECK(std::filesystem::is_regular_file(output_directory.path() / "octree.storeindex")); } diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index da6ab2d7..28d65412 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -121,10 +121,10 @@ io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes& unc } template -void check_error(const Result& result, const io::envelope::ErrorCode expected) +void check_error(const Result& result, const ::Error::Code expected) { REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code == expected); + CHECK(result.error().code() == expected); } io::envelope::Bytes bytes_from_string(const std::string_view text) @@ -261,7 +261,7 @@ TEST_CASE("CRC-32C protects independently compressed data") corrupted.front() ^= std::byte { 0x01 }; check_error(io::envelope::checked_decompress( corrupted, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, compressed->checksum), - io::envelope::ErrorCode::ChecksumMismatch); + ::Error::Code::CorruptData); } SECTION("with zstd compression") @@ -281,16 +281,16 @@ TEST_CASE("CRC-32C protects independently compressed data") wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; check_error( io::envelope::checked_decompress(compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c, wrong_checksum), - io::envelope::ErrorCode::ChecksumMismatch); + ::Error::Code::CorruptData); } } SECTION("malformed checksum") { check_error(io::envelope::checked_decompress(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, "E3069283"), - io::envelope::ErrorCode::ChecksumMismatch); + ::Error::Code::CorruptData); check_error(io::envelope::checked_decompress(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, {}), - io::envelope::ErrorCode::ChecksumMismatch); + ::Error::Code::CorruptData); } } @@ -318,7 +318,7 @@ TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") auto corrupted = envelope; corrupted.compressed_data.front() ^= std::byte { 0x01 }; - check_error(io::envelope::deserialize(encode_envelope(corrupted)), io::envelope::ErrorCode::ChecksumMismatch); + check_error(io::envelope::deserialize(encode_envelope(corrupted)), ::Error::Code::CorruptData); } SECTION("older version") @@ -364,11 +364,11 @@ TEST_CASE("Checked compression round trips and validates its checksum") auto corrupted = compressed->compressed_data; corrupted.back() ^= std::byte { 0x01 }; check_error(io::envelope::checked_decompress(corrupted, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}), - io::envelope::ErrorCode::ChecksumMismatch); + ::Error::Code::CorruptData); check_error(io::envelope::checked_decompress( compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size() - 1), - io::envelope::ErrorCode::SizeLimitExceeded); + ::Error::Code::ResourceExhausted); } } @@ -386,7 +386,7 @@ TEST_CASE("Checked decompression uses its maximum when the format omits the cont check_error(io::envelope::checked_decompress( compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size() - 1), - io::envelope::ErrorCode::SizeLimitExceeded); + ::Error::Code::ResourceExhausted); } } @@ -400,49 +400,49 @@ TEST_CASE("Envelope rejects incompatible metadata") { auto envelope = decode_envelope(*serialized); ++envelope.magic; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::InvalidMagic); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("class name") { auto envelope = decode_envelope(*serialized); envelope.class_name = "other.Payload"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::WrongClassName); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("class version") { auto envelope = decode_envelope(*serialized); envelope.class_version = 99; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::UnsupportedClassVersion); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::Unsupported); } SECTION("checksum algorithm") { auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = static_cast(99); - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::UnsupportedChecksumAlgorithm); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::Unsupported); } SECTION("compression algorithm") { auto envelope = decode_envelope(*serialized); envelope.compression_algorithm = static_cast(99); - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::UnsupportedCompressionAlgorithm); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::Unsupported); } SECTION("algorithm combination") { auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::None; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::InvalidAlgorithmCombination); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("external checksum") { auto envelope = decode_envelope(*serialized); envelope.checksum = "not used by zstd"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::InvalidAlgorithmCombination); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("malformed CRC-32C checksum") @@ -450,7 +450,7 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; envelope.checksum = "1234"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::ChecksumMismatch); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("incorrect CRC-32C checksum") @@ -458,42 +458,46 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; envelope.checksum = "00000000"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::ChecksumMismatch); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("uncompressed size is smaller than the payload") { auto envelope = decode_envelope(*serialized); --envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); + const auto result = io::envelope::deserialize(encode_envelope(envelope)); + check_error(result, ::Error::Code::CorruptData); + REQUIRE_FALSE(result.error().frames().empty()); + CHECK(result.error().frames().back().original_code == ::Error::Code::ResourceExhausted); + CHECK(result.error().frames().back().new_code == ::Error::Code::CorruptData); } SECTION("uncompressed size is larger than the payload") { auto envelope = decode_envelope(*serialized); ++envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("zero uncompressed size does not mean unspecified") { auto envelope = decode_envelope(*serialized); envelope.uncompressed_size = 0; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("uncompressed size exceeds the hard limit") { auto envelope = decode_envelope(*serialized); envelope.uncompressed_size = io::envelope::default_max_decompressed_size + 1; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::SizeLimitExceeded); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::ResourceExhausted); } SECTION("uncompressed size exceeds a caller limit") { auto envelope = decode_envelope(*serialized); check_error(io::envelope::deserialize(encode_envelope(envelope), static_cast(envelope.uncompressed_size - 1)), - io::envelope::ErrorCode::SizeLimitExceeded); + ::Error::Code::ResourceExhausted); } } @@ -507,14 +511,14 @@ TEST_CASE("Envelope validates the size of uncompressed data") { auto envelope = decode_envelope(*serialized); --envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } SECTION("declared size is larger") { auto envelope = decode_envelope(*serialized); ++envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } } @@ -523,7 +527,7 @@ TEST_CASE("Envelope reports malformed serialized data") SECTION("envelope") { const io::envelope::Bytes malformed { std::byte { 0x01 }, std::byte { 0x02 } }; - check_error(io::envelope::deserialize(malformed), io::envelope::ErrorCode::DeserializationFailed); + check_error(io::envelope::deserialize(malformed), ::Error::Code::CorruptData); } SECTION("payload") @@ -538,6 +542,6 @@ TEST_CASE("Envelope reports malformed serialized data") .uncompressed_size = 1, .compressed_data = { std::byte { 0x01 } }, }; - check_error(io::envelope::deserialize(encode_envelope(envelope)), io::envelope::ErrorCode::DeserializationFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); } } diff --git a/unittests/terrainlib/io_conversion.cpp b/unittests/terrainlib/io_conversion.cpp index 5fd818c8..479acfcb 100644 --- a/unittests/terrainlib/io_conversion.cpp +++ b/unittests/terrainlib/io_conversion.cpp @@ -78,7 +78,7 @@ TEST_CASE("io::conversion reports invalid inputs") { const auto result = io::conversion::to_raster(cv::Mat {}); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code == io::conversion::ErrorCode::EmptyInput); + CHECK(result.error().code() == ::Error::Code::InvalidInput); } SECTION("multidimensional matrix") @@ -87,7 +87,7 @@ TEST_CASE("io::conversion reports invalid inputs") const cv::Mat source(static_cast(sizes.size()), sizes.data(), CV_8UC1); const auto result = io::conversion::to_raster(source); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code == io::conversion::ErrorCode::UnsupportedDimensions); + CHECK(result.error().code() == ::Error::Code::Unsupported); } SECTION("pixel type mismatch") @@ -95,12 +95,7 @@ TEST_CASE("io::conversion reports invalid inputs") const cv::Mat source(2, 2, CV_8UC3); const auto result = io::conversion::to_raster(source); REQUIRE_FALSE(result.has_value()); - const io::conversion::Error expected { - io::conversion::ErrorCode::PixelTypeMismatch, - CV_8UC1, - CV_8UC3, - }; - CHECK(result.error() == expected); + CHECK(result.error().code() == ::Error::Code::InvalidInput); } SECTION("unsupported pixel") @@ -111,13 +106,13 @@ TEST_CASE("io::conversion reports invalid inputs") const cv::Mat source(2, 2, CV_8UC1); const auto result = io::conversion::to_raster(source); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code == io::conversion::ErrorCode::UnsupportedPixelType); + CHECK(result.error().code() == ::Error::Code::Unsupported); } SECTION("empty raster") { const auto result = io::conversion::to_mat(radix::Raster {}); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code == io::conversion::ErrorCode::EmptyInput); + CHECK(result.error().code() == ::Error::Code::InvalidInput); } } diff --git a/unittests/terrainlib/sf_validate_index.cpp b/unittests/terrainlib/sf_validate_index.cpp index 1afd61e3..10f18d55 100644 --- a/unittests/terrainlib/sf_validate_index.cpp +++ b/unittests/terrainlib/sf_validate_index.cpp @@ -22,5 +22,6 @@ TEST_CASE("Structura Fundamentalis validation reports the first Inner node", "[s const auto result = sf::validate_index(index); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().key == root); + CHECK(result.error().code() == ::Error::Code::CorruptData); + CHECK(result.error().to_string().contains(root.to_string())); } diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp index 45ef42df..2c26d8c4 100644 --- a/unittests/terrainlib/store_codec.cpp +++ b/unittests/terrainlib/store_codec.cpp @@ -30,7 +30,7 @@ class WriteOnlyCodec final : public store::Codec { { return { add_extension(node_path, ".out") }; } - std::expected write(const std::filesystem::path&, const int&) const override + std::expected write(const std::filesystem::path&, const int&) const override { ++writes; return {}; @@ -82,13 +82,11 @@ TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec const auto read = codec.read(path); REQUIRE_FALSE(read.has_value()); - CHECK(read.error().operation == store::CodecOperation::Read); - CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); + CHECK(read.error().code() == ::Error::Code::Unsupported); const auto write = codec.write(path, 42); REQUIRE_FALSE(write.has_value()); - CHECK(write.error().operation == store::CodecOperation::Write); - CHECK(write.error().category == store::CodecErrorCategory::UnsupportedOperation); + CHECK(write.error().code() == ::Error::Code::Unsupported); } TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") @@ -98,7 +96,7 @@ TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][cod CHECK(codec.writes == 1); const auto read = codec.read("node"); REQUIRE_FALSE(read.has_value()); - CHECK(read.error().category == store::CodecErrorCategory::UnsupportedOperation); + CHECK(read.error().code() == ::Error::Code::Unsupported); } TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") @@ -119,6 +117,5 @@ TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") const auto result = codec.write(node_path, triangle_mesh(0.0)); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().operation == store::CodecOperation::Write); - CHECK(result.error().category == store::CodecErrorCategory::Domain); + CHECK(result.error().code() == ::Error::Code::Internal); } diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index c2490b28..5dd94943 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -182,8 +182,7 @@ TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][op const auto unknown = mesh::codec::from_extension(".unknown"); REQUIRE_FALSE(unknown.has_value()); - CHECK(unknown.error().category == store::CodecErrorCategory::UnsupportedCodec); - CHECK(unknown.error().operation == store::CodecOperation::Resolve); + CHECK(unknown.error().code() == ::Error::Code::Unsupported); } TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store][open]") @@ -200,8 +199,7 @@ TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store] const auto result = mesh::storage::open_index(index_path); REQUIRE_FALSE(result.has_value()); - REQUIRE(std::holds_alternative(result.error())); - CHECK(std::get(result.error()).category == store::CodecErrorCategory::UnsupportedCodec); + CHECK(result.error().code() == ::Error::Code::Unsupported); } TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") @@ -219,23 +217,21 @@ TEST_CASE("octree opening retains index failures without directory fallback", "[ const auto unknown_layout = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_layout.has_value()); - REQUIRE(std::holds_alternative(unknown_layout.error())); - CHECK(std::get(unknown_layout.error()).id == "unknown-layout"); + CHECK(unknown_layout.error().code() == ::Error::Code::Unsupported); + CHECK(unknown_layout.error().to_string().contains("unknown-layout")); index_file.layout_id = "flat"; index_file.codec_selector = ".unknown"; REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); const auto unknown_codec = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_codec.has_value()); - REQUIRE(std::holds_alternative(unknown_codec.error())); - CHECK(std::get(unknown_codec.error()).category == store::CodecErrorCategory::UnsupportedCodec); + CHECK(unknown_codec.error().code() == ::Error::Code::Unsupported); const std::array malformed_bytes { 0xff, 0x00, 0x01 }; REQUIRE(io::write_bytes_to_path(malformed_bytes, index_path).has_value()); const auto malformed = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(malformed.has_value()); - REQUIRE(std::holds_alternative(malformed.error())); - CHECK(std::get(malformed.error()).category == store::IndexFormatErrorCategory::Malformed); + CHECK(malformed.error().code() == ::Error::Code::CorruptData); const auto independent_metadata = octree::storage::read_store_metadata(directory.path()); REQUIRE(independent_metadata.has_value()); diff --git a/unittests/terrainlib/store_index.cpp b/unittests/terrainlib/store_index.cpp index 966ef4d5..58540e8e 100644 --- a/unittests/terrainlib/store_index.cpp +++ b/unittests/terrainlib/store_index.cpp @@ -71,10 +71,10 @@ TEST_CASE("shared index rejects invalid keys", "[store][index][raster]") store::Index index; const Key invalid { 4, { 16, 0 } }; - CHECK(index.get(invalid) == std::unexpected(store::InvalidKey { invalid })); - CHECK(index.add(invalid) == std::unexpected(store::InvalidKey { invalid })); - CHECK(index.remove(invalid) == std::unexpected(store::InvalidKey { invalid })); - CHECK(index.is_present(invalid) == std::unexpected(store::InvalidKey { invalid })); - CHECK(index.is_absent(invalid) == std::unexpected(store::InvalidKey { invalid })); - CHECK(index.is(store::NodeStatus::Leaf, invalid) == std::unexpected(store::InvalidKey { invalid })); + CHECK(index.get(invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(index.add(invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(index.remove(invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(index.is_present(invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(index.is_absent(invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(index.is(store::NodeStatus::Leaf, invalid).error().code() == ::Error::Code::InvalidInput); } diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index 15349238..b5e860ff 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -59,30 +58,26 @@ store::path_layout::Mapping test_mapping read_int(const std::filesystem::path& path) +std::expected read_int(const std::filesystem::path& path) { auto bytes = io::read_bytes_from_path(path); - if (!bytes || bytes->size() != sizeof(int)) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::Io, - "test integer read failed", - }); + if (!bytes) { + return std::unexpected(std::move(bytes).error().with_context("read test integer")); + } + if (bytes->size() != sizeof(int)) { + return std::unexpected(::Error::make(::Error::Code::CorruptData, "test integer has the wrong byte count")); } int value; std::memcpy(&value, bytes->data(), sizeof(value)); return value; } -std::expected write_int(const int value, const std::filesystem::path& path) +std::expected write_int(const int value, const std::filesystem::path& path) { const auto bytes = std::span(reinterpret_cast(&value), sizeof(value)); - if (!io::write_bytes_to_path(bytes, path)) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Io, - "test integer write failed", - }); + auto written = io::write_bytes_to_path(bytes, path); + if (!written) { + return std::unexpected(std::move(written).error().with_context("write test integer")); } return {}; } @@ -94,9 +89,9 @@ class IntCodec final : public store::Codec { return { add_extension(node_path, ".data") }; } - std::expected read(const std::filesystem::path& node_path) const override { return read_int(paths(node_path).front()); } + std::expected read(const std::filesystem::path& node_path) const override { return read_int(paths(node_path).front()); } - std::expected write(const std::filesystem::path& node_path, const int& value) const override + std::expected write(const std::filesystem::path& node_path, const int& value) const override { return write_int(value, paths(node_path).front()); } @@ -116,28 +111,20 @@ class MultiFileCodec final : public store::Codec { return { add_extension(node_path, ".data"), add_extension(node_path, ".metadata") }; } - std::expected read(const std::filesystem::path& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { - const auto value = read_int(paths(node_path).front()); + auto value = read_int(paths(node_path).front()); if (!value) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::Io, - "multi-file read failed", - }); + return std::unexpected(std::move(value).error().with_context("read multi-file test value")); } return value.value(); } - std::expected write(const std::filesystem::path& node_path, const int& value) const override + std::expected write(const std::filesystem::path& node_path, const int& value) const override { const auto node_paths = paths(node_path); if (!write_int(value, node_paths[0]) || !write_int(value + 1, node_paths[1])) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Io, - "multi-file write failed", - }); + return std::unexpected(::Error::make(::Error::Code::Io, "multi-file test write failed")); } return {}; } @@ -155,41 +142,25 @@ class ConfigurableCodec final : public store::Codec { return { add_extension(node_path, extension) }; } - std::expected read(const std::filesystem::path& node_path) const override + std::expected read(const std::filesystem::path& node_path) const override { if (fail_read) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::Domain, - "injected decode failure", - }); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "injected decode failure")); } - const auto value = read_int(paths(node_path).front()); + auto value = read_int(paths(node_path).front()); if (!value) { - return std::unexpected(store::CodecError { - store::CodecOperation::Read, - store::CodecErrorCategory::Io, - "configurable read failed", - }); + return std::unexpected(std::move(value).error().with_context("read configurable test value")); } return value.value(); } - std::expected write(const std::filesystem::path& node_path, const int& value) const override + std::expected write(const std::filesystem::path& node_path, const int& value) const override { if (fail_write) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Domain, - "injected encode failure", - }); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "injected encode failure")); } if (!write_int(value, paths(node_path).front())) { - return std::unexpected(store::CodecError { - store::CodecOperation::Write, - store::CodecErrorCategory::Io, - "configurable write failed", - }); + return std::unexpected(::Error::make(::Error::Code::Io, "configurable test write failed")); } return {}; } @@ -201,16 +172,12 @@ class ConfigurableCodec final : public store::Codec { std::atomic_uint32_t index_writes = 0; -std::expected, store::IndexFormatError> unused_index_read(const std::filesystem::path& path) +std::expected, ::Error> unused_index_read(const std::filesystem::path& path) { - return std::unexpected(store::IndexFormatError { - store::IndexFormatErrorCategory::Open, - path, - "not used", - }); + return std::unexpected(::Error::make(::Error::Code::NotFound, "read unused test index from", path)); } -std::expected count_index_write(const std::filesystem::path&, const store::IndexMetadata&) +std::expected count_index_write(const std::filesystem::path&, const store::IndexMetadata&) { ++index_writes; return {}; @@ -260,7 +227,7 @@ TEST_CASE("byte writes report directory creation errors", "[io][bytes]") const auto result = io::write_bytes_to_path(std::span {}, blocker / "payload"); REQUIRE_FALSE(result.has_value()); - CHECK(result.error() == io::Error(io::Error::CreateDirectories)); + CHECK(result.error().code() == ::Error::Code::Io); } TEMPLATE_TEST_CASE("shared raw storage has no hidden octree dependency", "[store][storage]", octree::StoreTraits, raster_store::StoreTraits) @@ -285,10 +252,10 @@ TEST_CASE("shared storage rejects invalid 2D keys", "[store][storage]") auto storage = make_storage(directory.path()); const radix::tile::Id invalid { 3, { 8, 0 } }; - CHECK(std::holds_alternative>(storage.load(invalid).error())); - CHECK(std::holds_alternative>(storage.save(invalid, 1).error())); - CHECK(std::holds_alternative>(storage.has(invalid).error())); - CHECK(std::holds_alternative>(storage.remove(invalid).error())); + CHECK(storage.load(invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(storage.save(invalid, 1).error().code() == ::Error::Code::InvalidInput); + CHECK(storage.has(invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(storage.remove(invalid).error().code() == ::Error::Code::InvalidInput); } TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") @@ -300,7 +267,7 @@ TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") REQUIRE(storage.save(root, 1).has_value()); const auto rejected = storage.save(root, 2); REQUIRE_FALSE(rejected.has_value()); - CHECK(std::holds_alternative(rejected.error())); + CHECK(rejected.error().code() == ::Error::Code::AlreadyExists); CHECK(storage.load(root).value() == 1); storage.settings().allow_overwrite = true; @@ -429,7 +396,7 @@ TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") const auto rejected = target.copy_from(root, source); REQUIRE_FALSE(rejected.has_value()); - CHECK(std::holds_alternative(rejected.error())); + CHECK(rejected.error().code() == ::Error::Code::AlreadyExists); CHECK(target.load(root).value() == 1); target.settings().allow_overwrite = true; @@ -456,7 +423,7 @@ TEST_CASE("failed multi-file overwrite leaves the target node unindexed", "[stor const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - CHECK(std::holds_alternative(copied.error())); + CHECK(copied.error().code() == ::Error::Code::Io); CHECK_FALSE(target.has(root).value()); CHECK_FALSE(target.index().get(root).value().has_value()); CHECK(std::filesystem::equivalent(source.paths(root)->front(), target_paths.front())); @@ -478,7 +445,7 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - CHECK(std::holds_alternative>(copied.error())); + CHECK(copied.error().code() == ::Error::Code::NotFound); CHECK(target.has(root).value()); CHECK(target.load(root).value() == 7); } @@ -498,8 +465,7 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - REQUIRE(std::holds_alternative(copied.error())); - CHECK(std::get(copied.error()).operation == store::CodecOperation::Read); + CHECK(copied.error().code() == ::Error::Code::CorruptData); CHECK(target.has(root).value()); CHECK(target.load(root).value() == 7); } @@ -519,8 +485,7 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - REQUIRE(std::holds_alternative(copied.error())); - CHECK(std::get(copied.error()).operation == store::CodecOperation::Write); + CHECK(copied.error().code() == ::Error::Code::InvalidInput); CHECK_FALSE(target.has(root).value()); } } diff --git a/unittests/terrainlib/store_traverse.cpp b/unittests/terrainlib/store_traverse.cpp index 9eba2bc0..ec70e8fa 100644 --- a/unittests/terrainlib/store_traverse.cpp +++ b/unittests/terrainlib/store_traverse.cpp @@ -50,5 +50,6 @@ TEST_CASE("shared traversal rejects an invalid explicit root", "[store][traverse store::Index index; const auto result = store::traverse(index, [](const Key&, const store::NodeStatus) { }, store::AlwaysRefine {}, invalid); - CHECK(result == std::unexpected(store::InvalidKey { invalid })); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == ::Error::Code::InvalidInput); } From 05788d646b937609f040174d4155d1f1da8c9573 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 27 Aug 2026 06:23:14 +0200 Subject: [PATCH 29/43] Simplify unified error propagation --- src/dag_builder/build.cpp | 10 ++-- src/dag_builder/codec/Dag.h | 44 +++++++------- src/sf_merger/NodeWriter.h | 16 ++--- src/sf_merger/cut.h | 24 ++++---- src/sf_merger/merge.h | 36 +++++------- src/terrainlib/Error.cpp | 14 +++-- src/terrainlib/Error.h | 2 - src/terrainlib/io/compression.cpp | 8 +-- src/terrainlib/mesh/codec/Gltf.h | 38 +++++++++--- src/terrainlib/mesh/codec/SfMesh.h | 16 +++-- src/terrainlib/octree/StoreTraits.h | 2 + src/terrainlib/octree/storage/IndexFile.h | 11 +--- src/terrainlib/octree/storage/open_runtime.h | 6 +- src/terrainlib/raster_store/StoreTraits.h | 2 + src/terrainlib/sf/finalize_storage.h | 10 +--- src/terrainlib/store/Index.h | 38 ++++++------ src/terrainlib/store/RawStorage.h | 39 +++++-------- src/terrainlib/store/Storage.h | 61 +++++++++----------- src/terrainlib/store/ThreadSafeStorage.h | 6 +- src/terrainlib/store/Traits.h | 15 ++--- src/terrainlib/store/open.h | 8 +-- src/terrainlib/store/traverse.h | 17 +++--- unittests/terrainlib/envelope.cpp | 4 +- 23 files changed, 210 insertions(+), 217 deletions(-) diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index f2e9d090..1c23ad37 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -457,7 +457,7 @@ std::expected>, ::Error> gather_relevant_inp const auto start = space.find_smallest_node_encompassing_bounds(root_bounds) .value_or(octree::Id::root()); std::vector> result(octree::Id::max_level() + 1); - const auto traversal = store::traverse( + auto traversal = store::traverse( index, [&](const octree::Id &id, const store::NodeStatus status) { if (status == store::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { @@ -469,7 +469,7 @@ std::expected>, ::Error> gather_relevant_inp }, start); if (!traversal.has_value()) { - return std::unexpected(traversal.error()); + return std::unexpected(std::move(traversal).error()); } return result; } @@ -588,9 +588,9 @@ std::expected build_levels( dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range) { - const auto validation = sf::validate_index(input_storage.index()); + auto validation = sf::validate_index(input_storage.index()); if (!validation.has_value()) { - return std::unexpected(validation.error()); + return validation; } const octree::OddLevelShifted shifted_space = octree::OddLevelShifted::earth(); const octree::Space space = octree::Space::earth(); @@ -598,7 +598,7 @@ std::expected build_levels( const auto root_bounds = shifted_space.get_node_bounds_with_children(root_node); auto input_by_level_result = gather_relevant_input_leaves(input_storage.index(), space, root_bounds); if (!input_by_level_result.has_value()) { - return std::unexpected(input_by_level_result.error()); + return std::unexpected(std::move(input_by_level_result).error()); } auto input_by_level = std::move(input_by_level_result.value()); diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index a1d57060..6f7df016 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -11,11 +11,6 @@ namespace dag::codec { -inline ::Error file_error(std::string operation, ::Error error) -{ - return std::move(error).with_context(std::move(operation)); -} - inline std::expected validate_batch(const dag::ClusterBatch& batch, const ::Error::Code code) { if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { @@ -39,54 +34,61 @@ class Dag final : public store::Codec { const auto node_paths = paths(node_path); auto clustering_payload = io::envelope::read_from_path(node_paths[0]); if (!clustering_payload) { - return std::unexpected(file_error("read DAG clustering", std::move(clustering_payload).error())); + return std::unexpected(std::move(clustering_payload).error()); } auto metadata_payload = io::envelope::read_from_path(node_paths[1]); if (!metadata_payload) { - return std::unexpected(file_error("read DAG metadata", std::move(metadata_payload).error())); + return std::unexpected(std::move(metadata_payload).error()); } auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); if (!clustering) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, clustering.error())); + return std::unexpected(::Error::make( + ::Error::Code::CorruptData, "invalid DAG clustering in \"" + node_paths[0].string() + "\": " + clustering.error())); } auto metadata = dag::format::decode_metadata(std::move(*metadata_payload)); if (!metadata) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, metadata.error())); + return std::unexpected(::Error::make( + ::Error::Code::CorruptData, "invalid DAG metadata in \"" + node_paths[1].string() + "\": " + metadata.error())); } dag::ClusterBatch batch { .metadata = std::move(*metadata), .clustering = std::move(*clustering), }; if (auto valid = validate_batch(batch, ::Error::Code::CorruptData); !valid) { - return std::unexpected(valid.error()); + return std::unexpected(std::move(valid).error().with_context( + "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\"")); } return batch; } std::expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override { + const auto node_paths = paths(node_path); if (auto valid = validate_batch(batch, ::Error::Code::InvalidInput); !valid) { - return valid; + return std::unexpected(std::move(valid).error().with_context( + "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\"")); } auto clustering = dag::format::encode_clustering(batch.clustering); if (!clustering) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, clustering.error())); + return std::unexpected(::Error::make( + ::Error::Code::InvalidInput, "cannot encode DAG clustering for \"" + node_paths[0].string() + "\": " + clustering.error())); } const dag::format::NodeMetadata metadata = dag::format::encode_metadata(batch.metadata); if (auto valid = dag::format::validate(*clustering); !valid) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, valid.error())); + return std::unexpected(::Error::make( + ::Error::Code::InvalidInput, "invalid DAG clustering for \"" + node_paths[0].string() + "\": " + valid.error())); } if (auto valid = dag::format::validate(metadata); !valid) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, valid.error())); + return std::unexpected(::Error::make( + ::Error::Code::InvalidInput, "invalid DAG metadata for \"" + node_paths[1].string() + "\": " + valid.error())); } - const auto node_paths = paths(node_path); auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); if (!data_written) { - return std::unexpected(file_error("write DAG clustering", std::move(data_written).error())); + return data_written; } auto metadata_written = io::envelope::write_to_path(metadata, node_paths[1]); if (!metadata_written) { - return std::unexpected(file_error("write DAG metadata", std::move(metadata_written).error())); + return metadata_written; } return {}; } @@ -101,13 +103,15 @@ class MetadataView final : public store::Codec { std::expected read(const std::filesystem::path& node_path) const override { - auto payload = io::envelope::read_from_path(paths(node_path).front()); + const std::filesystem::path path = paths(node_path).front(); + auto payload = io::envelope::read_from_path(path); if (!payload) { - return std::unexpected(file_error("read DAG metadata", std::move(payload).error())); + return std::unexpected(std::move(payload).error()); } auto metadata = dag::format::decode_metadata(std::move(*payload)); if (!metadata) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, metadata.error())); + return std::unexpected( + ::Error::make(::Error::Code::CorruptData, "invalid DAG metadata in \"" + path.string() + "\": " + metadata.error())); } return std::move(*metadata); } diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 4f06f145..807331c3 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -25,13 +25,13 @@ class NodeWriter { const octree::Id &id, const SimpleMesh &mesh) { mesh::validate(mesh); - const auto save_result = this->_storage.save(id, mesh); + auto save_result = this->_storage.save(id, mesh); if (!save_result.has_value()) { return save_result; } - const auto path_result = this->_storage.path_for(id); + auto path_result = this->_storage.path_for(id); if (!path_result.has_value()) { - return std::unexpected(path_result.error()); + return std::unexpected(std::move(path_result).error()); } auto p = path_result.value(); // change extension to .png @@ -45,8 +45,8 @@ class NodeWriter { std::expected copy_subtree_to_output( const octree::Id &id, const NodeLoader &loader) { - std::optional<::Error> error; - const auto traversal = store::traverse( + std::optional<::Error> error = std::nullopt; + auto traversal = store::traverse( loader.storage().index(), [&](const octree::Id &child_id, const store::NodeStatus &status) { if (error.has_value()) { @@ -57,15 +57,15 @@ class NodeWriter { } DEBUG_ASSERT(status == store::NodeStatus::Leaf); - const auto result = this->_storage.copy_from(child_id, loader.storage()); + auto result = this->_storage.copy_from(child_id, loader.storage()); if (!result.has_value()) { - error = result.error(); + error = std::move(result).error(); } }, [&](const octree::Id &) { return !error.has_value(); }, id); if (!traversal.has_value()) { - return std::unexpected(traversal.error()); + return traversal; } if (error.has_value()) { return std::unexpected(std::move(error.value())); diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index d74537e0..eb11abea 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -42,18 +42,18 @@ inline std::expected cut_leaf_node( const Cow clipped = clip_on_mask(mesh, mask, ctx.keep_inside); if (clipped.is_ref()) { LOG_TRACE("Mesh was fully inside the mask"); - const auto copy_result = ctx.output.copy_from(id, ctx.input); + auto copy_result = ctx.output.copy_from(id, ctx.input); if (!copy_result.has_value()) { - return std::unexpected(copy_result.error()); + return copy_result; } } else { const SimpleMesh &clipped_mesh = clipped; if (!clipped_mesh.is_empty()) { LOG_TRACE("Mesh was clipped from {} vertices and {} triangles to {} vertices and {} triangles", mesh.vertex_count(), mesh.face_count(), clipped_mesh.vertex_count(), clipped_mesh.face_count()); - const auto save_result = ctx.output.save(id, clipped_mesh); + auto save_result = ctx.output.save(id, clipped_mesh); if (!save_result.has_value()) { - return std::unexpected(save_result.error()); + return save_result; } } else { LOG_TRACE("Mesh was fully outside the mask"); @@ -77,7 +77,7 @@ inline std::expected cut_virtual_node( const auto child_bounds = geometry::pad_bounds_relative(ctx.space.get_node_bounds(child_id), 0.125); LOG_TRACE("Clipping mask for {}", child_id); const MeshMask child_mask = clip_mask_on_bounds(mask, child_bounds); - const auto child_result = cut_node(ctx, child_id, child_mask); + auto child_result = cut_node(ctx, child_id, child_mask); if (!child_result.has_value()) { return child_result; } @@ -127,20 +127,16 @@ inline std::expected cut_dataset( const MeshMask& mask, mesh::storage::Storage &output, const bool keep_inside) { - const auto validation = sf::validate_index(input.index()); + auto validation = sf::validate_index(input.index()); if (!validation.has_value()) { - return std::unexpected(validation.error()); + return validation; } Context ctx(input, output, octree::Space::earth(), keep_inside); - const auto cut_result = cut_node(ctx, octree::Id::root(), mask); + auto cut_result = cut_node(ctx, octree::Id::root(), mask); if (!cut_result.has_value()) { return cut_result; } - const auto finalization = sf::finalize_storage(output); - if (!finalization.has_value()) { - return std::unexpected(finalization.error()); - } - return {}; + return sf::finalize_storage(output); } inline std::expected cut_dataset( @@ -156,7 +152,7 @@ inline std::expected cut_dataset( options.preferred_extension = ".glb"; auto output_result = mesh::storage::open_folder_indexed(output_path, std::move(options)); if (!output_result.has_value()) { - return std::unexpected(output_result.error()); + return std::unexpected(std::move(output_result).error()); } mesh::storage::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index 47b97f2f..4d2b7380 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -65,9 +65,9 @@ class Merger { const Context& ctx ) { LOG_DEBUG("[{}] Start merging (left = {}, right = {})", id, left_status, right_status); - const auto has_result = this->_output.has_node(id); + auto has_result = this->_output.has_node(id); if (!has_result.has_value()) { - return std::unexpected(has_result.error()); + return std::unexpected(std::move(has_result).error()); } if (has_result.value()) { LOG_DEBUG("[{}] Already merged, skipping...", id); @@ -82,7 +82,7 @@ class Merger { DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); for (const auto &child_id : children) { - const auto child_result = this->merge_node(child_id, result.context); + auto child_result = this->merge_node(child_id, result.context); if (!child_result.has_value()) { return child_result; } @@ -94,33 +94,33 @@ class Merger { if (result.source == merge::Source::Left && left_status == Status::Missing) { auto mesh_opt = this->_left.load_node(id); if (mesh_opt.has_value()) { - const auto write_result = this->_output.write_node(id, *mesh_opt); + auto write_result = this->_output.write_node(id, *mesh_opt); if (!write_result.has_value()) { - return std::unexpected(write_result.error()); + return write_result; } } } else if (result.source == merge::Source::Right && right_status == Status::Missing) { auto mesh_opt = this->_right.load_node(id); if (mesh_opt.has_value()) { - const auto write_result = this->_output.write_node(id, *mesh_opt); + auto write_result = this->_output.write_node(id, *mesh_opt); if (!write_result.has_value()) { - return std::unexpected(write_result.error()); + return write_result; } } } else { - const auto copy_result = this->_output.copy_subtree_to_output( + auto copy_result = this->_output.copy_subtree_to_output( id, result.source == merge::Source::Left ? this->_left : this->_right); if (!copy_result.has_value()) { - return std::unexpected(copy_result.error()); + return copy_result; } } return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was merged", id); - const auto write_result = this->_output.write_node(id, result.mesh); + auto write_result = this->_output.write_node(id, result.mesh); if (!write_result.has_value()) { - return std::unexpected(write_result.error()); + return write_result; } return {}; } else if constexpr (std::is_same_v) { @@ -187,9 +187,9 @@ inline std::expected merge_datasets( mesh::storage::Storage &output_dataset, const std::optional mask = std::nullopt) { for (const mesh::storage::IndexedStorage *input : {&left_dataset, &right_dataset}) { - const auto validation = sf::validate_index(input->index()); + auto validation = sf::validate_index(input->index()); if (!validation.has_value()) { - return std::unexpected(validation.error()); + return validation; } } @@ -207,22 +207,18 @@ inline std::expected merge_datasets( if (mask.has_value()) { merge::visitor::Masked visitor {mask.value(), space}; Merger merger(visitor, left, right, output); - const auto result = merger.merge_root(); + auto result = merger.merge_root(); if (!result.has_value()) { return result; } } else { merge::visitor::Simple visitor; Merger merger(visitor, left, right, output); - const auto result = merger.merge_root(); + auto result = merger.merge_root(); if (!result.has_value()) { return result; } } - const auto finalization = sf::finalize_storage(output_dataset); - if (!finalization.has_value()) { - return std::unexpected(finalization.error()); - } - return {}; + return sf::finalize_storage(output_dataset); } diff --git a/src/terrainlib/Error.cpp b/src/terrainlib/Error.cpp index ad4892bf..c63b1477 100644 --- a/src/terrainlib/Error.cpp +++ b/src/terrainlib/Error.cpp @@ -86,25 +86,27 @@ Error Error::make(const Code code, Error Error::with_context(std::string message, const std::source_location location) && { - m_frames.push_back(Frame { + Error result = std::move(*this); + result.m_frames.push_back(Frame { std::move(message), location, std::nullopt, std::nullopt, }); - return std::move(*this); + return result; } Error Error::reclassified(const Code code, std::string message, const std::source_location location) && { - m_frames.push_back(Frame { + Error result = std::move(*this); + result.m_frames.push_back(Frame { std::move(message), location, - m_code, + result.m_code, code, }); - m_code = code; - return std::move(*this); + result.m_code = code; + return result; } std::string Error::to_string() const diff --git a/src/terrainlib/Error.h b/src/terrainlib/Error.h index e9ad15da..0e2486ed 100644 --- a/src/terrainlib/Error.h +++ b/src/terrainlib/Error.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -68,7 +67,6 @@ class Error { std::source_location location = std::source_location::current()) &&; Code code() const { return m_code; } - std::span frames() const { return m_frames; } std::string to_string() const; private: diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index 460ff976..2c32139e 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -86,8 +86,8 @@ namespace { std::expected compress_with_checksum( const Bytes& uncompressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) { - if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { - return std::unexpected(validation.error()); + if (auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return std::unexpected(std::move(validation).error()); } if (uncompressed_data.size() > default_max_decompressed_size) { return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "uncompressed data exceeds the compression size limit")); @@ -129,8 +129,8 @@ std::expected checked_decompress(const Bytes& compressed_data, const std::string_view checksum, const std::size_t max_decompressed_size) { - if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { - return std::unexpected(validation.error()); + if (auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return std::unexpected(std::move(validation).error()); } if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { return std::unexpected(::Error::make(::Error::Code::InvalidInput, "a checksum was supplied for an algorithm that does not accept one")); diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index 3b603ff8..ac740243 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -31,26 +32,49 @@ class Gltf final : public store::Codec { std::expected read(const std::filesystem::path& node_path) const override { - const auto result = mesh::io::gltf::load_from_path(paths(node_path).front()); + const std::filesystem::path path = paths(node_path).front(); + const auto result = mesh::io::gltf::load_from_path(path); if (!result.has_value()) { - return std::unexpected(::Error::make(result.error() == mesh::io::LoadMeshErrorKind::FileNotFound ? ::Error::Code::NotFound : ::Error::Code::CorruptData, - result.error().description())); + switch (static_cast(result.error())) { + case mesh::io::LoadMeshErrorKind::UnsupportedFormat: + return std::unexpected(::Error::make(::Error::Code::Unsupported, "load unsupported glTF format from", path)); + case mesh::io::LoadMeshErrorKind::FileNotFound: + return std::unexpected(::Error::make(::Error::Code::NotFound, "open glTF file", path)); + case mesh::io::LoadMeshErrorKind::InvalidFormat: + return std::unexpected(::Error::make(::Error::Code::CorruptData, "decode invalid glTF file", path)); + case mesh::io::LoadMeshErrorKind::OutOfMemory: + return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "load glTF file: out of memory", path)); + } + std::terminate(); } return std::move(result.value()); } std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { + const std::filesystem::path path = paths(node_path).front(); try { - const auto result = mesh::io::gltf::save_to_path(mesh, paths(node_path).front()); + const auto result = mesh::io::gltf::save_to_path(mesh, path); if (!result.has_value()) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, result.error().description())); + switch (static_cast(result.error())) { + case mesh::io::SaveMeshErrorKind::UnsupportedFormat: + return std::unexpected(::Error::make(::Error::Code::Unsupported, "write unsupported glTF format to", path)); + case mesh::io::SaveMeshErrorKind::OpenFile: + return std::unexpected(::Error::make(::Error::Code::Io, "open glTF file for writing", path)); + case mesh::io::SaveMeshErrorKind::WriteFile: + return std::unexpected(::Error::make(::Error::Code::Io, "write glTF file", path)); + case mesh::io::SaveMeshErrorKind::OutOfMemory: + return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "write glTF file: out of memory", path)); + } + std::terminate(); } return {}; + } catch (const std::bad_alloc&) { + throw; } catch (const std::exception& error) { - return std::unexpected(::Error::make(::Error::Code::Internal, error.what())); + return std::unexpected(::Error::make(::Error::Code::Internal, "write glTF file \"" + path.string() + "\": " + error.what())); } catch (...) { - return std::unexpected(::Error::make(::Error::Code::Internal, "unknown glTF writer exception")); + return std::unexpected(::Error::make(::Error::Code::Internal, "write glTF file \"" + path.string() + "\": unknown exception")); } } diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h index 74ad6ee2..ac288192 100644 --- a/src/terrainlib/mesh/codec/SfMesh.h +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -19,16 +19,18 @@ class SfMesh final : public store::Codec { std::expected read(const std::filesystem::path& node_path) const override { - auto payload = ::io::envelope::read_from_path(paths(node_path).front()); + const std::filesystem::path path = paths(node_path).front(); + auto payload = ::io::envelope::read_from_path(path); if (!payload) { return std::unexpected(std::move(payload).error().with_context("read SF mesh")); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, valid.error())); + return std::unexpected( + ::Error::make(::Error::Code::CorruptData, "invalid SF mesh in \"" + path.string() + "\": " + valid.error())); } auto decoded = mesh::sf::decode_payload(std::move(*payload)); if (!decoded) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "could not decode SF mesh payload")); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "could not decode SF mesh \"" + path.string() + "\"")); } return std::move(*decoded); } @@ -40,14 +42,16 @@ class SfMesh final : public store::Codec { std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const { + const std::filesystem::path path = paths(node_path).front(); auto payload = mesh::sf::encode_payload(mesh, options); if (!payload) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "could not encode SF mesh payload")); + return std::unexpected(::Error::make(::Error::Code::InvalidInput, "could not encode SF mesh \"" + path.string() + "\"")); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, valid.error())); + return std::unexpected( + ::Error::make(::Error::Code::InvalidInput, "invalid SF mesh for \"" + path.string() + "\": " + valid.error())); } - auto result = ::io::envelope::write_to_path(*payload, paths(node_path).front()); + auto result = ::io::envelope::write_to_path(*payload, path); if (!result) { return std::unexpected(std::move(result).error().with_context("write SF mesh")); } diff --git a/src/terrainlib/octree/StoreTraits.h b/src/terrainlib/octree/StoreTraits.h index 54252824..67334210 100644 --- a/src/terrainlib/octree/StoreTraits.h +++ b/src/terrainlib/octree/StoreTraits.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "octree/Id.h" @@ -14,6 +15,7 @@ struct StoreTraits { static constexpr auto parent(const Key& key) { return key.parent(); } static constexpr auto children(const Key& key) { return key.children(); } static constexpr bool is_valid(const Key& key) { return key.level() <= Key::max_level() && key.index_on_level() <= Key::max_index_on_level(key.level()); } + static std::string key_to_string(const Key& key) { return key.to_string(); } }; } // namespace octree diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 0dde3a39..5eb80290 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -123,17 +123,12 @@ inline std::expected, std::string> decode_index(const return result; } -inline ::Error read_error(const std::filesystem::path& path, ::Error error) -{ - return std::move(error).with_context("read octree store file " + path.string()); -} - inline std::expected read_store_metadata(const std::filesystem::path& base_path) { const std::filesystem::path path = base_path / metadata_file_name; auto metadata = io::envelope::read_from_path(path); if (!metadata) { - return std::unexpected(read_error(path, std::move(metadata).error())); + return std::unexpected(std::move(metadata).error()); } if (auto valid = validate(*metadata); !valid) { return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid store metadata in " + path.string() + ": " + valid.error())); @@ -145,12 +140,12 @@ inline std::expected, ::Error> read_index_file { auto metadata = read_store_metadata(path.parent_path()); if (!metadata) { - return std::unexpected(metadata.error()); + return std::unexpected(std::move(metadata).error()); } auto encoded_index = io::envelope::read_from_path(path); if (!encoded_index) { - return std::unexpected(read_error(path, std::move(encoded_index).error())); + return std::unexpected(std::move(encoded_index).error()); } auto index = decode_index(*encoded_index); if (!index) { diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index 86406012..e8f78ddd 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -61,7 +61,7 @@ std::expected, ::Error> open_folder(const if (index_exists) { auto indexed = store::open_index(index_path, format, payload_class, resolve_codec); if (!indexed) { - return std::unexpected(indexed.error()); + return std::unexpected(std::move(indexed).error()); } return store::Storage(std::move(*indexed)); } @@ -70,7 +70,7 @@ std::expected, ::Error> open_folder(const std::string codec_selector = options.preferred_extension.value_or(std::move(default_codec_selector)); auto codec = resolve_codec(codec_selector); if (!codec) { - return std::unexpected(codec.error()); + return std::unexpected(std::move(codec).error()); } return store::make_storage(base_path, format, mapping, std::move(payload_class), std::move(codec_selector), std::move(*codec)); @@ -85,7 +85,7 @@ std::expected, ::Error> open_folder { auto result = open_folder(base_path, std::move(payload_class), std::move(default_codec_selector), std::move(resolve_codec), std::move(options)); if (!result) { - return std::unexpected(result.error()); + return std::unexpected(std::move(result).error()); } return store::IndexedStorage(std::move(*result)); } diff --git a/src/terrainlib/raster_store/StoreTraits.h b/src/terrainlib/raster_store/StoreTraits.h index bd1dea66..111f941b 100644 --- a/src/terrainlib/raster_store/StoreTraits.h +++ b/src/terrainlib/raster_store/StoreTraits.h @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -41,6 +42,7 @@ struct StoreTraits { const uint32_t extent = uint32_t { 1 } << key.zoom_level; return key.coords.x < extent && key.coords.y < extent; } + static std::string key_to_string(const Key& key) { return to_string(key); } }; } // namespace raster_store diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h index 0da0329e..3fc7eaa3 100644 --- a/src/terrainlib/sf/finalize_storage.h +++ b/src/terrainlib/sf/finalize_storage.h @@ -11,18 +11,14 @@ namespace sf { inline std::expected finalize_storage(mesh::storage::Storage& storage) { - const auto save_result = storage.save_or_create_index(); + auto save_result = storage.save_or_create_index(); if (!save_result.has_value()) { - return std::unexpected(save_result.error()); + return save_result; } const auto index = storage.index(); DEBUG_ASSERT(index.has_value()); - const auto validation = validate_index(index->get()); - if (!validation.has_value()) { - return std::unexpected(validation.error()); - } - return {}; + return validate_index(index->get()); } } // namespace sf diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h index 3f000bbb..85850f42 100644 --- a/src/terrainlib/store/Index.h +++ b/src/terrainlib/store/Index.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "Error.h" #include "log.h" @@ -24,7 +25,7 @@ class Index { std::expected, ::Error> get(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } const NodeStatus* status = get_raw_unchecked(key); if (status == nullptr) { @@ -36,7 +37,7 @@ class Index { std::expected add(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } NodeStatus* status = get_raw_unchecked(key); @@ -62,13 +63,13 @@ class Index { } if (!Traits::is_valid(parent.value())) { - return std::unexpected(invalid_key_error(parent.value())); + return std::unexpected(store::invalid_key_error(parent.value())); } NodeStatus* parent_status = get_raw_unchecked(parent.value()); if (parent_status == nullptr) { - const auto parent_result = add(parent.value()); + auto parent_result = add(parent.value()); if (!parent_result.has_value()) { - return std::unexpected(parent_result.error()); + return std::unexpected(std::move(parent_result).error()); } set_raw_unchecked(parent.value(), NodeStatus::Virtual); set_raw_unchecked(key, NodeStatus::Leaf); @@ -91,7 +92,7 @@ class Index { std::expected remove(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } NodeStatus* status = get_raw_unchecked(key); @@ -114,23 +115,23 @@ class Index { std::expected is_present(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } return m_index.contains(key); } std::expected is_absent(const Key& key) const { - const auto present = is_present(key); + auto present = is_present(key); if (!present.has_value()) { - return std::unexpected(present.error()); + return std::unexpected(std::move(present).error()); } return !present.value(); } std::expected is(const NodeStatus status, const Key& key) const { - const auto result = get(key); + auto result = get(key); if (!result.has_value()) { - return std::unexpected(result.error()); + return std::unexpected(std::move(result).error()); } return result.value() == status; } @@ -138,21 +139,21 @@ class Index { std::expected get_raw(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } return get_raw_unchecked(key); } std::expected get_raw(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } return get_raw_unchecked(key); } std::expected set_raw(const Key& key, const NodeStatus status) { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } set_raw_unchecked(key, status); return {}; @@ -160,7 +161,7 @@ class Index { std::expected remove_raw(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } remove_raw_unchecked(key); return {}; @@ -176,11 +177,6 @@ class Index { const_iterator cend() const { return m_index.cend(); } private: - static ::Error invalid_key_error(const Key& key) - { - return ::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + key_to_string(key)); - } - NodeStatus* get_raw_unchecked(const Key& key) { const auto iterator = m_index.find(key); @@ -201,7 +197,7 @@ class Index { return true; } if (!Traits::is_valid(parent.value())) { - return std::unexpected(invalid_key_error(parent.value())); + return std::unexpected(store::invalid_key_error(parent.value())); } NodeStatus* parent_status = get_raw_unchecked(parent.value()); diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index ffe0a982..7d42642e 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -35,40 +35,32 @@ class RawStorage { std::expected load(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } - const auto result = m_codec->read(m_layout.node_path(key)); - if (!result.has_value()) { - return std::unexpected(result.error()); - } - return std::move(*result); + return m_codec->read(m_layout.node_path(key)); } std::expected save(const Key& key, const NodeData& data) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); - } - const auto result = m_codec->write(m_layout.node_path(key), data); - if (!result.has_value()) { - return std::unexpected(result.error()); + return std::unexpected(store::invalid_key_error(key)); } - return {}; + return m_codec->write(m_layout.node_path(key), data); } std::expected, ::Error> paths(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } return m_codec->paths(m_layout.node_path(key)); } std::expected has(const Key& key) const { - const auto node_paths = paths(key); + auto node_paths = paths(key); if (!node_paths.has_value()) { - return std::unexpected(node_paths.error()); + return std::unexpected(std::move(node_paths).error()); } if (node_paths->empty()) { return false; @@ -88,9 +80,9 @@ class RawStorage { std::expected remove(const Key& key) const { - const auto node_paths = paths(key); + auto node_paths = paths(key); if (!node_paths.has_value()) { - return std::unexpected(node_paths.error()); + return std::unexpected(std::move(node_paths).error()); } bool removed = false; for (const auto& path : node_paths.value()) { @@ -112,14 +104,14 @@ class RawStorage { std::expected copy_from(const Key& key, const RawStorage& source, BeforeModify&& before_modify) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } - const auto source_exists = source.has(key); + auto source_exists = source.has(key); if (!source_exists.has_value()) { - return std::unexpected(source_exists.error()); + return std::unexpected(std::move(source_exists).error()); } if (!source_exists.value()) { - return std::unexpected(::Error::make(::Error::Code::NotFound, "source node " + key_to_string(key) + " is missing")); + return std::unexpected(::Error::make(::Error::Code::NotFound, "source node " + Traits::key_to_string(key) + " is missing")); } const std::filesystem::path probe("__codec_probe__/node"); @@ -174,11 +166,6 @@ class RawStorage { const Codec& codec() const { return *m_codec; } private: - static ::Error invalid_key_error(const Key& key) - { - return ::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + key_to_string(key)); - } - path_layout::Resolver m_layout; std::unique_ptr> m_codec; }; diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index 9bb73927..33ee67b5 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -24,8 +24,6 @@ class Storage { using Key = typename Traits::Key; using key_type = Key; using value_type = NodeData; - using load_error = ::Error; - using save_error = ::Error; struct Persistence { IndexFormat format; @@ -83,16 +81,16 @@ class Storage { std::expected load(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } if (m_index.has_value()) { - const auto status = m_index->get(key); + auto status = m_index->get(key); if (!status.has_value()) { - return std::unexpected(status.error()); + return std::unexpected(std::move(status).error()); } if (!status->has_value() || status->value() == NodeStatus::Virtual) { return std::unexpected( - ::Error::make(::Error::Code::NotFound, "node " + key_to_string(key) + " is not physically present in the index")); + ::Error::make(::Error::Code::NotFound, "node " + Traits::key_to_string(key) + " is not physically present in the index")); } } return m_raw.load(key); @@ -101,11 +99,11 @@ class Storage { std::expected save(const Key& key, const NodeData& data) { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } - const auto exists = has(key); + auto exists = has(key); if (!exists.has_value()) { - return std::unexpected(exists.error()); + return std::unexpected(std::move(exists).error()); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); @@ -114,14 +112,14 @@ class Storage { node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front())); } - const auto result = m_raw.save(key, data); + auto result = m_raw.save(key, data); if (!result.has_value()) { return result; } if (m_index.has_value()) { - const auto added = m_index->add(key); + auto added = m_index->add(key); if (!added.has_value()) { - return std::unexpected(added.error()); + return std::unexpected(std::move(added).error()); } m_dirty = m_dirty || added.value(); } @@ -131,11 +129,11 @@ class Storage { std::expected copy_from(const Key& key, const Storage& source) { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } - const auto exists = has(key); + auto exists = has(key); if (!exists.has_value()) { - return std::unexpected(exists.error()); + return std::unexpected(std::move(exists).error()); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); @@ -147,22 +145,22 @@ class Storage { if (!exists.value() || !m_index.has_value()) { return {}; } - const auto removed = m_index->remove(key); + auto removed = m_index->remove(key); if (!removed.has_value()) { - return std::unexpected(removed.error()); + return std::unexpected(std::move(removed).error()); } m_dirty = m_dirty || removed.value(); return {}; }; - const auto result = m_raw.copy_from(key, source.m_raw, prepare_target); + auto result = m_raw.copy_from(key, source.m_raw, prepare_target); if (!result.has_value()) { return result; } if (m_index.has_value()) { - const auto added = m_index->add(key); + auto added = m_index->add(key); if (!added.has_value()) { - return std::unexpected(added.error()); + return std::unexpected(std::move(added).error()); } m_dirty = m_dirty || added.value(); } @@ -171,14 +169,14 @@ class Storage { std::expected remove(const Key& key) { - const auto removed = m_raw.remove(key); + auto removed = m_raw.remove(key); if (!removed.has_value()) { return removed; } if (m_index.has_value()) { - const auto index_removed = m_index->remove(key); + auto index_removed = m_index->remove(key); if (!index_removed.has_value()) { - return std::unexpected(index_removed.error()); + return std::unexpected(std::move(index_removed).error()); } m_dirty = m_dirty || index_removed.value(); } @@ -188,14 +186,14 @@ class Storage { std::expected has(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(invalid_key_error(key)); + return std::unexpected(store::invalid_key_error(key)); } if (!m_index.has_value()) { return m_raw.has(key); } - const auto status = m_index->get(key); + auto status = m_index->get(key); if (!status.has_value()) { - return std::unexpected(status.error()); + return std::unexpected(std::move(status).error()); } return status->has_value() && status->value() != NodeStatus::Virtual; } @@ -204,9 +202,9 @@ class Storage { std::expected path_for(const Key& key) const { - const auto node_paths = paths(key); + auto node_paths = paths(key); if (!node_paths.has_value()) { - return std::unexpected(node_paths.error()); + return std::unexpected(std::move(node_paths).error()); } return node_paths->empty() ? m_raw.layout().node_path(key) : node_paths->front(); } @@ -253,7 +251,7 @@ class Storage { m_persistence->payload_class, m_persistence->codec_selector, }; - const auto result = m_persistence->format.write(m_persistence->index_path, metadata); + auto result = m_persistence->format.write(m_persistence->index_path, metadata); if (result.has_value()) { m_dirty = false; } @@ -279,11 +277,6 @@ class Storage { void set_index_dirty(const bool dirty = true) const { m_dirty = dirty; } private: - static ::Error invalid_key_error(const Key& key) - { - return ::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + key_to_string(key)); - } - void finalize_displaced_state() noexcept { if (!m_dirty || !m_index.has_value()) { diff --git a/src/terrainlib/store/ThreadSafeStorage.h b/src/terrainlib/store/ThreadSafeStorage.h index 56427038..13c30002 100644 --- a/src/terrainlib/store/ThreadSafeStorage.h +++ b/src/terrainlib/store/ThreadSafeStorage.h @@ -17,8 +17,6 @@ class ThreadSafeStorage { public: using value_type = typename Storage::value_type; using key_type = typename Storage::key_type; - using load_error = typename Storage::load_error; - using save_error = typename Storage::save_error; explicit ThreadSafeStorage(Storage&& storage) : m_storage(std::move(storage)) @@ -32,7 +30,7 @@ class ThreadSafeStorage { Storage release() && { return std::move(m_storage); } - std::expected load(const key_type& key) const + auto load(const key_type& key) const { std::shared_lock lock(m_mutex); return m_storage.load(key); @@ -46,7 +44,7 @@ class ThreadSafeStorage { std::filesystem::path base_path() const noexcept { return m_storage.base_path(); } - std::expected save(const key_type& key, const value_type& value) const + auto save(const key_type& key, const value_type& value) const { std::unique_lock lock(m_mutex); return m_storage.save(key, value); diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h index ef0efa11..438df4bf 100644 --- a/src/terrainlib/store/Traits.h +++ b/src/terrainlib/store/Traits.h @@ -1,8 +1,11 @@ #pragma once #include +#include #include +#include "Error.h" + namespace store { template @@ -13,16 +16,14 @@ concept HierarchyTraits = requires(typename Traits::Key key) { Traits::parent(key); Traits::children(key); { Traits::is_valid(key) } -> std::same_as; + { Traits::key_to_string(key) } -> std::same_as; }; -template -std::string key_to_string(const Key& key) +template +::Error invalid_key_error(const typename Traits::Key& key, + const std::source_location location = std::source_location::current()) { - if constexpr (requires { key.to_string(); }) { - return key.to_string(); - } else { - return to_string(key); - } + return ::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + Traits::key_to_string(key), location); } } // namespace store diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h index 0734ac3e..9af6f369 100644 --- a/src/terrainlib/store/open.h +++ b/src/terrainlib/store/open.h @@ -16,9 +16,9 @@ std::expected, ::Error> open_index( const std::filesystem::path& index_path, const IndexFormat format, const std::string_view expected_payload_class, CodecResolver&& resolve_codec) { using Key = typename Traits::Key; - const auto metadata_result = format.read(index_path); + auto metadata_result = format.read(index_path); if (!metadata_result.has_value()) { - return std::unexpected(metadata_result.error()); + return std::unexpected(std::move(metadata_result).error()); } IndexMetadata metadata = std::move(metadata_result.value()); if (metadata.payload_class != expected_payload_class) { @@ -27,7 +27,7 @@ std::expected, ::Error> open_index( for (const auto& [key, status] : metadata.index) { static_cast(status); if (!Traits::is_valid(key)) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "index contains invalid hierarchy key " + key_to_string(key))); + return std::unexpected(::Error::make(::Error::Code::CorruptData, "index contains invalid hierarchy key " + Traits::key_to_string(key))); } } @@ -37,7 +37,7 @@ std::expected, ::Error> open_index( } auto codec = resolve_codec(metadata.codec_selector); if (!codec.has_value()) { - return std::unexpected(codec.error()); + return std::unexpected(std::move(codec).error()); } const std::filesystem::path base_path = index_path.parent_path(); diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h index c03cb49b..ecf618b7 100644 --- a/src/terrainlib/store/traverse.h +++ b/src/terrainlib/store/traverse.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "store/Index.h" @@ -30,11 +31,11 @@ std::expected traverse(const Index& index, { using Key = typename Traits::Key; if (!Traits::is_valid(root)) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + key_to_string(root))); + return std::unexpected(store::invalid_key_error(root)); } - const auto root_status = index.get(root); + auto root_status = index.get(root); if (!root_status.has_value()) { - return std::unexpected(root_status.error()); + return std::unexpected(std::move(root_status).error()); } if (!root_status.value().has_value()) { return {}; @@ -43,9 +44,9 @@ std::expected traverse(const Index& index, if (order == TraversalOrder::DepthFirst) { std::function(const Key&)> depth_first; depth_first = [&](const Key& current) -> std::expected { - const auto status = index.get(current); + auto status = index.get(current); if (!status.has_value()) { - return std::unexpected(status.error()); + return std::unexpected(std::move(status).error()); } if (!status.value().has_value()) { return {}; @@ -56,7 +57,7 @@ std::expected traverse(const Index& index, const auto children = Traits::children(current); if (children.has_value()) { for (const Key& child : children.value()) { - const auto result = depth_first(child); + auto result = depth_first(child); if (!result.has_value()) { return result; } @@ -74,9 +75,9 @@ std::expected traverse(const Index& index, const Key current = queue.front(); queue.pop(); - const auto status = index.get(current); + auto status = index.get(current); if (!status.has_value()) { - return std::unexpected(status.error()); + return std::unexpected(std::move(status).error()); } if (!status.value().has_value()) { continue; diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index 28d65412..00734344 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -467,9 +467,7 @@ TEST_CASE("Envelope rejects incompatible metadata") --envelope.uncompressed_size; const auto result = io::envelope::deserialize(encode_envelope(envelope)); check_error(result, ::Error::Code::CorruptData); - REQUIRE_FALSE(result.error().frames().empty()); - CHECK(result.error().frames().back().original_code == ::Error::Code::ResourceExhausted); - CHECK(result.error().frames().back().new_code == ::Error::Code::CorruptData); + CHECK(result.error().to_string().find("reclassified ResourceExhausted -> CorruptData") != std::string::npos); } SECTION("uncompressed size is larger than the payload") From d3719274d7df6ae87681e1f986e2b37992473369 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:30:32 +0200 Subject: [PATCH 30/43] Simplify glTF exception handling --- src/terrainlib/mesh/codec/Gltf.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index ac740243..0fe849e5 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -69,8 +68,6 @@ class Gltf final : public store::Codec { std::terminate(); } return {}; - } catch (const std::bad_alloc&) { - throw; } catch (const std::exception& error) { return std::unexpected(::Error::make(::Error::Code::Internal, "write glTF file \"" + path.string() + "\": " + error.what())); } catch (...) { From 1663f34fa3a03912762781d9cd217912c73ebc9a Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:35:50 +0200 Subject: [PATCH 31/43] Simplify error reclassification frames --- src/terrainlib/Error.cpp | 11 ++--------- src/terrainlib/Error.h | 3 --- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/terrainlib/Error.cpp b/src/terrainlib/Error.cpp index c63b1477..257a1d27 100644 --- a/src/terrainlib/Error.cpp +++ b/src/terrainlib/Error.cpp @@ -44,7 +44,7 @@ Error::Error(const Code code, Frame frame) Error Error::make(const Code code, std::string message, const std::source_location location) { - return Error(code, Frame { std::move(message), location, std::nullopt, std::nullopt }); + return Error(code, Frame { std::move(message), location }); } Error Error::make(const Code code, @@ -90,8 +90,6 @@ Error Error::with_context(std::string message, const std::source_location locati result.m_frames.push_back(Frame { std::move(message), location, - std::nullopt, - std::nullopt, }); return result; } @@ -100,10 +98,8 @@ Error Error::reclassified(const Code code, std::string message, const std::sourc { Error result = std::move(*this); result.m_frames.push_back(Frame { - std::move(message), + fmt::format("{} (reclassified {} -> {})", message, code_name(result.m_code), code_name(code)), location, - result.m_code, - code, }); result.m_code = code; return result; @@ -119,9 +115,6 @@ std::string Error::to_string() const frame->location.file_name(), frame->location.line(), frame->location.function_name()); - if (frame->original_code.has_value() && frame->new_code.has_value()) { - result += fmt::format("\n reclassified {} -> {}", code_name(frame->original_code.value()), code_name(frame->new_code.value())); - } } return result; } diff --git a/src/terrainlib/Error.h b/src/terrainlib/Error.h index 0e2486ed..e5d9e8c4 100644 --- a/src/terrainlib/Error.h +++ b/src/terrainlib/Error.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -32,8 +31,6 @@ class Error { struct Frame { std::string message; std::source_location location; - std::optional original_code; - std::optional new_code; }; static Error make(Code code, From a4ad9354a584c8b1495c0657c977d41293534009 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 28 Aug 2026 22:54:39 +0200 Subject: [PATCH 32/43] Simplify unified error handling --- src/dag_builder/build.cpp | 12 ++-- src/dag_builder/build.h | 4 +- src/dag_builder/codec/Dag.h | 51 +++++++------- src/dag_builder/codec/from_extension.h | 8 +-- src/dag_builder/storage.h | 8 +-- src/sf_builder/terrainbuilder.cpp | 2 +- src/sf_builder/terrainbuilder.h | 2 +- src/sf_merger/NodeWriter.h | 12 ++-- src/sf_merger/cut.h | 16 ++--- src/sf_merger/merge.h | 13 ++-- src/terrainlib/Error.cpp | 45 +++++++++++++ src/terrainlib/Error.h | 67 +++++++++++++++++++ src/terrainlib/io/bytes.cpp | 22 +++--- src/terrainlib/io/bytes.h | 4 +- src/terrainlib/io/compression.cpp | 56 ++++++++-------- src/terrainlib/io/compression.h | 4 +- src/terrainlib/io/conversion.h | 36 +++++----- src/terrainlib/io/envelope.h | 6 +- src/terrainlib/io/envelope.inl | 58 ++++++++-------- src/terrainlib/io/envelope_file.h | 12 ++-- src/terrainlib/mesh/codec/Gltf.h | 24 +++---- src/terrainlib/mesh/codec/SfMesh.h | 20 +++--- src/terrainlib/mesh/codec/from_extension.h | 4 +- src/terrainlib/mesh/storage.h | 6 +- src/terrainlib/octree/storage/IndexFile.h | 22 +++--- src/terrainlib/octree/storage/open_runtime.h | 26 +++---- src/terrainlib/sf/finalize_storage.h | 2 +- src/terrainlib/sf/validate_index.h | 6 +- src/terrainlib/store/Codec.h | 8 +-- src/terrainlib/store/Index.h | 48 ++++++------- src/terrainlib/store/IndexFormat.h | 4 +- src/terrainlib/store/RawStorage.h | 50 +++++++------- src/terrainlib/store/Storage.h | 59 ++++++++-------- src/terrainlib/store/Traits.h | 4 +- src/terrainlib/store/open.h | 16 ++--- src/terrainlib/store/traverse.h | 14 ++-- unittests/CMakeLists.txt | 1 + .../dag_builder/storage_compatibility.cpp | 10 +-- unittests/sf_builder/storage_finalization.cpp | 2 +- unittests/sf_merger/storage_boundaries.cpp | 4 +- unittests/terrainlib/envelope.cpp | 54 +++++++-------- unittests/terrainlib/error.cpp | 40 +++++++++++ unittests/terrainlib/io_conversion.cpp | 10 +-- unittests/terrainlib/sf_validate_index.cpp | 2 +- unittests/terrainlib/store_codec.cpp | 10 +-- unittests/terrainlib/store_compatibility.cpp | 10 +-- unittests/terrainlib/store_index.cpp | 12 ++-- unittests/terrainlib/store_storage.cpp | 62 ++++++++--------- unittests/terrainlib/store_traverse.cpp | 2 +- 49 files changed, 559 insertions(+), 411 deletions(-) create mode 100644 unittests/terrainlib/error.cpp diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 1c23ad37..e1eca1f7 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -51,7 +51,7 @@ std::optional load_and_clusterize_mesh( const auto result = storage.load(id); if (!result) { - if (result.error().code() != ::Error::Code::NotFound) { + if (result.error().code() != Error::Code::NotFound) { LOG_ERROR( "Failed to read node {}: {}", id, @@ -449,7 +449,7 @@ LevelWorkplan build_level_workplan( } // Pre-filter input nodes to only those that intersect the target root bounds. -std::expected>, ::Error> gather_relevant_input_leaves( +Expected>> gather_relevant_input_leaves( const store::Index &index, const octree::Space &space, const radix::geometry::Aabb3d &root_bounds) @@ -469,7 +469,7 @@ std::expected>, ::Error> gather_relevant_inp }, start); if (!traversal.has_value()) { - return std::unexpected(std::move(traversal).error()); + return Error::propagate(std::move(traversal)); } return result; } @@ -583,7 +583,7 @@ std::unordered_set build_level( // Builds the DAG from input_storage into output_storage, restricted to levels within level_range. // Iterates octree levels from finest to coarsest, simplifying and re-clustering geometry at each // level from its children. -std::expected build_levels( +Expected build_levels( const mesh::storage::IndexedStorage &input_storage, dag::storage::IndexedStorage &output_storage, const BuildOptions &options, @@ -598,7 +598,7 @@ std::expected build_levels( const auto root_bounds = shifted_space.get_node_bounds_with_children(root_node); auto input_by_level_result = gather_relevant_input_leaves(input_storage.index(), space, root_bounds); if (!input_by_level_result.has_value()) { - return std::unexpected(std::move(input_by_level_result).error()); + return Error::propagate(std::move(input_by_level_result)); } auto input_by_level = std::move(input_by_level_result.value()); @@ -649,7 +649,7 @@ std::expected build_levels( } // Builds the complete DAG from input_storage into output_storage. -std::expected build_full( +Expected build_full( const mesh::storage::IndexedStorage &input_storage, dag::storage::IndexedStorage &output_storage, const BuildOptions &options) { diff --git a/src/dag_builder/build.h b/src/dag_builder/build.h index 8247ef21..041fad1a 100644 --- a/src/dag_builder/build.h +++ b/src/dag_builder/build.h @@ -32,12 +32,12 @@ struct BuildOptions { ContinuationMode continuation_mode = ContinuationMode::Error; }; -std::expected build_full( +Expected build_full( const mesh::storage::IndexedStorage &input_storage, dag::storage::IndexedStorage &output_storage, const BuildOptions &options); -std::expected build_levels( +Expected build_levels( const mesh::storage::IndexedStorage &input_storage, dag::storage::IndexedStorage &output_storage, const BuildOptions &options, diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index 6f7df016..e67f5391 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -11,10 +11,10 @@ namespace dag::codec { -inline std::expected validate_batch(const dag::ClusterBatch& batch, const ::Error::Code code) +inline Expected validate_batch(const dag::ClusterBatch& batch, const Error::Code code) { if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { - return std::unexpected(::Error::make(code, "DAG metadata group-assignment count does not match clustering cluster count")); + return Error::fail(code, "DAG metadata group-assignment count does not match clustering cluster count"); } return {}; } @@ -29,58 +29,58 @@ class Dag final : public store::Codec { }; } - std::expected read(const std::filesystem::path& node_path) const override + Expected read(const std::filesystem::path& node_path) const override { const auto node_paths = paths(node_path); auto clustering_payload = io::envelope::read_from_path(node_paths[0]); if (!clustering_payload) { - return std::unexpected(std::move(clustering_payload).error()); + return Error::propagate(std::move(clustering_payload)); } auto metadata_payload = io::envelope::read_from_path(node_paths[1]); if (!metadata_payload) { - return std::unexpected(std::move(metadata_payload).error()); + return Error::propagate(std::move(metadata_payload)); } auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); if (!clustering) { - return std::unexpected(::Error::make( - ::Error::Code::CorruptData, "invalid DAG clustering in \"" + node_paths[0].string() + "\": " + clustering.error())); + return Error::fail( + Error::Code::CorruptData, "invalid DAG clustering in \"" + node_paths[0].string() + "\": " + clustering.error()); } auto metadata = dag::format::decode_metadata(std::move(*metadata_payload)); if (!metadata) { - return std::unexpected(::Error::make( - ::Error::Code::CorruptData, "invalid DAG metadata in \"" + node_paths[1].string() + "\": " + metadata.error())); + return Error::fail( + Error::Code::CorruptData, "invalid DAG metadata in \"" + node_paths[1].string() + "\": " + metadata.error()); } dag::ClusterBatch batch { .metadata = std::move(*metadata), .clustering = std::move(*clustering), }; - if (auto valid = validate_batch(batch, ::Error::Code::CorruptData); !valid) { - return std::unexpected(std::move(valid).error().with_context( - "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\"")); + if (auto valid = validate_batch(batch, Error::Code::CorruptData); !valid) { + return Error::propagate(std::move(valid), + "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); } return batch; } - std::expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override + Expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override { const auto node_paths = paths(node_path); - if (auto valid = validate_batch(batch, ::Error::Code::InvalidInput); !valid) { - return std::unexpected(std::move(valid).error().with_context( - "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\"")); + if (auto valid = validate_batch(batch, Error::Code::InvalidInput); !valid) { + return Error::propagate(std::move(valid), + "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); } auto clustering = dag::format::encode_clustering(batch.clustering); if (!clustering) { - return std::unexpected(::Error::make( - ::Error::Code::InvalidInput, "cannot encode DAG clustering for \"" + node_paths[0].string() + "\": " + clustering.error())); + return Error::fail( + Error::Code::InvalidInput, "cannot encode DAG clustering for \"" + node_paths[0].string() + "\": " + clustering.error()); } const dag::format::NodeMetadata metadata = dag::format::encode_metadata(batch.metadata); if (auto valid = dag::format::validate(*clustering); !valid) { - return std::unexpected(::Error::make( - ::Error::Code::InvalidInput, "invalid DAG clustering for \"" + node_paths[0].string() + "\": " + valid.error())); + return Error::fail( + Error::Code::InvalidInput, "invalid DAG clustering for \"" + node_paths[0].string() + "\": " + valid.error()); } if (auto valid = dag::format::validate(metadata); !valid) { - return std::unexpected(::Error::make( - ::Error::Code::InvalidInput, "invalid DAG metadata for \"" + node_paths[1].string() + "\": " + valid.error())); + return Error::fail( + Error::Code::InvalidInput, "invalid DAG metadata for \"" + node_paths[1].string() + "\": " + valid.error()); } auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); if (!data_written) { @@ -101,17 +101,16 @@ class MetadataView final : public store::Codec { return { add_extension(node_path, ".dagmeta") }; } - std::expected read(const std::filesystem::path& node_path) const override + Expected read(const std::filesystem::path& node_path) const override { const std::filesystem::path path = paths(node_path).front(); auto payload = io::envelope::read_from_path(path); if (!payload) { - return std::unexpected(std::move(payload).error()); + return Error::propagate(std::move(payload)); } auto metadata = dag::format::decode_metadata(std::move(*payload)); if (!metadata) { - return std::unexpected( - ::Error::make(::Error::Code::CorruptData, "invalid DAG metadata in \"" + path.string() + "\": " + metadata.error())); + return Error::fail(Error::Code::CorruptData, "invalid DAG metadata in \"" + path.string() + "\": " + metadata.error()); } return std::move(*metadata); } diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h index d2bf0953..76546828 100644 --- a/src/dag_builder/codec/from_extension.h +++ b/src/dag_builder/codec/from_extension.h @@ -11,20 +11,20 @@ namespace dag::codec { -inline std::expected>, ::Error> from_extension(const std::string_view extension) +inline Expected>> from_extension(const std::string_view extension) { if (extension == ".dag") { return std::make_unique(); } - return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported DAG codec selector: " + std::string(extension))); + return Error::fail(Error::Code::Unsupported, "unsupported DAG codec selector: " + std::string(extension)); } -inline std::expected>, ::Error> metadata_from_extension(const std::string_view extension) +inline Expected>> metadata_from_extension(const std::string_view extension) { if (extension == ".dag") { return std::make_unique(); } - return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported DAG metadata codec selector: " + std::string(extension))); + return Error::fail(Error::Code::Unsupported, "unsupported DAG metadata codec selector: " + std::string(extension)); } } // namespace dag::codec diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 849ff2b1..ba8555d4 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -20,7 +20,7 @@ using MetadataStorage = store::Storage; using IndexedMetadataStorage = store::IndexedStorage; -inline std::expected open_index( +inline Expected open_index( const std::filesystem::path &path) { return store::open_index( path, @@ -29,7 +29,7 @@ inline std::expected open_index( dag::codec::from_extension); } -inline std::expected open_folder( +inline Expected open_folder( const std::filesystem::path &path, OpenOptions options = {}) { return octree::storage::open_folder( @@ -40,7 +40,7 @@ inline std::expected open_folder( std::move(options)); } -inline std::expected +inline Expected open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { return octree::storage::open_folder_indexed( path, @@ -50,7 +50,7 @@ open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) std::move(options)); } -inline std::expected +inline Expected open_metadata_indexed(const std::filesystem::path &path) { const std::filesystem::path index_path = path.filename() == octree::storage::index_file_name diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index b052229e..3585687d 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -172,7 +172,7 @@ T expect(const std::optional &opt, const std::string &msg) { } } -std::expected build_all_patches( +Expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index 1ad13fec..64ad66d0 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -31,7 +31,7 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -std::expected build_all_patches( +Expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 807331c3..88c10e60 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -16,12 +16,12 @@ class NodeWriter { public: NodeWriter(mesh::storage::Storage &storage) : _storage(storage) {} - std::expected has_node( + Expected has_node( const octree::Id &id) { return this->_storage.has(id); } - std::expected write_node( + Expected write_node( const octree::Id &id, const SimpleMesh &mesh) { mesh::validate(mesh); @@ -31,7 +31,7 @@ class NodeWriter { } auto path_result = this->_storage.path_for(id); if (!path_result.has_value()) { - return std::unexpected(std::move(path_result).error()); + return Error::propagate(std::move(path_result)); } auto p = path_result.value(); // change extension to .png @@ -42,10 +42,10 @@ class NodeWriter { return {}; } - std::expected copy_subtree_to_output( + Expected copy_subtree_to_output( const octree::Id &id, const NodeLoader &loader) { - std::optional<::Error> error = std::nullopt; + std::optional error = std::nullopt; auto traversal = store::traverse( loader.storage().index(), [&](const octree::Id &child_id, const store::NodeStatus &status) { @@ -68,7 +68,7 @@ class NodeWriter { return traversal; } if (error.has_value()) { - return std::unexpected(std::move(error.value())); + return Error::propagate(std::move(error.value())); } return {}; } diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index eb11abea..919fa97b 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -23,12 +23,12 @@ struct Context { const bool keep_inside; }; -std::expected cut_node( +Expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask); -inline std::expected cut_leaf_node( +inline Expected cut_leaf_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -62,7 +62,7 @@ inline std::expected cut_leaf_node( return {}; } -inline std::expected cut_virtual_node( +inline Expected cut_virtual_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -85,7 +85,7 @@ inline std::expected cut_virtual_node( return {}; } -inline std::expected cut_node( +inline Expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask @@ -122,7 +122,7 @@ inline std::expected cut_node( return {}; } -inline std::expected cut_dataset( +inline Expected cut_dataset( const mesh::storage::IndexedStorage &input, const MeshMask& mask, mesh::storage::Storage &output, @@ -139,7 +139,7 @@ inline std::expected cut_dataset( return sf::finalize_storage(output); } -inline std::expected cut_dataset( +inline Expected cut_dataset( const mesh::storage::IndexedStorage &input_dataset, const MeshMask& mask, const std::filesystem::path &output_path, @@ -152,11 +152,11 @@ inline std::expected cut_dataset( options.preferred_extension = ".glb"; auto output_result = mesh::storage::open_folder_indexed(output_path, std::move(options)); if (!output_result.has_value()) { - return std::unexpected(std::move(output_result).error()); + return Error::propagate(std::move(output_result)); } mesh::storage::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { - return std::unexpected(::Error::make(::Error::Code::AlreadyExists, "output dataset already contains nodes: " + output_path.string())); + return Error::fail(Error::Code::AlreadyExists, "output dataset already contains nodes: " + output_path.string()); } return cut_dataset(input_dataset, mask, output_dataset, keep_inside); diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index 4d2b7380..c064aa7f 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -37,7 +37,6 @@ class Merger { using Status = store::NodeStatusOrMissing; using Context = Visitor::Context; using Result = merge::Result; - using Expected = std::expected; Merger( Visitor& visitor, @@ -46,19 +45,19 @@ class Merger { NodeWriter output) : _visitor(visitor), _left(left), _right(right), _output(output) { } - Expected merge_root() { + Expected merge_root() { const octree::Id id = octree::Id::root(); const Context ctx = this->_visitor.make_root_context(); return merge_node(id, ctx); } - Expected merge_node(const octree::Id &id, const Context& ctx) { + Expected merge_node(const octree::Id &id, const Context& ctx) { const Status left_status = this->_left.get_status(id); const Status right_status = this->_right.get_status(id); return this->merge_node(id, left_status, right_status, ctx); } - Expected merge_node( + Expected merge_node( const octree::Id &id, const Status left_status, const Status right_status, @@ -67,7 +66,7 @@ class Merger { LOG_DEBUG("[{}] Start merging (left = {}, right = {})", id, left_status, right_status); auto has_result = this->_output.has_node(id); if (!has_result.has_value()) { - return std::unexpected(std::move(has_result).error()); + return Error::propagate(std::move(has_result)); } if (has_result.value()) { LOG_DEBUG("[{}] Already merged, skipping...", id); @@ -75,7 +74,7 @@ class Merger { } const auto merge_result = this->call_merge(id, left_status, right_status, ctx); - return std::visit([&](const auto &result) -> Expected { + return std::visit([&](const auto &result) -> Expected { using Result = std::decay_t; if constexpr (std::is_same_v>) { LOG_DEBUG("[{}] needs recursion", id); @@ -181,7 +180,7 @@ class Merger { NodeWriter _output; }; -inline std::expected merge_datasets( +inline Expected merge_datasets( const mesh::storage::IndexedStorage &left_dataset, const mesh::storage::IndexedStorage &right_dataset, mesh::storage::Storage &output_dataset, diff --git a/src/terrainlib/Error.cpp b/src/terrainlib/Error.cpp index 257a1d27..33d3200d 100644 --- a/src/terrainlib/Error.cpp +++ b/src/terrainlib/Error.cpp @@ -84,6 +84,51 @@ Error Error::make(const Code code, location); } +std::unexpected Error::fail(const Code code, std::string message, const std::source_location location) +{ + return std::unexpected { make(code, std::move(message), location) }; +} + +std::unexpected Error::fail(const Code code, + const std::string_view operation, + const std::filesystem::path& path, + const std::source_location location) +{ + return std::unexpected { make(code, operation, path, location) }; +} + +std::unexpected Error::fail(const Code code, + const std::string_view operation, + const std::error_code& cause, + const std::source_location location) +{ + return std::unexpected { make(code, operation, cause, location) }; +} + +std::unexpected Error::fail(const Code code, + const std::string_view operation, + const std::filesystem::path& path, + const std::error_code& cause, + const std::source_location location) +{ + return std::unexpected { make(code, operation, path, cause, location) }; +} + +std::unexpected Error::fail(const Code code, + const std::string_view operation, + const std::filesystem::path& source, + const std::filesystem::path& destination, + const std::error_code& cause, + const std::source_location location) +{ + return std::unexpected { make(code, operation, source, destination, cause, location) }; +} + +std::unexpected Error::propagate(Error&& error, const std::source_location location) +{ + return std::unexpected { std::move(error).with_context("propagated", location) }; +} + Error Error::with_context(std::string message, const std::source_location location) && { Error result = std::move(*this); diff --git a/src/terrainlib/Error.h b/src/terrainlib/Error.h index e5d9e8c4..bace3f3d 100644 --- a/src/terrainlib/Error.h +++ b/src/terrainlib/Error.h @@ -1,10 +1,12 @@ #pragma once +#include #include #include #include #include #include +#include #include class Error { @@ -56,6 +58,44 @@ class Error { const std::error_code& cause, std::source_location location = std::source_location::current()); + static std::unexpected fail(Code code, + std::string message, + std::source_location location = std::source_location::current()); + static std::unexpected fail(Code code, + std::string_view operation, + const std::filesystem::path& path, + std::source_location location = std::source_location::current()); + static std::unexpected fail(Code code, + std::string_view operation, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + static std::unexpected fail(Code code, + std::string_view operation, + const std::filesystem::path& path, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + static std::unexpected fail(Code code, + std::string_view operation, + const std::filesystem::path& source, + const std::filesystem::path& destination, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + + template + static std::unexpected propagate(std::expected&& result, + std::source_location location = std::source_location::current()); + template + static std::unexpected propagate(std::expected&& result, + std::string message, + std::source_location location = std::source_location::current()); + template + static std::unexpected propagate(std::expected&& result, + Code code, + std::string message, + std::source_location location = std::source_location::current()); + static std::unexpected propagate(Error&& error, + std::source_location location = std::source_location::current()); + [[nodiscard]] Error with_context( std::string message, std::source_location location = std::source_location::current()) &&; @@ -72,3 +112,30 @@ class Error { Code m_code; std::vector m_frames; }; + +template +using Expected = std::expected; + +template +std::unexpected Error::propagate(std::expected&& result, const std::source_location location) +{ + return propagate(std::move(result).error(), location); +} + +template +std::unexpected Error::propagate( + std::expected&& result, std::string message, const std::source_location location) +{ + return std::unexpected { + std::move(result).error().with_context(std::move(message), location), + }; +} + +template +std::unexpected Error::propagate( + std::expected&& result, const Code code, std::string message, const std::source_location location) +{ + return std::unexpected { + std::move(result).error().reclassified(code, std::move(message), location), + }; +} diff --git a/src/terrainlib/io/bytes.cpp b/src/terrainlib/io/bytes.cpp index f3fedf1c..7c81c557 100644 --- a/src/terrainlib/io/bytes.cpp +++ b/src/terrainlib/io/bytes.cpp @@ -7,31 +7,31 @@ namespace io { -std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path& path, bool make_dirs) +Expected write_bytes_to_path(const std::span bytes, const std::filesystem::path& path, bool make_dirs) { LOG_TRACE("Writing bytes to path {}", path); if (make_dirs) { const std::error_code error = utils::create_parent_directories(path); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "create parent directories for", path, error)); + return Error::fail(Error::Code::Io, "create parent directories for", path, error); } } std::ofstream file(path, std::ios::binary); if (!file.is_open()) { - return std::unexpected(::Error::make(::Error::Code::Io, "open file for writing", path)); + return Error::fail(Error::Code::Io, "open file for writing", path); } file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); if (!file.good()) { - return std::unexpected(::Error::make(::Error::Code::Io, "write bytes to", path)); + return Error::fail(Error::Code::Io, "write bytes to", path); } return {}; } -std::expected, ::Error> read_bytes_from_path(const std::filesystem::path& path) +Expected> read_bytes_from_path(const std::filesystem::path& path) { LOG_TRACE("Reading bytes from path {}", path); @@ -40,16 +40,16 @@ std::expected, ::Error> read_bytes_from_path(const std::fil std::error_code existence_error; const bool exists = std::filesystem::exists(path, existence_error); if (!existence_error && !exists) { - return std::unexpected(::Error::make(::Error::Code::NotFound, "open file for reading", path)); + return Error::fail(Error::Code::NotFound, "open file for reading", path); } - return std::unexpected(existence_error - ? ::Error::make(::Error::Code::Io, "check existence of", path, existence_error) - : ::Error::make(::Error::Code::Io, "open file for reading", path)); + return existence_error + ? Error::fail(Error::Code::Io, "check existence of", path, existence_error) + : Error::fail(Error::Code::Io, "open file for reading", path); } const std::streamsize size = file.tellg(); if (size < 0) { - return std::unexpected(::Error::make(::Error::Code::Io, "determine size of", path)); + return Error::fail(Error::Code::Io, "determine size of", path); } std::vector buffer(static_cast(size)); @@ -57,7 +57,7 @@ std::expected, ::Error> read_bytes_from_path(const std::fil file.read(reinterpret_cast(buffer.data()), size); if (!file.good()) { - return std::unexpected(::Error::make(::Error::Code::Io, "read bytes from", path)); + return Error::fail(Error::Code::Io, "read bytes from", path); } return buffer; diff --git a/src/terrainlib/io/bytes.h b/src/terrainlib/io/bytes.h index ebb6f2c9..4d1af334 100644 --- a/src/terrainlib/io/bytes.h +++ b/src/terrainlib/io/bytes.h @@ -10,7 +10,7 @@ namespace io { -std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path& path, bool make_dirs = true); -std::expected, ::Error> read_bytes_from_path(const std::filesystem::path& path); +Expected write_bytes_to_path(const std::span bytes, const std::filesystem::path& path, bool make_dirs = true); +Expected> read_bytes_from_path(const std::filesystem::path& path); } diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index 2c32139e..3789ed68 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -47,7 +47,7 @@ namespace { return checksum; } - std::expected validate_algorithms(const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) + Expected validate_algorithms(const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) { switch (checksum_algorithm) { case ChecksumAlgorithm::None: @@ -55,8 +55,8 @@ namespace { case ChecksumAlgorithm::Crc32c: break; default: - return std::unexpected(::Error::make(::Error::Code::Unsupported, - "unsupported checksum algorithm " + std::to_string(static_cast(checksum_algorithm)))); + return Error::fail(Error::Code::Unsupported, + "unsupported checksum algorithm " + std::to_string(static_cast(checksum_algorithm))); } switch (compression_algorithm) { @@ -65,8 +65,8 @@ namespace { case CompressionAlgorithm::ZstdDefaultCompressionWithChecksum: break; default: - return std::unexpected(::Error::make(::Error::Code::Unsupported, - "unsupported compression algorithm " + std::to_string(static_cast(compression_algorithm)))); + return Error::fail(Error::Code::Unsupported, + "unsupported compression algorithm " + std::to_string(static_cast(compression_algorithm))); } const bool no_compression = compression_algorithm == CompressionAlgorithm::None @@ -75,7 +75,7 @@ namespace { || compression_algorithm == CompressionAlgorithm::ZstdDefaultCompressionWithChecksum) && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib || checksum_algorithm == ChecksumAlgorithm::Crc32c); if (!no_compression && !zstd_with_checksum) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "invalid compression and checksum algorithm combination")); + return Error::fail(Error::Code::InvalidInput, "invalid compression and checksum algorithm combination"); } return {}; @@ -83,14 +83,14 @@ namespace { } // namespace -std::expected compress_with_checksum( +Expected compress_with_checksum( const Bytes& uncompressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) { if (auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { - return std::unexpected(std::move(validation).error()); + return Error::propagate(std::move(validation)); } if (uncompressed_data.size() > default_max_decompressed_size) { - return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "uncompressed data exceeds the compression size limit")); + return Error::fail(Error::Code::ResourceExhausted, "uncompressed data exceeds the compression size limit"); } const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c ? crc32c_checksum(uncompressed_data) : std::string {}; @@ -101,13 +101,13 @@ std::expected compress_with_checksum( CompressionContext context { ZSTD_createCCtx(), &ZSTD_freeCCtx }; if (!context) { - return std::unexpected(::Error::make(::Error::Code::Internal, "failed to create the Zstandard compression context")); + return Error::fail(Error::Code::Internal, "failed to create the Zstandard compression context"); } const int compression_level = compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum ? ZSTD_maxCLevel() : 0; if (ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_compressionLevel, compression_level)) || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { - return std::unexpected(::Error::make(::Error::Code::Internal, "failed to configure the Zstandard compression context")); + return Error::fail(Error::Code::Internal, "failed to configure the Zstandard compression context"); } const std::size_t capacity = ZSTD_compressBound(uncompressed_data.size()); @@ -115,30 +115,30 @@ std::expected compress_with_checksum( const std::size_t compressed_size = ZSTD_compress2(context.get(), compressed_data.data(), compressed_data.size(), uncompressed_data.data(), uncompressed_data.size()); if (ZSTD_isError(compressed_size)) { - return std::unexpected(::Error::make(::Error::Code::Internal, - std::string("Zstandard compression failed: ") + ZSTD_getErrorName(compressed_size))); + return Error::fail(Error::Code::Internal, + std::string("Zstandard compression failed: ") + ZSTD_getErrorName(compressed_size)); } compressed_data.resize(compressed_size); return CompressedData { std::move(compressed_data), checksum }; } -std::expected checked_decompress(const Bytes& compressed_data, +Expected checked_decompress(const Bytes& compressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm, const std::string_view checksum, const std::size_t max_decompressed_size) { if (auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { - return std::unexpected(std::move(validation).error()); + return Error::propagate(std::move(validation)); } if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "a checksum was supplied for an algorithm that does not accept one")); + return Error::fail(Error::Code::InvalidInput, "a checksum was supplied for an algorithm that does not accept one"); } if (checksum_algorithm == ChecksumAlgorithm::Crc32c && (checksum.size() != 8 || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { return (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f'); }))) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid CRC32C checksum encoding")); + return Error::fail(Error::Code::CorruptData, "invalid CRC32C checksum encoding"); } const std::size_t effective_max_decompressed_size = std::min(max_decompressed_size, default_max_decompressed_size); @@ -146,28 +146,28 @@ std::expected checked_decompress(const Bytes& compressed_data, Bytes uncompressed_data; if (compression_algorithm == CompressionAlgorithm::None) { if (compressed_data.size() > effective_max_decompressed_size) { - return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit")); + return Error::fail(Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit"); } uncompressed_data = compressed_data; } else { ZSTD_frameHeader frame_header {}; const std::size_t frame_header_result = ZSTD_getFrameHeader(&frame_header, compressed_data.data(), compressed_data.size()); if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid or incomplete Zstandard frame header")); + return Error::fail(Error::Code::CorruptData, "invalid or incomplete Zstandard frame header"); } if (frame_header.checksumFlag == 0) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "Zstandard frame does not contain the required checksum")); + return Error::fail(Error::Code::CorruptData, "Zstandard frame does not contain the required checksum"); } const unsigned long long frame_content_size = ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); if (frame_content_size == ZSTD_CONTENTSIZE_ERROR || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN && frame_content_size > std::numeric_limits::max())) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid Zstandard frame content size")); + return Error::fail(Error::Code::CorruptData, "invalid Zstandard frame content size"); } const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { - return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit")); + return Error::fail(Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit"); } const std::size_t allocation_size = content_size_is_known ? static_cast(frame_content_size) : effective_max_decompressed_size; @@ -176,23 +176,23 @@ std::expected checked_decompress(const Bytes& compressed_data, = ZSTD_decompress(uncompressed_data.data(), uncompressed_data.size(), compressed_data.data(), compressed_data.size()); if (ZSTD_isError(decompressed_size)) { if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "Zstandard checksum mismatch")); + return Error::fail(Error::Code::CorruptData, "Zstandard checksum mismatch"); } if (!content_size_is_known && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { - return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit")); + return Error::fail(Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit"); } - return std::unexpected(::Error::make(::Error::Code::CorruptData, - std::string("Zstandard decompression failed: ") + ZSTD_getErrorName(decompressed_size))); + return Error::fail(Error::Code::CorruptData, + std::string("Zstandard decompression failed: ") + ZSTD_getErrorName(decompressed_size)); } if (content_size_is_known && decompressed_size != uncompressed_data.size()) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "Zstandard decompressed size does not match its frame header")); + return Error::fail(Error::Code::CorruptData, "Zstandard decompressed size does not match its frame header"); } uncompressed_data.resize(decompressed_size); } if (checksum_algorithm == ChecksumAlgorithm::Crc32c && crc32c_checksum(uncompressed_data) != checksum) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "CRC32C checksum mismatch")); + return Error::fail(Error::Code::CorruptData, "CRC32C checksum mismatch"); } return uncompressed_data; } diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h index f4c6a5ad..5c5b45a2 100644 --- a/src/terrainlib/io/compression.h +++ b/src/terrainlib/io/compression.h @@ -34,10 +34,10 @@ struct CompressedData { std::string checksum; }; -std::expected compress_with_checksum( +Expected compress_with_checksum( const Bytes& uncompressed_data, CompressionAlgorithm compression_algorithm, ChecksumAlgorithm checksum_algorithm); -std::expected checked_decompress(const Bytes& compressed_data, +Expected checked_decompress(const Bytes& compressed_data, CompressionAlgorithm compression_algorithm, ChecksumAlgorithm checksum_algorithm, std::string_view checksum, diff --git a/src/terrainlib/io/conversion.h b/src/terrainlib/io/conversion.h index 49000c7b..578ba09f 100644 --- a/src/terrainlib/io/conversion.h +++ b/src/terrainlib/io/conversion.h @@ -17,10 +17,10 @@ namespace io::conversion { namespace detail { - inline ::Error allocation_error( + inline std::unexpected allocation_error( const std::source_location location = std::source_location::current()) noexcept { - return ::Error::make(::Error::Code::ResourceExhausted, "allocate raster conversion output", location); + return Error::fail(Error::Code::ResourceExhausted, "allocate raster conversion output", location); } template @@ -133,26 +133,26 @@ namespace detail { } // namespace detail template -std::expected, ::Error> to_raster(const cv::Mat& source) +Expected> to_raster(const cv::Mat& source) { if constexpr (!detail::Traits::supported) { - return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported raster pixel type")); + return Error::fail(Error::Code::Unsupported, "unsupported raster pixel type"); } else { if (source.empty()) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "cannot convert an empty OpenCV matrix to a raster")); + return Error::fail(Error::Code::InvalidInput, "cannot convert an empty OpenCV matrix to a raster"); } if (source.dims != 2) { - return std::unexpected(::Error::make(::Error::Code::Unsupported, - "unsupported OpenCV matrix dimension count " + std::to_string(source.dims))); + return Error::fail(Error::Code::Unsupported, + "unsupported OpenCV matrix dimension count " + std::to_string(source.dims)); } if (source.type() != detail::cv_type) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, + return Error::fail(Error::Code::InvalidInput, "OpenCV matrix type " + std::to_string(source.type()) + " does not match expected type " - + std::to_string(detail::cv_type))); + + std::to_string(detail::cv_type)); } if (source.cols < 0 || source.rows < 0 || static_cast(source.cols) > std::numeric_limits::max() || static_cast(source.rows) > std::numeric_limits::max()) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "OpenCV matrix dimensions are outside the raster range")); + return Error::fail(Error::Code::InvalidInput, "OpenCV matrix dimensions are outside the raster range"); } try { @@ -163,25 +163,25 @@ std::expected, ::Error> to_raster(const cv::Mat& source) detail::copy_mat_to_raster(source, destination); return destination; } catch (const std::bad_alloc&) { - return std::unexpected(detail::allocation_error()); + return detail::allocation_error(); } catch (const cv::Exception& error) { - return std::unexpected(::Error::make(::Error::Code::Internal, "OpenCV raster conversion failed: " + error.msg)); + return Error::fail(Error::Code::Internal, "OpenCV raster conversion failed: " + error.msg); } } } template -std::expected to_mat(const radix::Raster& source) +Expected to_mat(const radix::Raster& source) { if constexpr (!detail::Traits::supported) { - return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported raster pixel type")); + return Error::fail(Error::Code::Unsupported, "unsupported raster pixel type"); } else { if (source.width() == 0 || source.height() == 0) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "cannot convert an empty raster to an OpenCV matrix")); + return Error::fail(Error::Code::InvalidInput, "cannot convert an empty raster to an OpenCV matrix"); } if (source.width() > static_cast(std::numeric_limits::max()) || source.height() > static_cast(std::numeric_limits::max())) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "raster dimensions are outside the OpenCV matrix range")); + return Error::fail(Error::Code::InvalidInput, "raster dimensions are outside the OpenCV matrix range"); } try { @@ -189,9 +189,9 @@ std::expected to_mat(const radix::Raster& source) detail::copy_raster_to_mat(source, destination); return destination; } catch (const std::bad_alloc&) { - return std::unexpected(detail::allocation_error()); + return detail::allocation_error(); } catch (const cv::Exception& error) { - return std::unexpected(::Error::make(::Error::Code::Internal, "OpenCV raster conversion failed: " + error.msg)); + return Error::fail(Error::Code::Internal, "OpenCV raster conversion failed: " + error.msg); } } } diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h index 2cce906e..8ce377b6 100644 --- a/src/terrainlib/io/envelope.h +++ b/src/terrainlib/io/envelope.h @@ -115,17 +115,17 @@ struct PayloadSchema { }; template -std::expected serialize(const typename Schema::template payload_type& payload, +Expected serialize(const typename Schema::template payload_type& payload, CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template -std::expected serialize(const typename Schema::latest_type& payload, +Expected serialize(const typename Schema::latest_type& payload, CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template -std::expected deserialize( +Expected deserialize( std::span bytes, std::size_t max_decompressed_size = default_max_decompressed_size); } // namespace io::envelope diff --git a/src/terrainlib/io/envelope.inl b/src/terrainlib/io/envelope.inl index 0831ab77..f71348d0 100644 --- a/src/terrainlib/io/envelope.inl +++ b/src/terrainlib/io/envelope.inl @@ -8,28 +8,28 @@ namespace io::envelope { namespace detail { template -std::expected serialize_to_bytes(const Value& value) +Expected serialize_to_bytes(const Value& value) { Bytes bytes; zpp::bits::out output(bytes, zpp::bits::alloc_limit()); const zpp::bits::errc result = output(value); if (zpp::bits::failure(result)) { const auto cause = std::make_error_code(result.code); - const auto code = result.code == std::errc::not_enough_memory ? ::Error::Code::ResourceExhausted : ::Error::Code::Internal; - return std::unexpected(::Error::make(code, "serialize envelope data", cause)); + const auto code = result.code == std::errc::not_enough_memory ? Error::Code::ResourceExhausted : Error::Code::Internal; + return Error::fail(code, "serialize envelope data", cause); } return bytes; } template -std::expected deserialize_from_bytes(const std::span bytes) +Expected deserialize_from_bytes(const std::span bytes) { Value value{}; zpp::bits::in input(bytes, zpp::bits::alloc_limit()); const zpp::bits::errc result = input(value); if (zpp::bits::failure(result) || input.position() != bytes.size()) { const auto cause = std::make_error_code(zpp::bits::failure(result) ? result.code : std::errc::bad_message); - return std::unexpected(::Error::make(::Error::Code::CorruptData, "deserialize envelope data", cause)); + return Error::fail(Error::Code::CorruptData, "deserialize envelope data", cause); } return value; } @@ -46,7 +46,7 @@ typename Schema::latest_type convert_to_latest(Current current) } template -std::expected deserialize_version( +Expected deserialize_version( const std::uint32_t class_version, const std::span payload_bytes) { @@ -54,7 +54,7 @@ std::expected deserialize_version( if (class_version == CurrentVersion::number) { auto payload = deserialize_from_bytes(payload_bytes); if (!payload) { - return std::unexpected(std::move(payload.error()).with_context("deserializing versioned envelope payload")); + return Error::propagate(std::move(payload), "deserializing versioned envelope payload"); } return convert_to_latest(std::move(*payload)); } @@ -62,15 +62,15 @@ std::expected deserialize_version( if constexpr (Index + 1 < Schema::version_count) { return deserialize_version(class_version, payload_bytes); } else { - return std::unexpected(::Error::make(::Error::Code::Unsupported, - "unsupported envelope class version " + std::to_string(class_version))); + return Error::fail(Error::Code::Unsupported, + "unsupported envelope class version " + std::to_string(class_version)); } } } // namespace detail template -std::expected serialize( +Expected serialize( const typename Schema::template payload_type &payload, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) @@ -80,12 +80,12 @@ std::expected serialize( auto payload_bytes = detail::serialize_to_bytes(payload); if (!payload_bytes) { - return std::unexpected(std::move(payload_bytes.error()).with_context("serializing envelope payload")); + return Error::propagate(std::move(payload_bytes), "serializing envelope payload"); } auto compressed = compress_with_checksum(*payload_bytes, compression_algorithm, checksum_algorithm); if (!compressed) { - return std::unexpected(std::move(compressed.error()).with_context("compressing envelope payload")); + return Error::propagate(std::move(compressed), "compressing envelope payload"); } const Envelope envelope{ @@ -102,7 +102,7 @@ std::expected serialize( } template -std::expected serialize( +Expected serialize( const typename Schema::latest_type &payload, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) @@ -114,31 +114,31 @@ std::expected serialize( } template -std::expected deserialize( +Expected deserialize( const std::span bytes, const std::size_t max_decompressed_size) { auto envelope = detail::deserialize_from_bytes(bytes); if (!envelope) { - return std::unexpected(std::move(envelope.error()).with_context("reading envelope header")); + return Error::propagate(std::move(envelope), "reading envelope header"); } if (envelope->magic != magic) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid envelope magic")); + return Error::fail(Error::Code::CorruptData, "invalid envelope magic"); } if (envelope->class_name != Schema::class_name) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, - "unexpected envelope class \"" + envelope->class_name + "\", expected \"" + std::string(Schema::class_name) + "\"")); + return Error::fail(Error::Code::CorruptData, + "unexpected envelope class \"" + envelope->class_name + "\", expected \"" + std::string(Schema::class_name) + "\""); } if (!Schema::supports_version_number(envelope->class_version)) { - return std::unexpected(::Error::make(::Error::Code::Unsupported, - "unsupported envelope class version " + std::to_string(envelope->class_version))); + return Error::fail(Error::Code::Unsupported, + "unsupported envelope class version " + std::to_string(envelope->class_version)); } const std::size_t effective_max_decompressed_size = std::min(max_decompressed_size, default_max_decompressed_size); if (envelope->uncompressed_size > std::numeric_limits::max() || envelope->uncompressed_size > effective_max_decompressed_size) { - return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "envelope payload exceeds the configured size limit")); + return Error::fail(Error::Code::ResourceExhausted, "envelope payload exceeds the configured size limit"); } auto payload_bytes = checked_decompress( @@ -148,18 +148,18 @@ std::expected deserialize( envelope->checksum, static_cast(envelope->uncompressed_size)); if (!payload_bytes) { - if (payload_bytes.error().code() == ::Error::Code::ResourceExhausted) { - return std::unexpected(std::move(payload_bytes.error()).reclassified( - ::Error::Code::CorruptData, "decompressed payload exceeds the size declared by the envelope")); + if (payload_bytes.error().code() == Error::Code::ResourceExhausted) { + return Error::propagate(std::move(payload_bytes), + Error::Code::CorruptData, "decompressed payload exceeds the size declared by the envelope"); } - if (payload_bytes.error().code() == ::Error::Code::InvalidInput) { - return std::unexpected(std::move(payload_bytes.error()).reclassified( - ::Error::Code::CorruptData, "envelope declares an invalid compression and checksum combination")); + if (payload_bytes.error().code() == Error::Code::InvalidInput) { + return Error::propagate(std::move(payload_bytes), + Error::Code::CorruptData, "envelope declares an invalid compression and checksum combination"); } - return std::unexpected(std::move(payload_bytes.error()).with_context("decompressing envelope payload")); + return Error::propagate(std::move(payload_bytes), "decompressing envelope payload"); } if (payload_bytes->size() != envelope->uncompressed_size) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "decompressed payload size does not match the envelope declaration")); + return Error::fail(Error::Code::CorruptData, "decompressed payload size does not match the envelope declaration"); } return detail::deserialize_version(envelope->class_version, *payload_bytes); diff --git a/src/terrainlib/io/envelope_file.h b/src/terrainlib/io/envelope_file.h index dead8c3a..daa5d5c4 100644 --- a/src/terrainlib/io/envelope_file.h +++ b/src/terrainlib/io/envelope_file.h @@ -14,23 +14,23 @@ namespace io::envelope { template -std::expected read_from_path( +Expected read_from_path( const std::filesystem::path& path, const std::size_t max_decompressed_size = default_max_decompressed_size) { auto file_bytes = ::io::read_bytes_from_path(path); if (!file_bytes) { - return std::unexpected(std::move(file_bytes.error()).with_context("reading envelope file \"" + path.string() + "\"")); + return Error::propagate(std::move(file_bytes), "reading envelope file \"" + path.string() + "\""); } const auto bytes = std::as_bytes(std::span { *file_bytes }); auto result = deserialize(bytes, max_decompressed_size); if (!result) { - return std::unexpected(std::move(result.error()).with_context("decoding envelope file \"" + path.string() + "\"")); + return Error::propagate(std::move(result), "decoding envelope file \"" + path.string() + "\""); } return std::move(*result); } template -std::expected write_to_path(const typename Schema::latest_type& payload, +Expected write_to_path(const typename Schema::latest_type& payload, const std::filesystem::path& path, const bool make_dirs = true, const CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, @@ -38,7 +38,7 @@ std::expected write_to_path(const typename Schema::latest_type& p { auto serialized = serialize(payload, compression_algorithm, checksum_algorithm); if (!serialized) { - return std::unexpected(std::move(serialized.error()).with_context("encoding envelope file \"" + path.string() + "\"")); + return Error::propagate(std::move(serialized), "encoding envelope file \"" + path.string() + "\""); } const auto bytes = std::span { reinterpret_cast(serialized->data()), @@ -46,7 +46,7 @@ std::expected write_to_path(const typename Schema::latest_type& p }; auto result = ::io::write_bytes_to_path(bytes, path, make_dirs); if (!result) { - return std::unexpected(std::move(result.error()).with_context("writing envelope file \"" + path.string() + "\"")); + return Error::propagate(std::move(result), "writing envelope file \"" + path.string() + "\""); } return {}; } diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index 0fe849e5..e78bba02 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -29,27 +29,27 @@ class Gltf final : public store::Codec { return { add_extension(node_path, m_container == GltfContainer::Binary ? ".glb" : ".gltf") }; } - std::expected read(const std::filesystem::path& node_path) const override + Expected read(const std::filesystem::path& node_path) const override { const std::filesystem::path path = paths(node_path).front(); const auto result = mesh::io::gltf::load_from_path(path); if (!result.has_value()) { switch (static_cast(result.error())) { case mesh::io::LoadMeshErrorKind::UnsupportedFormat: - return std::unexpected(::Error::make(::Error::Code::Unsupported, "load unsupported glTF format from", path)); + return Error::fail(Error::Code::Unsupported, "load unsupported glTF format from", path); case mesh::io::LoadMeshErrorKind::FileNotFound: - return std::unexpected(::Error::make(::Error::Code::NotFound, "open glTF file", path)); + return Error::fail(Error::Code::NotFound, "open glTF file", path); case mesh::io::LoadMeshErrorKind::InvalidFormat: - return std::unexpected(::Error::make(::Error::Code::CorruptData, "decode invalid glTF file", path)); + return Error::fail(Error::Code::CorruptData, "decode invalid glTF file", path); case mesh::io::LoadMeshErrorKind::OutOfMemory: - return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "load glTF file: out of memory", path)); + return Error::fail(Error::Code::ResourceExhausted, "load glTF file: out of memory", path); } std::terminate(); } return std::move(result.value()); } - std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override + Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { const std::filesystem::path path = paths(node_path).front(); try { @@ -57,21 +57,21 @@ class Gltf final : public store::Codec { if (!result.has_value()) { switch (static_cast(result.error())) { case mesh::io::SaveMeshErrorKind::UnsupportedFormat: - return std::unexpected(::Error::make(::Error::Code::Unsupported, "write unsupported glTF format to", path)); + return Error::fail(Error::Code::Unsupported, "write unsupported glTF format to", path); case mesh::io::SaveMeshErrorKind::OpenFile: - return std::unexpected(::Error::make(::Error::Code::Io, "open glTF file for writing", path)); + return Error::fail(Error::Code::Io, "open glTF file for writing", path); case mesh::io::SaveMeshErrorKind::WriteFile: - return std::unexpected(::Error::make(::Error::Code::Io, "write glTF file", path)); + return Error::fail(Error::Code::Io, "write glTF file", path); case mesh::io::SaveMeshErrorKind::OutOfMemory: - return std::unexpected(::Error::make(::Error::Code::ResourceExhausted, "write glTF file: out of memory", path)); + return Error::fail(Error::Code::ResourceExhausted, "write glTF file: out of memory", path); } std::terminate(); } return {}; } catch (const std::exception& error) { - return std::unexpected(::Error::make(::Error::Code::Internal, "write glTF file \"" + path.string() + "\": " + error.what())); + return Error::fail(Error::Code::Internal, "write glTF file \"" + path.string() + "\": " + error.what()); } catch (...) { - return std::unexpected(::Error::make(::Error::Code::Internal, "write glTF file \"" + path.string() + "\": unknown exception")); + return Error::fail(Error::Code::Internal, "write glTF file \"" + path.string() + "\": unknown exception"); } } diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h index ac288192..db64bb02 100644 --- a/src/terrainlib/mesh/codec/SfMesh.h +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -17,43 +17,41 @@ class SfMesh final : public store::Codec { return { add_extension(node_path, ".sfmesh") }; } - std::expected read(const std::filesystem::path& node_path) const override + Expected read(const std::filesystem::path& node_path) const override { const std::filesystem::path path = paths(node_path).front(); auto payload = ::io::envelope::read_from_path(path); if (!payload) { - return std::unexpected(std::move(payload).error().with_context("read SF mesh")); + return Error::propagate(std::move(payload), "read SF mesh"); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return std::unexpected( - ::Error::make(::Error::Code::CorruptData, "invalid SF mesh in \"" + path.string() + "\": " + valid.error())); + return Error::fail(Error::Code::CorruptData, "invalid SF mesh in \"" + path.string() + "\": " + valid.error()); } auto decoded = mesh::sf::decode_payload(std::move(*payload)); if (!decoded) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "could not decode SF mesh \"" + path.string() + "\"")); + return Error::fail(Error::Code::CorruptData, "could not decode SF mesh \"" + path.string() + "\""); } return std::move(*decoded); } - std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override + Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { return write(node_path, mesh, {}); } - std::expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const + Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const { const std::filesystem::path path = paths(node_path).front(); auto payload = mesh::sf::encode_payload(mesh, options); if (!payload) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "could not encode SF mesh \"" + path.string() + "\"")); + return Error::fail(Error::Code::InvalidInput, "could not encode SF mesh \"" + path.string() + "\""); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return std::unexpected( - ::Error::make(::Error::Code::InvalidInput, "invalid SF mesh for \"" + path.string() + "\": " + valid.error())); + return Error::fail(Error::Code::InvalidInput, "invalid SF mesh for \"" + path.string() + "\": " + valid.error()); } auto result = ::io::envelope::write_to_path(*payload, path); if (!result) { - return std::unexpected(std::move(result).error().with_context("write SF mesh")); + return Error::propagate(std::move(result), "write SF mesh"); } return {}; } diff --git a/src/terrainlib/mesh/codec/from_extension.h b/src/terrainlib/mesh/codec/from_extension.h index d8e5ae7c..58d91153 100644 --- a/src/terrainlib/mesh/codec/from_extension.h +++ b/src/terrainlib/mesh/codec/from_extension.h @@ -12,7 +12,7 @@ namespace mesh::codec { -inline std::expected>, ::Error> from_extension(const std::string_view extension) +inline Expected>> from_extension(const std::string_view extension) { if (extension == ".sfmesh") { return std::make_unique(); @@ -23,7 +23,7 @@ inline std::expected>, ::Error> from_ if (extension == ".gltf") { return std::make_unique(GltfContainer::Json); } - return std::unexpected(::Error::make(::Error::Code::Unsupported, "unsupported mesh codec selector: " + std::string(extension))); + return Error::fail(Error::Code::Unsupported, "unsupported mesh codec selector: " + std::string(extension)); } } // namespace mesh::codec diff --git a/src/terrainlib/mesh/storage.h b/src/terrainlib/mesh/storage.h index 13536bff..8a84830d 100644 --- a/src/terrainlib/mesh/storage.h +++ b/src/terrainlib/mesh/storage.h @@ -18,17 +18,17 @@ using Storage = store::Storage; using IndexedStorage = store::IndexedStorage; using OpenOptions = octree::storage::OpenOptions; -inline std::expected open_index(const std::filesystem::path& path) +inline Expected open_index(const std::filesystem::path& path) { return store::open_index(path, octree::storage::index_format(), payload_class, mesh::codec::from_extension); } -inline std::expected open_folder(const std::filesystem::path& path, OpenOptions options = {}) +inline Expected open_folder(const std::filesystem::path& path, OpenOptions options = {}) { return octree::storage::open_folder(path, std::string(payload_class), ".sfmesh", mesh::codec::from_extension, std::move(options)); } -inline std::expected open_folder_indexed(const std::filesystem::path& path, OpenOptions options = {}) +inline Expected open_folder_indexed(const std::filesystem::path& path, OpenOptions options = {}) { return octree::storage::open_folder_indexed(path, std::string(payload_class), ".sfmesh", mesh::codec::from_extension, std::move(options)); } diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 5eb80290..a1c42f2e 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -123,33 +123,33 @@ inline std::expected, std::string> decode_index(const return result; } -inline std::expected read_store_metadata(const std::filesystem::path& base_path) +inline Expected read_store_metadata(const std::filesystem::path& base_path) { const std::filesystem::path path = base_path / metadata_file_name; auto metadata = io::envelope::read_from_path(path); if (!metadata) { - return std::unexpected(std::move(metadata).error()); + return Error::propagate(std::move(metadata)); } if (auto valid = validate(*metadata); !valid) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid store metadata in " + path.string() + ": " + valid.error())); + return Error::fail(Error::Code::CorruptData, "invalid store metadata in " + path.string() + ": " + valid.error()); } return std::move(*metadata); } -inline std::expected, ::Error> read_index_file(const std::filesystem::path& path) +inline Expected> read_index_file(const std::filesystem::path& path) { auto metadata = read_store_metadata(path.parent_path()); if (!metadata) { - return std::unexpected(std::move(metadata).error()); + return Error::propagate(std::move(metadata)); } auto encoded_index = io::envelope::read_from_path(path); if (!encoded_index) { - return std::unexpected(std::move(encoded_index).error()); + return Error::propagate(std::move(encoded_index)); } auto index = decode_index(*encoded_index); if (!index) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "invalid store index in " + path.string() + ": " + index.error())); + return Error::fail(Error::Code::CorruptData, "invalid store index in " + path.string() + ": " + index.error()); } return store::IndexMetadata { std::move(*index), @@ -159,7 +159,7 @@ inline std::expected, ::Error> read_index_file }; } -inline std::expected write_index_file(const std::filesystem::path& path, const store::IndexMetadata& metadata) +inline Expected write_index_file(const std::filesystem::path& path, const store::IndexMetadata& metadata) { const StoreMetadata store_metadata { .layout_id = metadata.layout_id, @@ -167,16 +167,16 @@ inline std::expected write_index_file(const std::filesystem::path .codec_selector = metadata.codec_selector, }; if (auto valid = validate(store_metadata); !valid) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "invalid store metadata: " + valid.error())); + return Error::fail(Error::Code::InvalidInput, "invalid store metadata: " + valid.error()); } const std::filesystem::path metadata_path = path.parent_path() / metadata_file_name; if (auto written = io::envelope::write_to_path(store_metadata, metadata_path); !written) { - return std::unexpected(std::move(written).error().with_context("write octree store metadata " + metadata_path.string())); + return Error::propagate(std::move(written), "write octree store metadata " + metadata_path.string()); } const StoreIndex encoded_index = encode_index(metadata.index); if (auto written = io::envelope::write_to_path(encoded_index, path); !written) { - return std::unexpected(std::move(written).error().with_context("write octree store index " + path.string())); + return Error::propagate(std::move(written), "write octree store index " + path.string()); } return {}; } diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index e8f78ddd..236948ab 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -20,7 +20,7 @@ struct OpenOptions { }; template -std::expected, ::Error> open_folder(const std::filesystem::path& base_path, +Expected> open_folder(const std::filesystem::path& base_path, std::string payload_class, std::string default_codec_selector, CodecResolver resolve_codec, @@ -29,16 +29,16 @@ std::expected, ::Error> open_folder(const std::error_code error; const bool base_exists = std::filesystem::exists(base_path, error); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "check existence of", base_path, error)); + return Error::fail(Error::Code::Io, "check existence of", base_path, error); } if (base_exists && !std::filesystem::is_directory(base_path, error)) { - return std::unexpected(::Error::make( - ::Error::Code::Io, "check directory", base_path, error ? error : std::make_error_code(std::errc::not_a_directory))); + return Error::fail( + Error::Code::Io, "check directory", base_path, error ? error : std::make_error_code(std::errc::not_a_directory)); } if (!base_exists) { std::filesystem::create_directories(base_path, error); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "create directories", base_path, error)); + return Error::fail(Error::Code::Io, "create directories", base_path, error); } } @@ -47,21 +47,21 @@ std::expected, ::Error> open_folder(const const std::filesystem::path metadata_path = base_path / metadata_file_name; const bool index_exists = std::filesystem::exists(index_path, error); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "check existence of", index_path, error)); + return Error::fail(Error::Code::Io, "check existence of", index_path, error); } const bool metadata_exists = std::filesystem::exists(metadata_path, error); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "check existence of", metadata_path, error)); + return Error::fail(Error::Code::Io, "check existence of", metadata_path, error); } if (index_exists != metadata_exists) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, + return Error::fail(Error::Code::CorruptData, "octree dataset metadata and index must either both exist or both be absent: " - + (index_exists ? metadata_path : index_path).string())); + + (index_exists ? metadata_path : index_path).string()); } if (index_exists) { auto indexed = store::open_index(index_path, format, payload_class, resolve_codec); if (!indexed) { - return std::unexpected(std::move(indexed).error()); + return Error::propagate(std::move(indexed)); } return store::Storage(std::move(*indexed)); } @@ -70,14 +70,14 @@ std::expected, ::Error> open_folder(const std::string codec_selector = options.preferred_extension.value_or(std::move(default_codec_selector)); auto codec = resolve_codec(codec_selector); if (!codec) { - return std::unexpected(std::move(codec).error()); + return Error::propagate(std::move(codec)); } return store::make_storage(base_path, format, mapping, std::move(payload_class), std::move(codec_selector), std::move(*codec)); } template -std::expected, ::Error> open_folder_indexed(const std::filesystem::path& base_path, +Expected> open_folder_indexed(const std::filesystem::path& base_path, std::string payload_class, std::string default_codec_selector, CodecResolver resolve_codec, @@ -85,7 +85,7 @@ std::expected, ::Error> open_folder { auto result = open_folder(base_path, std::move(payload_class), std::move(default_codec_selector), std::move(resolve_codec), std::move(options)); if (!result) { - return std::unexpected(std::move(result).error()); + return Error::propagate(std::move(result)); } return store::IndexedStorage(std::move(*result)); } diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h index 3fc7eaa3..1fe469cd 100644 --- a/src/terrainlib/sf/finalize_storage.h +++ b/src/terrainlib/sf/finalize_storage.h @@ -9,7 +9,7 @@ namespace sf { -inline std::expected finalize_storage(mesh::storage::Storage& storage) +inline Expected finalize_storage(mesh::storage::Storage& storage) { auto save_result = storage.save_or_create_index(); if (!save_result.has_value()) { diff --git a/src/terrainlib/sf/validate_index.h b/src/terrainlib/sf/validate_index.h index b9507b9c..b0d3ad8d 100644 --- a/src/terrainlib/sf/validate_index.h +++ b/src/terrainlib/sf/validate_index.h @@ -8,12 +8,12 @@ namespace sf { -inline std::expected validate_index(const store::Index& index) +inline Expected validate_index(const store::Index& index) { for (const auto& [key, status] : index) { if (status == store::NodeStatus::Inner) { - return std::unexpected(::Error::make( - ::Error::Code::CorruptData, "Structura Fundamentalis topology contains Inner node " + key.to_string())); + return Error::fail( + Error::Code::CorruptData, "Structura Fundamentalis topology contains Inner node " + key.to_string()); } } return {}; diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h index e7ff4c0c..7b8c01c3 100644 --- a/src/terrainlib/store/Codec.h +++ b/src/terrainlib/store/Codec.h @@ -17,14 +17,14 @@ class Codec { virtual std::vector paths(const std::filesystem::path& node_path) const = 0; - virtual std::expected read(const std::filesystem::path&) const + virtual Expected read(const std::filesystem::path&) const { - return std::unexpected(::Error::make(::Error::Code::Unsupported, "codec does not support reading")); + return Error::fail(Error::Code::Unsupported, "codec does not support reading"); } - virtual std::expected write(const std::filesystem::path&, const NodeData&) const + virtual Expected write(const std::filesystem::path&, const NodeData&) const { - return std::unexpected(::Error::make(::Error::Code::Unsupported, "codec does not support writing")); + return Error::fail(Error::Code::Unsupported, "codec does not support writing"); } protected: diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h index 85850f42..8174a3bf 100644 --- a/src/terrainlib/store/Index.h +++ b/src/terrainlib/store/Index.h @@ -22,10 +22,10 @@ class Index { using iterator = typename Container::iterator; using const_iterator = typename Container::const_iterator; - std::expected, ::Error> get(const Key& key) const + Expected> get(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } const NodeStatus* status = get_raw_unchecked(key); if (status == nullptr) { @@ -34,10 +34,10 @@ class Index { return *status; } - std::expected add(const Key& key) + Expected add(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } NodeStatus* status = get_raw_unchecked(key); @@ -63,13 +63,13 @@ class Index { } if (!Traits::is_valid(parent.value())) { - return std::unexpected(store::invalid_key_error(parent.value())); + return store::invalid_key_error(parent.value()); } NodeStatus* parent_status = get_raw_unchecked(parent.value()); if (parent_status == nullptr) { auto parent_result = add(parent.value()); if (!parent_result.has_value()) { - return std::unexpected(std::move(parent_result).error()); + return Error::propagate(std::move(parent_result)); } set_raw_unchecked(parent.value(), NodeStatus::Virtual); set_raw_unchecked(key, NodeStatus::Leaf); @@ -89,10 +89,10 @@ class Index { return true; } - std::expected remove(const Key& key) + Expected remove(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } NodeStatus* status = get_raw_unchecked(key); @@ -112,56 +112,56 @@ class Index { UNREACHABLE(); } - std::expected is_present(const Key& key) const + Expected is_present(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } return m_index.contains(key); } - std::expected is_absent(const Key& key) const + Expected is_absent(const Key& key) const { auto present = is_present(key); if (!present.has_value()) { - return std::unexpected(std::move(present).error()); + return Error::propagate(std::move(present)); } return !present.value(); } - std::expected is(const NodeStatus status, const Key& key) const + Expected is(const NodeStatus status, const Key& key) const { auto result = get(key); if (!result.has_value()) { - return std::unexpected(std::move(result).error()); + return Error::propagate(std::move(result)); } return result.value() == status; } - std::expected get_raw(const Key& key) + Expected get_raw(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } return get_raw_unchecked(key); } - std::expected get_raw(const Key& key) const + Expected get_raw(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } return get_raw_unchecked(key); } - std::expected set_raw(const Key& key, const NodeStatus status) + Expected set_raw(const Key& key, const NodeStatus status) { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } set_raw_unchecked(key, status); return {}; } - std::expected remove_raw(const Key& key) + Expected remove_raw(const Key& key) { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } remove_raw_unchecked(key); return {}; @@ -190,14 +190,14 @@ class Index { void set_raw_unchecked(const Key& key, const NodeStatus status) { m_index[key] = status; } void remove_raw_unchecked(const Key& key) { m_index.erase(key); } - std::expected update_parent_after_remove(const Key& key) + Expected update_parent_after_remove(const Key& key) { const auto parent = Traits::parent(key); if (!parent.has_value()) { return true; } if (!Traits::is_valid(parent.value())) { - return std::unexpected(store::invalid_key_error(parent.value())); + return store::invalid_key_error(parent.value()); } NodeStatus* parent_status = get_raw_unchecked(parent.value()); diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h index df8c79e6..71c04944 100644 --- a/src/terrainlib/store/IndexFormat.h +++ b/src/terrainlib/store/IndexFormat.h @@ -25,8 +25,8 @@ template struct IndexFormat { std::string_view index_filename; - std::expected, ::Error> (*read)(const std::filesystem::path& index_path); - std::expected (*write)(const std::filesystem::path& index_path, const IndexMetadata& metadata); + Expected> (*read)(const std::filesystem::path& index_path); + Expected (*write)(const std::filesystem::path& index_path, const IndexMetadata& metadata); std::optional> (*mapping_from_id)(std::string_view id); path_layout::Mapping (*default_mapping)(); }; diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index 7d42642e..1d30361c 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -32,35 +32,35 @@ class RawStorage { RawStorage(RawStorage&&) noexcept = default; RawStorage& operator=(RawStorage&&) noexcept = default; - std::expected load(const Key& key) const + Expected load(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } return m_codec->read(m_layout.node_path(key)); } - std::expected save(const Key& key, const NodeData& data) const + Expected save(const Key& key, const NodeData& data) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } return m_codec->write(m_layout.node_path(key), data); } - std::expected, ::Error> paths(const Key& key) const + Expected> paths(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } return m_codec->paths(m_layout.node_path(key)); } - std::expected has(const Key& key) const + Expected has(const Key& key) const { auto node_paths = paths(key); if (!node_paths.has_value()) { - return std::unexpected(std::move(node_paths).error()); + return Error::propagate(std::move(node_paths)); } if (node_paths->empty()) { return false; @@ -69,7 +69,7 @@ class RawStorage { std::error_code error; const bool exists = std::filesystem::exists(path, error); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "check existence of", path, error)); + return Error::fail(Error::Code::Io, "check existence of", path, error); } if (!exists) { return false; @@ -78,47 +78,47 @@ class RawStorage { return true; } - std::expected remove(const Key& key) const + Expected remove(const Key& key) const { auto node_paths = paths(key); if (!node_paths.has_value()) { - return std::unexpected(std::move(node_paths).error()); + return Error::propagate(std::move(node_paths)); } bool removed = false; for (const auto& path : node_paths.value()) { std::error_code error; removed = std::filesystem::remove(path, error) || removed; if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "remove", path, error)); + return Error::fail(Error::Code::Io, "remove", path, error); } } return removed; } - std::expected copy_from(const Key& key, const RawStorage& source) const + Expected copy_from(const Key& key, const RawStorage& source) const { - return copy_from(key, source, []() -> std::expected { return {}; }); + return copy_from(key, source, []() -> Expected { return {}; }); } template - std::expected copy_from(const Key& key, const RawStorage& source, BeforeModify&& before_modify) const + Expected copy_from(const Key& key, const RawStorage& source, BeforeModify&& before_modify) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } auto source_exists = source.has(key); if (!source_exists.has_value()) { - return std::unexpected(std::move(source_exists).error()); + return Error::propagate(std::move(source_exists)); } if (!source_exists.value()) { - return std::unexpected(::Error::make(::Error::Code::NotFound, "source node " + Traits::key_to_string(key) + " is missing")); + return Error::fail(Error::Code::NotFound, "source node " + Traits::key_to_string(key) + " is missing"); } const std::filesystem::path probe("__codec_probe__/node"); if (m_codec->paths(probe) != source.m_codec->paths(probe)) { auto loaded = source.m_codec->read(source.m_layout.node_path(key)); if (!loaded.has_value()) { - return std::unexpected(std::move(loaded).error().with_context("read source node while copying")); + return Error::propagate(std::move(loaded), "read source node while copying"); } const auto prepared = before_modify(); if (!prepared.has_value()) { @@ -126,7 +126,7 @@ class RawStorage { } auto written = m_codec->write(m_layout.node_path(key), loaded.value()); if (!written.has_value()) { - return std::unexpected(std::move(written).error().with_context("write target node while copying")); + return Error::propagate(std::move(written), "write target node while copying"); } return {}; } @@ -134,7 +134,7 @@ class RawStorage { const auto source_paths = source.m_codec->paths(source.m_layout.node_path(key)); const auto target_paths = m_codec->paths(m_layout.node_path(key)); if (source_paths.size() != target_paths.size()) { - return std::unexpected(::Error::make(::Error::Code::Internal, "matching codec probes produced different actual path counts")); + return Error::fail(Error::Code::Internal, "matching codec probes produced different actual path counts"); } if (source_paths == target_paths) { return {}; @@ -147,16 +147,16 @@ class RawStorage { std::error_code error; std::filesystem::remove(target_paths[index], error); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "remove", target_paths[index], error)); + return Error::fail(Error::Code::Io, "remove", target_paths[index], error); } std::filesystem::create_directories(target_paths[index].parent_path(), error); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "create directories", target_paths[index].parent_path(), error)); + return Error::fail(Error::Code::Io, "create directories", target_paths[index].parent_path(), error); } std::filesystem::create_hard_link(source_paths[index], target_paths[index], error); if (error) { - return std::unexpected(::Error::make( - ::Error::Code::Io, "create hard link", source_paths[index], target_paths[index], error)); + return Error::fail( + Error::Code::Io, "create hard link", source_paths[index], target_paths[index], error); } } return {}; diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index 33ee67b5..b77473af 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -78,38 +78,37 @@ class Storage { return *this; } - std::expected load(const Key& key) const + Expected load(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } if (m_index.has_value()) { auto status = m_index->get(key); if (!status.has_value()) { - return std::unexpected(std::move(status).error()); + return Error::propagate(std::move(status)); } if (!status->has_value() || status->value() == NodeStatus::Virtual) { - return std::unexpected( - ::Error::make(::Error::Code::NotFound, "node " + Traits::key_to_string(key) + " is not physically present in the index")); + return Error::fail(Error::Code::NotFound, "node " + Traits::key_to_string(key) + " is not physically present in the index"); } } return m_raw.load(key); } - std::expected save(const Key& key, const NodeData& data) + Expected save(const Key& key, const NodeData& data) { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } auto exists = has(key); if (!exists.has_value()) { - return std::unexpected(std::move(exists).error()); + return Error::propagate(std::move(exists)); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); - return std::unexpected(::Error::make(::Error::Code::AlreadyExists, + return Error::fail(Error::Code::AlreadyExists, "save node to", - node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front())); + node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front()); } auto result = m_raw.save(key, data); @@ -119,35 +118,35 @@ class Storage { if (m_index.has_value()) { auto added = m_index->add(key); if (!added.has_value()) { - return std::unexpected(std::move(added).error()); + return Error::propagate(std::move(added)); } m_dirty = m_dirty || added.value(); } return {}; } - std::expected copy_from(const Key& key, const Storage& source) + Expected copy_from(const Key& key, const Storage& source) { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } auto exists = has(key); if (!exists.has_value()) { - return std::unexpected(std::move(exists).error()); + return Error::propagate(std::move(exists)); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); - return std::unexpected(::Error::make(::Error::Code::AlreadyExists, + return Error::fail(Error::Code::AlreadyExists, "copy node to", - node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front())); + node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front()); } - const auto prepare_target = [&]() -> std::expected { + const auto prepare_target = [&]() -> Expected { if (!exists.value() || !m_index.has_value()) { return {}; } auto removed = m_index->remove(key); if (!removed.has_value()) { - return std::unexpected(std::move(removed).error()); + return Error::propagate(std::move(removed)); } m_dirty = m_dirty || removed.value(); return {}; @@ -160,14 +159,14 @@ class Storage { if (m_index.has_value()) { auto added = m_index->add(key); if (!added.has_value()) { - return std::unexpected(std::move(added).error()); + return Error::propagate(std::move(added)); } m_dirty = m_dirty || added.value(); } return {}; } - std::expected remove(const Key& key) + Expected remove(const Key& key) { auto removed = m_raw.remove(key); if (!removed.has_value()) { @@ -176,35 +175,35 @@ class Storage { if (m_index.has_value()) { auto index_removed = m_index->remove(key); if (!index_removed.has_value()) { - return std::unexpected(std::move(index_removed).error()); + return Error::propagate(std::move(index_removed)); } m_dirty = m_dirty || index_removed.value(); } return removed.value(); } - std::expected has(const Key& key) const + Expected has(const Key& key) const { if (!Traits::is_valid(key)) { - return std::unexpected(store::invalid_key_error(key)); + return store::invalid_key_error(key); } if (!m_index.has_value()) { return m_raw.has(key); } auto status = m_index->get(key); if (!status.has_value()) { - return std::unexpected(std::move(status).error()); + return Error::propagate(std::move(status)); } return status->has_value() && status->value() != NodeStatus::Virtual; } - std::expected, ::Error> paths(const Key& key) const { return m_raw.paths(key); } + Expected> paths(const Key& key) const { return m_raw.paths(key); } - std::expected path_for(const Key& key) const + Expected path_for(const Key& key) const { auto node_paths = paths(key); if (!node_paths.has_value()) { - return std::unexpected(std::move(node_paths).error()); + return Error::propagate(std::move(node_paths)); } return node_paths->empty() ? m_raw.layout().node_path(key) : node_paths->front(); } @@ -237,13 +236,13 @@ class Storage { return std::cref(m_index.value()); } - std::expected save_index() const + Expected save_index() const { if (!m_dirty) { return {}; } if (!m_index.has_value() || !m_persistence.has_value()) { - return std::unexpected(::Error::make(::Error::Code::Internal, "indexed storage has no persistence configuration")); + return Error::fail(Error::Code::Internal, "indexed storage has no persistence configuration"); } const IndexMetadata metadata { m_index.value(), @@ -258,7 +257,7 @@ class Storage { return result; } - std::expected save_or_create_index() + Expected save_or_create_index() { if (!m_index.has_value()) { m_index.emplace(); diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h index 438df4bf..b776d17e 100644 --- a/src/terrainlib/store/Traits.h +++ b/src/terrainlib/store/Traits.h @@ -20,10 +20,10 @@ concept HierarchyTraits = requires(typename Traits::Key key) { }; template -::Error invalid_key_error(const typename Traits::Key& key, +std::unexpected invalid_key_error(const typename Traits::Key& key, const std::source_location location = std::source_location::current()) { - return ::Error::make(::Error::Code::InvalidInput, "invalid hierarchy key " + Traits::key_to_string(key), location); + return Error::fail(Error::Code::InvalidInput, "invalid hierarchy key " + Traits::key_to_string(key), location); } } // namespace store diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h index 9af6f369..8c8e1597 100644 --- a/src/terrainlib/store/open.h +++ b/src/terrainlib/store/open.h @@ -12,32 +12,32 @@ namespace store { template -std::expected, ::Error> open_index( +Expected> open_index( const std::filesystem::path& index_path, const IndexFormat format, const std::string_view expected_payload_class, CodecResolver&& resolve_codec) { using Key = typename Traits::Key; auto metadata_result = format.read(index_path); if (!metadata_result.has_value()) { - return std::unexpected(std::move(metadata_result).error()); + return Error::propagate(std::move(metadata_result)); } IndexMetadata metadata = std::move(metadata_result.value()); if (metadata.payload_class != expected_payload_class) { - return std::unexpected(::Error::make(::Error::Code::Unsupported, "unexpected payload class: " + metadata.payload_class)); + return Error::fail(Error::Code::Unsupported, "unexpected payload class: " + metadata.payload_class); } for (const auto& [key, status] : metadata.index) { static_cast(status); if (!Traits::is_valid(key)) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "index contains invalid hierarchy key " + Traits::key_to_string(key))); + return Error::fail(Error::Code::CorruptData, "index contains invalid hierarchy key " + Traits::key_to_string(key)); } } const auto mapping = format.mapping_from_id(metadata.layout_id); if (!mapping.has_value()) { - return std::unexpected(::Error::make(::Error::Code::Unsupported, "unknown storage layout: " + metadata.layout_id)); + return Error::fail(Error::Code::Unsupported, "unknown storage layout: " + metadata.layout_id); } auto codec = resolve_codec(metadata.codec_selector); if (!codec.has_value()) { - return std::unexpected(std::move(codec).error()); + return Error::propagate(std::move(codec)); } const std::filesystem::path base_path = index_path.parent_path(); @@ -54,7 +54,7 @@ std::expected, ::Error> open_index( } template -std::expected, ::Error> make_storage(const std::filesystem::path& base_path, +Expected> make_storage(const std::filesystem::path& base_path, const IndexFormat format, const path_layout::Mapping mapping, std::string payload_class, @@ -65,7 +65,7 @@ std::expected, ::Error> make_storage(const std::filesy std::error_code error; std::filesystem::create_directories(base_path, error); if (error) { - return std::unexpected(::Error::make(::Error::Code::Io, "create directories", base_path, error)); + return Error::fail(Error::Code::Io, "create directories", base_path, error); } typename Storage::Persistence persistence { diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h index ecf618b7..85cc533c 100644 --- a/src/terrainlib/store/traverse.h +++ b/src/terrainlib/store/traverse.h @@ -23,7 +23,7 @@ struct AlwaysRefine { }; template -std::expected traverse(const Index& index, +Expected traverse(const Index& index, VisitFn&& visit, RefineFn&& refine = {}, const typename Traits::Key& root = Traits::root(), @@ -31,22 +31,22 @@ std::expected traverse(const Index& index, { using Key = typename Traits::Key; if (!Traits::is_valid(root)) { - return std::unexpected(store::invalid_key_error(root)); + return store::invalid_key_error(root); } auto root_status = index.get(root); if (!root_status.has_value()) { - return std::unexpected(std::move(root_status).error()); + return Error::propagate(std::move(root_status)); } if (!root_status.value().has_value()) { return {}; } if (order == TraversalOrder::DepthFirst) { - std::function(const Key&)> depth_first; - depth_first = [&](const Key& current) -> std::expected { + std::function(const Key&)> depth_first; + depth_first = [&](const Key& current) -> Expected { auto status = index.get(current); if (!status.has_value()) { - return std::unexpected(std::move(status).error()); + return Error::propagate(std::move(status)); } if (!status.value().has_value()) { return {}; @@ -77,7 +77,7 @@ std::expected traverse(const Index& index, auto status = index.get(current); if (!status.has_value()) { - return std::unexpected(std::move(status).error()); + return Error::propagate(std::move(status)); } if (!status.value().has_value()) { continue; diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 341dec90..f23bca02 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -22,6 +22,7 @@ add_executable(unittests_terrainlib terrainlib/dataset.cpp terrainlib/enumerate.cpp terrainlib/envelope.cpp + terrainlib/error.cpp terrainlib/fixed_vector.cpp terrainlib/geometry_frames.cpp terrainlib/grid.cpp diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index dac5a7bc..941ed1dd 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -83,7 +83,7 @@ TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") const auto result = codec.write(node_path, batch); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::InvalidInput); + CHECK(result.error().code() == Error::Code::InvalidInput); CHECK_FALSE(std::filesystem::exists(node_paths[0])); CHECK_FALSE(std::filesystem::exists(node_paths[1])); } @@ -99,7 +99,7 @@ TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") const auto result = codec.read(node_path); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::CorruptData); + CHECK(result.error().code() == Error::Code::CorruptData); } } @@ -131,11 +131,11 @@ TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[stor CHECK(metadata_codec.value()->paths("node") == std::vector { "node.dagmeta" }); const auto write_result = metadata_codec.value()->write("node", dag::NodeMetadata {}); REQUIRE_FALSE(write_result.has_value()); - CHECK(write_result.error().code() == ::Error::Code::Unsupported); + CHECK(write_result.error().code() == Error::Code::Unsupported); const auto unknown = dag::codec::from_extension(".unknown"); REQUIRE_FALSE(unknown.has_value()); - CHECK(unknown.error().code() == ::Error::Code::Unsupported); + CHECK(unknown.error().code() == Error::Code::Unsupported); } TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") @@ -218,7 +218,7 @@ TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf const auto built = dag::build_full(input, output, options); REQUIRE_FALSE(built.has_value()); - CHECK(built.error().code() == ::Error::Code::CorruptData); + CHECK(built.error().code() == Error::Code::CorruptData); CHECK(built.error().to_string().contains(root.to_string())); CHECK(output.index().empty()); } diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp index db713283..f75c370d 100644 --- a/unittests/sf_builder/storage_finalization.cpp +++ b/unittests/sf_builder/storage_finalization.cpp @@ -40,7 +40,7 @@ TEST_CASE("SF builder finalization retains an invalid written index for diagnosi const auto finalized = sf::finalize_storage(storage); REQUIRE_FALSE(finalized.has_value()); - CHECK(finalized.error().code() == ::Error::Code::CorruptData); + CHECK(finalized.error().code() == Error::Code::CorruptData); CHECK(finalized.error().to_string().contains(root.to_string())); CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp index 6c34aca7..bca78214 100644 --- a/unittests/sf_merger/storage_boundaries.cpp +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -80,7 +80,7 @@ TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][valida const auto merged = merge_datasets(left, right, output); REQUIRE_FALSE(merged.has_value()); - CHECK(merged.error().code() == ::Error::Code::CorruptData); + CHECK(merged.error().code() == Error::Code::CorruptData); CHECK(merged.error().to_string().contains(root.to_string())); CHECK_FALSE(std::filesystem::exists(output_directory.path() / "octree.storeindex")); } @@ -99,7 +99,7 @@ TEST_CASE("SF merge retains an invalid output index for diagnosis", "[sfmerger][ const auto merged = merge_datasets(left, right, output); REQUIRE_FALSE(merged.has_value()); - CHECK(merged.error().code() == ::Error::Code::CorruptData); + CHECK(merged.error().code() == Error::Code::CorruptData); CHECK(merged.error().to_string().contains(root.to_string())); CHECK(std::filesystem::is_regular_file(output_directory.path() / "octree.storeindex")); } diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index 00734344..06b21391 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -121,7 +121,7 @@ io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes& unc } template -void check_error(const Result& result, const ::Error::Code expected) +void check_error(const Result& result, const Error::Code expected) { REQUIRE_FALSE(result.has_value()); CHECK(result.error().code() == expected); @@ -261,7 +261,7 @@ TEST_CASE("CRC-32C protects independently compressed data") corrupted.front() ^= std::byte { 0x01 }; check_error(io::envelope::checked_decompress( corrupted, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, compressed->checksum), - ::Error::Code::CorruptData); + Error::Code::CorruptData); } SECTION("with zstd compression") @@ -281,16 +281,16 @@ TEST_CASE("CRC-32C protects independently compressed data") wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; check_error( io::envelope::checked_decompress(compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c, wrong_checksum), - ::Error::Code::CorruptData); + Error::Code::CorruptData); } } SECTION("malformed checksum") { check_error(io::envelope::checked_decompress(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, "E3069283"), - ::Error::Code::CorruptData); + Error::Code::CorruptData); check_error(io::envelope::checked_decompress(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, {}), - ::Error::Code::CorruptData); + Error::Code::CorruptData); } } @@ -318,7 +318,7 @@ TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") auto corrupted = envelope; corrupted.compressed_data.front() ^= std::byte { 0x01 }; - check_error(io::envelope::deserialize(encode_envelope(corrupted)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(corrupted)), Error::Code::CorruptData); } SECTION("older version") @@ -364,11 +364,11 @@ TEST_CASE("Checked compression round trips and validates its checksum") auto corrupted = compressed->compressed_data; corrupted.back() ^= std::byte { 0x01 }; check_error(io::envelope::checked_decompress(corrupted, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}), - ::Error::Code::CorruptData); + Error::Code::CorruptData); check_error(io::envelope::checked_decompress( compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size() - 1), - ::Error::Code::ResourceExhausted); + Error::Code::ResourceExhausted); } } @@ -386,7 +386,7 @@ TEST_CASE("Checked decompression uses its maximum when the format omits the cont check_error(io::envelope::checked_decompress( compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size() - 1), - ::Error::Code::ResourceExhausted); + Error::Code::ResourceExhausted); } } @@ -400,49 +400,49 @@ TEST_CASE("Envelope rejects incompatible metadata") { auto envelope = decode_envelope(*serialized); ++envelope.magic; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("class name") { auto envelope = decode_envelope(*serialized); envelope.class_name = "other.Payload"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("class version") { auto envelope = decode_envelope(*serialized); envelope.class_version = 99; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::Unsupported); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::Unsupported); } SECTION("checksum algorithm") { auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = static_cast(99); - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::Unsupported); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::Unsupported); } SECTION("compression algorithm") { auto envelope = decode_envelope(*serialized); envelope.compression_algorithm = static_cast(99); - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::Unsupported); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::Unsupported); } SECTION("algorithm combination") { auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::None; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("external checksum") { auto envelope = decode_envelope(*serialized); envelope.checksum = "not used by zstd"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("malformed CRC-32C checksum") @@ -450,7 +450,7 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; envelope.checksum = "1234"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("incorrect CRC-32C checksum") @@ -458,7 +458,7 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; envelope.checksum = "00000000"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("uncompressed size is smaller than the payload") @@ -466,7 +466,7 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); --envelope.uncompressed_size; const auto result = io::envelope::deserialize(encode_envelope(envelope)); - check_error(result, ::Error::Code::CorruptData); + check_error(result, Error::Code::CorruptData); CHECK(result.error().to_string().find("reclassified ResourceExhausted -> CorruptData") != std::string::npos); } @@ -474,28 +474,28 @@ TEST_CASE("Envelope rejects incompatible metadata") { auto envelope = decode_envelope(*serialized); ++envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("zero uncompressed size does not mean unspecified") { auto envelope = decode_envelope(*serialized); envelope.uncompressed_size = 0; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("uncompressed size exceeds the hard limit") { auto envelope = decode_envelope(*serialized); envelope.uncompressed_size = io::envelope::default_max_decompressed_size + 1; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::ResourceExhausted); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::ResourceExhausted); } SECTION("uncompressed size exceeds a caller limit") { auto envelope = decode_envelope(*serialized); check_error(io::envelope::deserialize(encode_envelope(envelope), static_cast(envelope.uncompressed_size - 1)), - ::Error::Code::ResourceExhausted); + Error::Code::ResourceExhausted); } } @@ -509,14 +509,14 @@ TEST_CASE("Envelope validates the size of uncompressed data") { auto envelope = decode_envelope(*serialized); --envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("declared size is larger") { auto envelope = decode_envelope(*serialized); ++envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } } @@ -525,7 +525,7 @@ TEST_CASE("Envelope reports malformed serialized data") SECTION("envelope") { const io::envelope::Bytes malformed { std::byte { 0x01 }, std::byte { 0x02 } }; - check_error(io::envelope::deserialize(malformed), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(malformed), Error::Code::CorruptData); } SECTION("payload") @@ -540,6 +540,6 @@ TEST_CASE("Envelope reports malformed serialized data") .uncompressed_size = 1, .compressed_data = { std::byte { 0x01 } }, }; - check_error(io::envelope::deserialize(encode_envelope(envelope)), ::Error::Code::CorruptData); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } } diff --git a/unittests/terrainlib/error.cpp b/unittests/terrainlib/error.cpp new file mode 100644 index 00000000..44756de3 --- /dev/null +++ b/unittests/terrainlib/error.cpp @@ -0,0 +1,40 @@ +#include +#include + +#include + +#include "Error.h" + +TEST_CASE("Error::fail creates an expected failure", "[error]") +{ + const Expected result = Error::fail(Error::Code::Io, "open input"); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Io); + CHECK(result.error().to_string().find("open input") != std::string::npos); +} + +TEST_CASE("Error::propagate adds a call-site frame", "[error]") +{ + Expected source = Error::fail(Error::Code::Io, "open input"); + const Expected result = Error::propagate(std::move(source)); + + REQUIRE_FALSE(result.has_value()); + const std::string description = result.error().to_string(); + CHECK(description.find("propagated") != std::string::npos); + CHECK(description.find("unittests/terrainlib/error.cpp") != std::string::npos); + CHECK(description.find("open input") != std::string::npos); +} + +TEST_CASE("Error::propagate uses a supplied context as its call-site frame", "[error]") +{ + Expected source = Error::fail(Error::Code::ResourceExhausted, "decode payload"); + const Expected result = Error::propagate(std::move(source), Error::Code::CorruptData, "read envelope"); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::CorruptData); + const std::string description = result.error().to_string(); + CHECK(description.find("read envelope (reclassified ResourceExhausted -> CorruptData)") != std::string::npos); + CHECK(description.find("unittests/terrainlib/error.cpp") != std::string::npos); + CHECK(description.find("decode payload") != std::string::npos); +} diff --git a/unittests/terrainlib/io_conversion.cpp b/unittests/terrainlib/io_conversion.cpp index 479acfcb..e93f6ddc 100644 --- a/unittests/terrainlib/io_conversion.cpp +++ b/unittests/terrainlib/io_conversion.cpp @@ -78,7 +78,7 @@ TEST_CASE("io::conversion reports invalid inputs") { const auto result = io::conversion::to_raster(cv::Mat {}); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::InvalidInput); + CHECK(result.error().code() == Error::Code::InvalidInput); } SECTION("multidimensional matrix") @@ -87,7 +87,7 @@ TEST_CASE("io::conversion reports invalid inputs") const cv::Mat source(static_cast(sizes.size()), sizes.data(), CV_8UC1); const auto result = io::conversion::to_raster(source); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::Unsupported); + CHECK(result.error().code() == Error::Code::Unsupported); } SECTION("pixel type mismatch") @@ -95,7 +95,7 @@ TEST_CASE("io::conversion reports invalid inputs") const cv::Mat source(2, 2, CV_8UC3); const auto result = io::conversion::to_raster(source); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::InvalidInput); + CHECK(result.error().code() == Error::Code::InvalidInput); } SECTION("unsupported pixel") @@ -106,13 +106,13 @@ TEST_CASE("io::conversion reports invalid inputs") const cv::Mat source(2, 2, CV_8UC1); const auto result = io::conversion::to_raster(source); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::Unsupported); + CHECK(result.error().code() == Error::Code::Unsupported); } SECTION("empty raster") { const auto result = io::conversion::to_mat(radix::Raster {}); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::InvalidInput); + CHECK(result.error().code() == Error::Code::InvalidInput); } } diff --git a/unittests/terrainlib/sf_validate_index.cpp b/unittests/terrainlib/sf_validate_index.cpp index 10f18d55..9b6de302 100644 --- a/unittests/terrainlib/sf_validate_index.cpp +++ b/unittests/terrainlib/sf_validate_index.cpp @@ -22,6 +22,6 @@ TEST_CASE("Structura Fundamentalis validation reports the first Inner node", "[s const auto result = sf::validate_index(index); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::CorruptData); + CHECK(result.error().code() == Error::Code::CorruptData); CHECK(result.error().to_string().contains(root.to_string())); } diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp index 2c26d8c4..06263375 100644 --- a/unittests/terrainlib/store_codec.cpp +++ b/unittests/terrainlib/store_codec.cpp @@ -30,7 +30,7 @@ class WriteOnlyCodec final : public store::Codec { { return { add_extension(node_path, ".out") }; } - std::expected write(const std::filesystem::path&, const int&) const override + Expected write(const std::filesystem::path&, const int&) const override { ++writes; return {}; @@ -82,11 +82,11 @@ TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec const auto read = codec.read(path); REQUIRE_FALSE(read.has_value()); - CHECK(read.error().code() == ::Error::Code::Unsupported); + CHECK(read.error().code() == Error::Code::Unsupported); const auto write = codec.write(path, 42); REQUIRE_FALSE(write.has_value()); - CHECK(write.error().code() == ::Error::Code::Unsupported); + CHECK(write.error().code() == Error::Code::Unsupported); } TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") @@ -96,7 +96,7 @@ TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][cod CHECK(codec.writes == 1); const auto read = codec.read("node"); REQUIRE_FALSE(read.has_value()); - CHECK(read.error().code() == ::Error::Code::Unsupported); + CHECK(read.error().code() == Error::Code::Unsupported); } TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") @@ -117,5 +117,5 @@ TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") const auto result = codec.write(node_path, triangle_mesh(0.0)); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::Internal); + CHECK(result.error().code() == Error::Code::Internal); } diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 5dd94943..f8915de8 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -182,7 +182,7 @@ TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][op const auto unknown = mesh::codec::from_extension(".unknown"); REQUIRE_FALSE(unknown.has_value()); - CHECK(unknown.error().code() == ::Error::Code::Unsupported); + CHECK(unknown.error().code() == Error::Code::Unsupported); } TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store][open]") @@ -199,7 +199,7 @@ TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store] const auto result = mesh::storage::open_index(index_path); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::Unsupported); + CHECK(result.error().code() == Error::Code::Unsupported); } TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") @@ -217,7 +217,7 @@ TEST_CASE("octree opening retains index failures without directory fallback", "[ const auto unknown_layout = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_layout.has_value()); - CHECK(unknown_layout.error().code() == ::Error::Code::Unsupported); + CHECK(unknown_layout.error().code() == Error::Code::Unsupported); CHECK(unknown_layout.error().to_string().contains("unknown-layout")); index_file.layout_id = "flat"; @@ -225,13 +225,13 @@ TEST_CASE("octree opening retains index failures without directory fallback", "[ REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); const auto unknown_codec = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(unknown_codec.has_value()); - CHECK(unknown_codec.error().code() == ::Error::Code::Unsupported); + CHECK(unknown_codec.error().code() == Error::Code::Unsupported); const std::array malformed_bytes { 0xff, 0x00, 0x01 }; REQUIRE(io::write_bytes_to_path(malformed_bytes, index_path).has_value()); const auto malformed = mesh::storage::open_folder_indexed(directory.path()); REQUIRE_FALSE(malformed.has_value()); - CHECK(malformed.error().code() == ::Error::Code::CorruptData); + CHECK(malformed.error().code() == Error::Code::CorruptData); const auto independent_metadata = octree::storage::read_store_metadata(directory.path()); REQUIRE(independent_metadata.has_value()); diff --git a/unittests/terrainlib/store_index.cpp b/unittests/terrainlib/store_index.cpp index 58540e8e..e3a20065 100644 --- a/unittests/terrainlib/store_index.cpp +++ b/unittests/terrainlib/store_index.cpp @@ -71,10 +71,10 @@ TEST_CASE("shared index rejects invalid keys", "[store][index][raster]") store::Index index; const Key invalid { 4, { 16, 0 } }; - CHECK(index.get(invalid).error().code() == ::Error::Code::InvalidInput); - CHECK(index.add(invalid).error().code() == ::Error::Code::InvalidInput); - CHECK(index.remove(invalid).error().code() == ::Error::Code::InvalidInput); - CHECK(index.is_present(invalid).error().code() == ::Error::Code::InvalidInput); - CHECK(index.is_absent(invalid).error().code() == ::Error::Code::InvalidInput); - CHECK(index.is(store::NodeStatus::Leaf, invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(index.get(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.add(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.remove(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.is_present(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.is_absent(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.is(store::NodeStatus::Leaf, invalid).error().code() == Error::Code::InvalidInput); } diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index b5e860ff..dcfd1f14 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -58,26 +58,26 @@ store::path_layout::Mapping test_mapping read_int(const std::filesystem::path& path) +Expected read_int(const std::filesystem::path& path) { auto bytes = io::read_bytes_from_path(path); if (!bytes) { - return std::unexpected(std::move(bytes).error().with_context("read test integer")); + return Error::propagate(std::move(bytes), "read test integer"); } if (bytes->size() != sizeof(int)) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "test integer has the wrong byte count")); + return Error::fail(Error::Code::CorruptData, "test integer has the wrong byte count"); } int value; std::memcpy(&value, bytes->data(), sizeof(value)); return value; } -std::expected write_int(const int value, const std::filesystem::path& path) +Expected write_int(const int value, const std::filesystem::path& path) { const auto bytes = std::span(reinterpret_cast(&value), sizeof(value)); auto written = io::write_bytes_to_path(bytes, path); if (!written) { - return std::unexpected(std::move(written).error().with_context("write test integer")); + return Error::propagate(std::move(written), "write test integer"); } return {}; } @@ -89,9 +89,9 @@ class IntCodec final : public store::Codec { return { add_extension(node_path, ".data") }; } - std::expected read(const std::filesystem::path& node_path) const override { return read_int(paths(node_path).front()); } + Expected read(const std::filesystem::path& node_path) const override { return read_int(paths(node_path).front()); } - std::expected write(const std::filesystem::path& node_path, const int& value) const override + Expected write(const std::filesystem::path& node_path, const int& value) const override { return write_int(value, paths(node_path).front()); } @@ -111,20 +111,20 @@ class MultiFileCodec final : public store::Codec { return { add_extension(node_path, ".data"), add_extension(node_path, ".metadata") }; } - std::expected read(const std::filesystem::path& node_path) const override + Expected read(const std::filesystem::path& node_path) const override { auto value = read_int(paths(node_path).front()); if (!value) { - return std::unexpected(std::move(value).error().with_context("read multi-file test value")); + return Error::propagate(std::move(value), "read multi-file test value"); } return value.value(); } - std::expected write(const std::filesystem::path& node_path, const int& value) const override + Expected write(const std::filesystem::path& node_path, const int& value) const override { const auto node_paths = paths(node_path); if (!write_int(value, node_paths[0]) || !write_int(value + 1, node_paths[1])) { - return std::unexpected(::Error::make(::Error::Code::Io, "multi-file test write failed")); + return Error::fail(Error::Code::Io, "multi-file test write failed"); } return {}; } @@ -142,25 +142,25 @@ class ConfigurableCodec final : public store::Codec { return { add_extension(node_path, extension) }; } - std::expected read(const std::filesystem::path& node_path) const override + Expected read(const std::filesystem::path& node_path) const override { if (fail_read) { - return std::unexpected(::Error::make(::Error::Code::CorruptData, "injected decode failure")); + return Error::fail(Error::Code::CorruptData, "injected decode failure"); } auto value = read_int(paths(node_path).front()); if (!value) { - return std::unexpected(std::move(value).error().with_context("read configurable test value")); + return Error::propagate(std::move(value), "read configurable test value"); } return value.value(); } - std::expected write(const std::filesystem::path& node_path, const int& value) const override + Expected write(const std::filesystem::path& node_path, const int& value) const override { if (fail_write) { - return std::unexpected(::Error::make(::Error::Code::InvalidInput, "injected encode failure")); + return Error::fail(Error::Code::InvalidInput, "injected encode failure"); } if (!write_int(value, paths(node_path).front())) { - return std::unexpected(::Error::make(::Error::Code::Io, "configurable test write failed")); + return Error::fail(Error::Code::Io, "configurable test write failed"); } return {}; } @@ -172,12 +172,12 @@ class ConfigurableCodec final : public store::Codec { std::atomic_uint32_t index_writes = 0; -std::expected, ::Error> unused_index_read(const std::filesystem::path& path) +Expected> unused_index_read(const std::filesystem::path& path) { - return std::unexpected(::Error::make(::Error::Code::NotFound, "read unused test index from", path)); + return Error::fail(Error::Code::NotFound, "read unused test index from", path); } -std::expected count_index_write(const std::filesystem::path&, const store::IndexMetadata&) +Expected count_index_write(const std::filesystem::path&, const store::IndexMetadata&) { ++index_writes; return {}; @@ -227,7 +227,7 @@ TEST_CASE("byte writes report directory creation errors", "[io][bytes]") const auto result = io::write_bytes_to_path(std::span {}, blocker / "payload"); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::Io); + CHECK(result.error().code() == Error::Code::Io); } TEMPLATE_TEST_CASE("shared raw storage has no hidden octree dependency", "[store][storage]", octree::StoreTraits, raster_store::StoreTraits) @@ -252,10 +252,10 @@ TEST_CASE("shared storage rejects invalid 2D keys", "[store][storage]") auto storage = make_storage(directory.path()); const radix::tile::Id invalid { 3, { 8, 0 } }; - CHECK(storage.load(invalid).error().code() == ::Error::Code::InvalidInput); - CHECK(storage.save(invalid, 1).error().code() == ::Error::Code::InvalidInput); - CHECK(storage.has(invalid).error().code() == ::Error::Code::InvalidInput); - CHECK(storage.remove(invalid).error().code() == ::Error::Code::InvalidInput); + CHECK(storage.load(invalid).error().code() == Error::Code::InvalidInput); + CHECK(storage.save(invalid, 1).error().code() == Error::Code::InvalidInput); + CHECK(storage.has(invalid).error().code() == Error::Code::InvalidInput); + CHECK(storage.remove(invalid).error().code() == Error::Code::InvalidInput); } TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") @@ -267,7 +267,7 @@ TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") REQUIRE(storage.save(root, 1).has_value()); const auto rejected = storage.save(root, 2); REQUIRE_FALSE(rejected.has_value()); - CHECK(rejected.error().code() == ::Error::Code::AlreadyExists); + CHECK(rejected.error().code() == Error::Code::AlreadyExists); CHECK(storage.load(root).value() == 1); storage.settings().allow_overwrite = true; @@ -396,7 +396,7 @@ TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") const auto rejected = target.copy_from(root, source); REQUIRE_FALSE(rejected.has_value()); - CHECK(rejected.error().code() == ::Error::Code::AlreadyExists); + CHECK(rejected.error().code() == Error::Code::AlreadyExists); CHECK(target.load(root).value() == 1); target.settings().allow_overwrite = true; @@ -423,7 +423,7 @@ TEST_CASE("failed multi-file overwrite leaves the target node unindexed", "[stor const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - CHECK(copied.error().code() == ::Error::Code::Io); + CHECK(copied.error().code() == Error::Code::Io); CHECK_FALSE(target.has(root).value()); CHECK_FALSE(target.index().get(root).value().has_value()); CHECK(std::filesystem::equivalent(source.paths(root)->front(), target_paths.front())); @@ -445,7 +445,7 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - CHECK(copied.error().code() == ::Error::Code::NotFound); + CHECK(copied.error().code() == Error::Code::NotFound); CHECK(target.has(root).value()); CHECK(target.load(root).value() == 7); } @@ -465,7 +465,7 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - CHECK(copied.error().code() == ::Error::Code::CorruptData); + CHECK(copied.error().code() == Error::Code::CorruptData); CHECK(target.has(root).value()); CHECK(target.load(root).value() == 7); } @@ -485,7 +485,7 @@ TEST_CASE("copy_from propagates source and codec failures", "[store][storage][co const auto copied = target.copy_from(root, source); REQUIRE_FALSE(copied.has_value()); - CHECK(copied.error().code() == ::Error::Code::InvalidInput); + CHECK(copied.error().code() == Error::Code::InvalidInput); CHECK_FALSE(target.has(root).value()); } } diff --git a/unittests/terrainlib/store_traverse.cpp b/unittests/terrainlib/store_traverse.cpp index ec70e8fa..465867d6 100644 --- a/unittests/terrainlib/store_traverse.cpp +++ b/unittests/terrainlib/store_traverse.cpp @@ -51,5 +51,5 @@ TEST_CASE("shared traversal rejects an invalid explicit root", "[store][traverse const auto result = store::traverse(index, [](const Key&, const store::NodeStatus) { }, store::AlwaysRefine {}, invalid); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == ::Error::Code::InvalidInput); + CHECK(result.error().code() == Error::Code::InvalidInput); } From de98c33b880e2d3c219a0587401ce2369c69610a Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:03:02 +0200 Subject: [PATCH 33/43] Compile libigl wrappers outside unity builds --- src/terrainlib/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index 882fb477..dc77c8fa 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -196,7 +196,11 @@ add_library(terrainlib uv/fit.h uv/unwrap.h uv/unwrap.cpp ) -set_source_files_properties(mesh/io/gltf.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) +set_source_files_properties( + mesh/igl/igl.cpp + mesh/io/gltf.cpp + PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON +) target_compile_features(terrainlib PUBLIC cxx_std_23) target_compile_definitions(terrainlib PUBLIC SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE) From 204b8e2de089886bf31a7e3df2a38845e631e051 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:52:17 +0200 Subject: [PATCH 34/43] Improve Expected error propagation --- src/dag_builder/build.cpp | 16 +++--- src/dag_builder/codec/Dag.h | 10 ++-- src/dag_builder/main.cpp | 8 +-- src/dag_convert_debug/main.cpp | 14 +++--- src/mesh_convert/main.cpp | 4 +- src/sf_builder/main.cpp | 4 +- src/sf_builder/terrainbuilder.cpp | 17 ++++--- src/sf_index_browser/main.cpp | 4 +- src/sf_merger/NodeLoader.h | 4 +- src/sf_merger/NodeWriter.h | 18 ++++--- src/sf_merger/cut.h | 27 ++++++---- src/sf_merger/main.cpp | 14 +++--- src/sf_merger/mask.h | 2 +- src/sf_merger/merge.h | 35 +++++++------ src/terrainlib/io/compression.cpp | 4 +- src/terrainlib/mesh/codec/Gltf.h | 4 +- src/terrainlib/octree/storage/IndexFile.h | 6 +-- src/terrainlib/octree/storage/open_runtime.h | 5 +- src/terrainlib/sf/finalize_storage.h | 2 +- src/terrainlib/store/Index.h | 10 ++-- src/terrainlib/store/RawStorage.h | 14 +++--- src/terrainlib/store/Storage.h | 52 +++++++++++--------- src/terrainlib/store/open.h | 8 +-- src/terrainlib/store/traverse.h | 17 ++++--- 24 files changed, 167 insertions(+), 132 deletions(-) diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index e1eca1f7..e398e51f 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -468,8 +468,8 @@ Expected>> gather_relevant_input_leaves( return radix::geometry::intersect(root_bounds, space.get_node_bounds(id)); }, start); - if (!traversal.has_value()) { - return Error::propagate(std::move(traversal)); + if (!traversal) { + return Error::propagate(std::move(traversal), "gather relevant input leaves"); } return result; } @@ -523,7 +523,7 @@ std::unordered_set build_level( auto debug_result = mesh::storage::open_folder( ctx.output_storage.base_path().string() + "-debug", std::move(options)); - if (!debug_result.has_value()) { + if (!debug_result) { LOG_ERROR_AND_EXIT( "Failed to open debug mesh storage: {}", debug_result.error().to_string()); @@ -552,7 +552,7 @@ std::unordered_set build_level( if (debug_storage) { const auto debug_mesh = clustering_to_mesh(result->clustering); const auto debug_save_result = debug_storage->save(target, debug_mesh); - if (!debug_save_result.has_value()) { + if (!debug_save_result) { LOG_ERROR_AND_EXIT( "Failed to save debug mesh for node {}: {}", target, @@ -589,7 +589,7 @@ Expected build_levels( const BuildOptions &options, const AnyRange &level_range) { auto validation = sf::validate_index(input_storage.index()); - if (!validation.has_value()) { + if (!validation) { return validation; } const octree::OddLevelShifted shifted_space = octree::OddLevelShifted::earth(); @@ -597,8 +597,8 @@ Expected build_levels( const octree::Id root_node = options.root_node; const auto root_bounds = shifted_space.get_node_bounds_with_children(root_node); auto input_by_level_result = gather_relevant_input_leaves(input_storage.index(), space, root_bounds); - if (!input_by_level_result.has_value()) { - return Error::propagate(std::move(input_by_level_result)); + if (!input_by_level_result) { + return Error::propagate(std::move(input_by_level_result), "prepare input leaves for DAG build"); } auto input_by_level = std::move(input_by_level_result.value()); @@ -636,7 +636,7 @@ Expected build_levels( ctx); // Persist per level so a finished level can be read back before the whole run completes - if (const auto result = ctx.output_storage.save_or_create_index(); !result.has_value()) { + if (const auto result = ctx.output_storage.save_or_create_index(); !result) { LOG_WARN( "Could not save index after level {}: {}", level, diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index e67f5391..32e4485f 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -34,11 +34,11 @@ class Dag final : public store::Codec { const auto node_paths = paths(node_path); auto clustering_payload = io::envelope::read_from_path(node_paths[0]); if (!clustering_payload) { - return Error::propagate(std::move(clustering_payload)); + return Error::propagate(std::move(clustering_payload), "read DAG clustering"); } auto metadata_payload = io::envelope::read_from_path(node_paths[1]); if (!metadata_payload) { - return Error::propagate(std::move(metadata_payload)); + return Error::propagate(std::move(metadata_payload), "read DAG metadata"); } auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); if (!clustering) { @@ -84,11 +84,11 @@ class Dag final : public store::Codec { } auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); if (!data_written) { - return data_written; + return Error::propagate(std::move(data_written), "write DAG clustering"); } auto metadata_written = io::envelope::write_to_path(metadata, node_paths[1]); if (!metadata_written) { - return metadata_written; + return Error::propagate(std::move(metadata_written), "write DAG metadata"); } return {}; } @@ -106,7 +106,7 @@ class MetadataView final : public store::Codec { const std::filesystem::path path = paths(node_path).front(); auto payload = io::envelope::read_from_path(path); if (!payload) { - return Error::propagate(std::move(payload)); + return Error::propagate(std::move(payload), "read DAG metadata"); } auto metadata = dag::format::decode_metadata(std::move(*payload)); if (!metadata) { diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index 1c892b6b..177924b7 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -16,7 +16,7 @@ int main(int argc, char **argv) { try { auto input_result = mesh::storage::open_folder_indexed(args.input_path); - if (!input_result.has_value()) { + if (!input_result) { LOG_ERROR( "Failed to open input dataset {}: {}", args.input_path, @@ -24,7 +24,7 @@ int main(int argc, char **argv) { return EXIT_FAILURE; } auto output_result = dag::storage::open_folder_indexed(args.output_path); - if (!output_result.has_value()) { + if (!output_result) { LOG_ERROR( "Failed to open output dataset {}: {}", args.output_path, @@ -62,12 +62,12 @@ int main(int argc, char **argv) { output_storage, options, args.level_range); - if (!build_result.has_value()) { + if (!build_result) { LOG_ERROR("Invalid Structura Fundamentalis input: {}", build_result.error().to_string()); return EXIT_FAILURE; } const auto index_result = output_storage.save_index(); - if (!index_result.has_value()) { + if (!index_result) { LOG_ERROR( "Failed to save output index in {}: {}", args.output_path, diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index d6f1dad4..005c054d 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -23,7 +23,7 @@ void export_node(const cli::Args &args) { node_path.replace_extension(); const dag::codec::Dag codec; const auto load_result = codec.read(node_path); - if (!load_result.has_value()) { + if (!load_result) { LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error().to_string()); return; } @@ -31,14 +31,14 @@ void export_node(const cli::Args &args) { const mesh::Simple mesh = clustering_to_mesh(load_result.value().clustering); const auto save_result = mesh::io::save_to_path(mesh, args.output_path); - if (!save_result.has_value()) { + if (!save_result) { LOG_ERROR("Failed to save mesh to {}: {}", args.output_path, save_result.error().description()); } } void export_storage(const cli::Args &args) { auto input_result = dag::storage::open_folder_indexed(args.input_path); - if (!input_result.has_value()) { + if (!input_result) { LOG_ERROR( "Failed to open input storage {}: {}", args.input_path, @@ -52,7 +52,7 @@ void export_storage(const cli::Args &args) { auto output_result = mesh::storage::open_folder( args.output_path, std::move(options)); - if (!output_result.has_value()) { + if (!output_result) { LOG_ERROR( "Failed to open output storage {}: {}", args.output_path, @@ -74,7 +74,7 @@ void export_storage(const cli::Args &args) { } const auto load_result = input_storage.load(id); - if (!load_result.has_value()) { + if (!load_result) { LOG_ERROR( "Failed to load node {}: {}", id, @@ -86,7 +86,7 @@ void export_storage(const cli::Args &args) { const mesh::Simple mesh = clustering_to_mesh(load_result.value().clustering); const auto save_result = output_storage.save(id, mesh); - if (!save_result.has_value()) { + if (!save_result) { LOG_ERROR( "Failed to save mesh for node {}: {}", id, @@ -102,7 +102,7 @@ void export_storage(const cli::Args &args) { progress_thread.join(); const auto index_result = output_storage.save_or_create_index(); - if (!index_result.has_value()) { + if (!index_result) { LOG_ERROR( "Failed to save index for {}: {}", args.output_path, diff --git a/src/mesh_convert/main.cpp b/src/mesh_convert/main.cpp index a396ce50..2d812705 100644 --- a/src/mesh_convert/main.cpp +++ b/src/mesh_convert/main.cpp @@ -12,7 +12,7 @@ void run(const cli::Args& args) { LOG_INFO("Loading input mesh..."); const std::expected load_result = mesh::io::load_from_path(args.input_path); - if (!load_result.has_value()) { + if (!load_result) { LOG_ERROR("Failed to load mesh: {}", load_result.error().description()); return; } @@ -31,7 +31,7 @@ void run(const cli::Args& args) { LOG_INFO("Writing output mesh..."); const std::expected save_result = mesh::io::save_to_path(mesh, args.output_path); - if (!save_result.has_value()) { + if (!save_result) { LOG_ERROR("Failed to save mesh: {}", save_result.error().description()); return; } diff --git a/src/sf_builder/main.cpp b/src/sf_builder/main.cpp index c592fdf1..888c2af4 100644 --- a/src/sf_builder/main.cpp +++ b/src/sf_builder/main.cpp @@ -25,7 +25,7 @@ OGRSpatialReference parse_srs(const std::string &user_input) { const auto result = srs::from_user_input(user_input); - if (!result.has_value()) { + if (!result) { LOG_ERROR(result.error()); exit(1); } @@ -326,7 +326,7 @@ int run(std::span args) { output_base_path, output_format, overwrite_existing); - if (!build_result.has_value()) { + if (!build_result) { LOG_ERROR( "Failed to finalize Structura Fundamentalis output: {}", build_result.error().to_string()); diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 3585687d..9c33f956 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -75,7 +75,7 @@ std::optional build_patch( mesh_srs, target_bounds_srs, target_bounds, texture_srs, texture_bounds); - if (!mesh_result.has_value()) { + if (!mesh_result) { const BuildMeshError error = mesh_result.error(); if (error == BuildMeshError::OutOfBounds) { const radix::tile::SrsBounds dataset_bounds = dataset.bounds(); @@ -154,7 +154,7 @@ void build_and_save_patch( // metadata["texture_bounds"] = fmt::format( // "{{ \"min\": {{ \"x\": {}, \"y\": {} }}, \"max\": {{ \"x\": {}, \"y\": {} }} }}", // texture_bounds.min.x, texture_bounds.min.y, texture_bounds.max.x, texture_bounds.max.y); - if (!mesh::io::save_to_path(mesh, output_path, mesh::io::SaveOptions{.metadata = metadata}).has_value()) { + if (!mesh::io::save_to_path(mesh, output_path, mesh::io::SaveOptions{.metadata = metadata})) { LOG_ERROR("Failed to save mesh to file {}", output_path); std::exit(2); } @@ -194,7 +194,7 @@ Expected build_all_patches( auto storage_result = mesh::storage::open_folder( output_base_path, std::move(open_options)); - if (!storage_result.has_value()) { + if (!storage_result) { LOG_ERROR_AND_EXIT( "Failed to open output dataset {}: {}", output_base_path, @@ -288,7 +288,7 @@ Expected build_all_patches( tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; const auto already_exists = storage.has(node); - if (!already_exists.has_value()) { + if (!already_exists) { LOG_ERROR( "Failed to inspect node {}: {}", node, @@ -320,7 +320,7 @@ Expected build_all_patches( const auto mesh = std::move(mesh_result.value()); mesh::validate(mesh); const auto save_result = storage.save(node, mesh); - if (!save_result.has_value()) { + if (!save_result) { LOG_ERROR( "Failed to save mesh for node {}: {}", node, @@ -347,6 +347,11 @@ Expected build_all_patches( } auto finalized_storage = std::move(storage).release(); - return sf::finalize_storage(finalized_storage); + auto finalized = sf::finalize_storage(finalized_storage); + if (!finalized) { + return Error::propagate( + std::move(finalized), "finalize generated terrain storage \"" + output_base_path.string() + "\""); + } + return {}; } } diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index ed6fd9a5..11f7db1a 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -82,7 +82,7 @@ class TreeView { if (parent.status == store::NodeStatusOrMissing::Leaf) { auto result = this->_storage.load(parent.id); DisplayEntry entry; - if (result.has_value()) { + if (result) { const auto &mesh = result.value(); entry = MeshNode{parent.id, mesh.vertex_count(), mesh.face_count()}; } else { @@ -192,7 +192,7 @@ mesh::storage::IndexedStorage open_path_indexed(const std::filesystem::path& pat auto result = std::filesystem::is_directory(path) ? mesh::storage::open_folder_indexed(path) : mesh::storage::open_index(path); - if (!result.has_value()) { + if (!result) { LOG_ERROR_AND_EXIT( "Failed to open dataset {}: {}", path, diff --git a/src/sf_merger/NodeLoader.h b/src/sf_merger/NodeLoader.h index 269db683..55e5a76a 100644 --- a/src/sf_merger/NodeLoader.h +++ b/src/sf_merger/NodeLoader.h @@ -37,7 +37,7 @@ class NodeLoader { // Try storage auto mesh_opt = this->_storage.load(id); - if (mesh_opt.has_value()) { + if (mesh_opt) { auto mesh = mesh_opt.value(); this->_cache.put(id, mesh); return mesh; @@ -54,7 +54,7 @@ class NodeLoader { // Try storage auto parent_mesh_opt = this->_storage.load(parent_id); - if (parent_mesh_opt.has_value()) { + if (parent_mesh_opt) { auto parent_mesh = parent_mesh_opt.value(); const auto bounds = this->_space.get_node_bounds(id); SimpleMesh clipped = mesh::clip_on_bounds(parent_mesh, bounds); diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 88c10e60..0b846cbe 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -26,12 +26,13 @@ class NodeWriter { const SimpleMesh &mesh) { mesh::validate(mesh); auto save_result = this->_storage.save(id, mesh); - if (!save_result.has_value()) { + if (!save_result) { return save_result; } auto path_result = this->_storage.path_for(id); - if (!path_result.has_value()) { - return Error::propagate(std::move(path_result)); + if (!path_result) { + return Error::propagate( + std::move(path_result), "resolve texture output path for node " + id.to_string()); } auto p = path_result.value(); // change extension to .png @@ -58,17 +59,20 @@ class NodeWriter { DEBUG_ASSERT(status == store::NodeStatus::Leaf); auto result = this->_storage.copy_from(child_id, loader.storage()); - if (!result.has_value()) { + if (!result) { error = std::move(result).error(); } }, [&](const octree::Id &) { return !error.has_value(); }, id); - if (!traversal.has_value()) { - return traversal; + if (!traversal) { + return Error::propagate( + std::move(traversal), "traverse source subtree rooted at " + id.to_string() + " while copying"); } if (error.has_value()) { - return Error::propagate(std::move(error.value())); + return std::unexpected { + std::move(error.value()).with_context("copy subtree rooted at " + id.to_string() + " to output"), + }; } return {}; } diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 919fa97b..4589012d 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -43,7 +43,7 @@ inline Expected cut_leaf_node( if (clipped.is_ref()) { LOG_TRACE("Mesh was fully inside the mask"); auto copy_result = ctx.output.copy_from(id, ctx.input); - if (!copy_result.has_value()) { + if (!copy_result) { return copy_result; } } else { @@ -52,7 +52,7 @@ inline Expected cut_leaf_node( LOG_TRACE("Mesh was clipped from {} vertices and {} triangles to {} vertices and {} triangles", mesh.vertex_count(), mesh.face_count(), clipped_mesh.vertex_count(), clipped_mesh.face_count()); auto save_result = ctx.output.save(id, clipped_mesh); - if (!save_result.has_value()) { + if (!save_result) { return save_result; } } else { @@ -78,7 +78,7 @@ inline Expected cut_virtual_node( LOG_TRACE("Clipping mask for {}", child_id); const MeshMask child_mask = clip_mask_on_bounds(mask, child_bounds); auto child_result = cut_node(ctx, child_id, child_mask); - if (!child_result.has_value()) { + if (!child_result) { return child_result; } } @@ -128,15 +128,21 @@ inline Expected cut_dataset( mesh::storage::Storage &output, const bool keep_inside) { auto validation = sf::validate_index(input.index()); - if (!validation.has_value()) { - return validation; + if (!validation) { + return Error::propagate( + std::move(validation), "validate cut input dataset \"" + input.base_path().string() + "\""); } Context ctx(input, output, octree::Space::earth(), keep_inside); auto cut_result = cut_node(ctx, octree::Id::root(), mask); - if (!cut_result.has_value()) { - return cut_result; + if (!cut_result) { + return Error::propagate(std::move(cut_result), "cut dataset \"" + input.base_path().string() + "\""); } - return sf::finalize_storage(output); + auto finalized = sf::finalize_storage(output); + if (!finalized) { + return Error::propagate( + std::move(finalized), "finalize cut output dataset \"" + output.base_path().string() + "\""); + } + return {}; } inline Expected cut_dataset( @@ -151,8 +157,9 @@ inline Expected cut_dataset( options.default_mapping = octree::store_layout::level_and_coordinate_directories(); options.preferred_extension = ".glb"; auto output_result = mesh::storage::open_folder_indexed(output_path, std::move(options)); - if (!output_result.has_value()) { - return Error::propagate(std::move(output_result)); + if (!output_result) { + return Error::propagate( + std::move(output_result), "open cut output dataset \"" + output_path.string() + "\""); } mesh::storage::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 217d39dd..3cc7b7bc 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -17,7 +17,7 @@ std::optional load_mask_from_path(const std::filesystem::path& path) { LOG_INFO("Loading mask file from {}", path); const glm::dvec2 radius_range = mask::pad_radius_range(earth::radius_range(), 2); auto result = mask::load_from_path(path, radius_range); - if (result.has_value()) { + if (result) { const auto mask = result.value(); LOG_DEBUG("Loaded mask successfully ({} vertices, {} triangles)", mask.vertex_count(), mask.face_count()); @@ -35,7 +35,7 @@ std::optional load_mask_from_path(const std::filesystem::path& path) { void run(const cli::MergeArgs& args) { LOG_TRACE("Loading base dataset from {}", args.base_path); auto base_result = mesh::storage::open_folder_indexed(args.base_path); - if (!base_result.has_value()) { + if (!base_result) { LOG_ERROR_AND_EXIT( "Failed to open base dataset {}: {}", args.base_path, @@ -45,7 +45,7 @@ void run(const cli::MergeArgs& args) { LOG_TRACE("Loading new dataset from {}", args.new_path); auto new_result = mesh::storage::open_folder_indexed(args.new_path); - if (!new_result.has_value()) { + if (!new_result) { LOG_ERROR_AND_EXIT( "Failed to open new dataset {}: {}", args.new_path, @@ -59,7 +59,7 @@ void run(const cli::MergeArgs& args) { options.default_mapping = base_dataset.layout().mapping(); options.preferred_extension = std::string(base_dataset.codec_selector().value_or(".sfmesh")); auto output_result = mesh::storage::open_folder(args.output_path, std::move(options)); - if (!output_result.has_value()) { + if (!output_result) { LOG_ERROR_AND_EXIT( "Failed to open output dataset {}: {}", args.output_path, @@ -74,7 +74,7 @@ void run(const cli::MergeArgs& args) { new_dataset, output_dataset, mask); - if (!merge_result.has_value()) { + if (!merge_result) { LOG_ERROR_AND_EXIT("Failed to merge datasets: {}", merge_result.error().to_string()); } } @@ -82,7 +82,7 @@ void run(const cli::MergeArgs& args) { void run(const cli::CutArgs& args) { LOG_TRACE("Loading input dataset from {}", args.input_path); auto input_result = mesh::storage::open_folder_indexed(args.input_path); - if (!input_result.has_value()) { + if (!input_result) { LOG_ERROR_AND_EXIT( "Failed to open input dataset {}: {}", args.input_path, @@ -95,7 +95,7 @@ void run(const cli::CutArgs& args) { mask, args.output_path, args.keep_inside); - if (!cut_result.has_value()) { + if (!cut_result) { LOG_ERROR_AND_EXIT("Failed to cut dataset: {}", cut_result.error().to_string()); } } diff --git a/src/sf_merger/mask.h b/src/sf_merger/mask.h index fde93152..304cbce3 100644 --- a/src/sf_merger/mask.h +++ b/src/sf_merger/mask.h @@ -569,7 +569,7 @@ inline std::expected load_referenced_from_path inline std::expected load_from_path(const std::filesystem::path &path, const glm::dvec2& radius_range) { auto ref_mask_res = load_referenced_from_path(path); - if (!ref_mask_res.has_value()) { + if (!ref_mask_res) { return std::unexpected(ref_mask_res.error()); } ReferencedPolygonMask ref_polygon_mask = std::move(ref_mask_res.value()); diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index c064aa7f..8f902a02 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -65,8 +65,9 @@ class Merger { ) { LOG_DEBUG("[{}] Start merging (left = {}, right = {})", id, left_status, right_status); auto has_result = this->_output.has_node(id); - if (!has_result.has_value()) { - return Error::propagate(std::move(has_result)); + if (!has_result) { + return Error::propagate( + std::move(has_result), "check merged output for node " + id.to_string()); } if (has_result.value()) { LOG_DEBUG("[{}] Already merged, skipping...", id); @@ -82,7 +83,7 @@ class Merger { const auto children = id.children().value(); for (const auto &child_id : children) { auto child_result = this->merge_node(child_id, result.context); - if (!child_result.has_value()) { + if (!child_result) { return child_result; } } @@ -94,7 +95,7 @@ class Merger { auto mesh_opt = this->_left.load_node(id); if (mesh_opt.has_value()) { auto write_result = this->_output.write_node(id, *mesh_opt); - if (!write_result.has_value()) { + if (!write_result) { return write_result; } } @@ -102,7 +103,7 @@ class Merger { auto mesh_opt = this->_right.load_node(id); if (mesh_opt.has_value()) { auto write_result = this->_output.write_node(id, *mesh_opt); - if (!write_result.has_value()) { + if (!write_result) { return write_result; } } @@ -110,7 +111,7 @@ class Merger { auto copy_result = this->_output.copy_subtree_to_output( id, result.source == merge::Source::Left ? this->_left : this->_right); - if (!copy_result.has_value()) { + if (!copy_result) { return copy_result; } } @@ -118,7 +119,7 @@ class Merger { } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was merged", id); auto write_result = this->_output.write_node(id, result.mesh); - if (!write_result.has_value()) { + if (!write_result) { return write_result; } return {}; @@ -187,8 +188,9 @@ inline Expected merge_datasets( const std::optional mask = std::nullopt) { for (const mesh::storage::IndexedStorage *input : {&left_dataset, &right_dataset}) { auto validation = sf::validate_index(input->index()); - if (!validation.has_value()) { - return validation; + if (!validation) { + return Error::propagate( + std::move(validation), "validate merge input dataset \"" + get_dataset_name(*input) + "\""); } } @@ -207,17 +209,22 @@ inline Expected merge_datasets( merge::visitor::Masked visitor {mask.value(), space}; Merger merger(visitor, left, right, output); auto result = merger.merge_root(); - if (!result.has_value()) { - return result; + if (!result) { + return Error::propagate(std::move(result), "merge masked datasets"); } } else { merge::visitor::Simple visitor; Merger merger(visitor, left, right, output); auto result = merger.merge_root(); - if (!result.has_value()) { - return result; + if (!result) { + return Error::propagate(std::move(result), "merge datasets"); } } - return sf::finalize_storage(output_dataset); + auto finalized = sf::finalize_storage(output_dataset); + if (!finalized) { + return Error::propagate( + std::move(finalized), "finalize merged output dataset \"" + get_dataset_name(output_dataset) + "\""); + } + return {}; } diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index 3789ed68..3343497d 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -87,7 +87,7 @@ Expected compress_with_checksum( const Bytes& uncompressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) { if (auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { - return Error::propagate(std::move(validation)); + return Error::propagate(std::move(validation), "validate compression algorithms"); } if (uncompressed_data.size() > default_max_decompressed_size) { return Error::fail(Error::Code::ResourceExhausted, "uncompressed data exceeds the compression size limit"); @@ -130,7 +130,7 @@ Expected checked_decompress(const Bytes& compressed_data, const std::size_t max_decompressed_size) { if (auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { - return Error::propagate(std::move(validation)); + return Error::propagate(std::move(validation), "validate decompression algorithms"); } if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { return Error::fail(Error::Code::InvalidInput, "a checksum was supplied for an algorithm that does not accept one"); diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index e78bba02..5a14f8e7 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -33,7 +33,7 @@ class Gltf final : public store::Codec { { const std::filesystem::path path = paths(node_path).front(); const auto result = mesh::io::gltf::load_from_path(path); - if (!result.has_value()) { + if (!result) { switch (static_cast(result.error())) { case mesh::io::LoadMeshErrorKind::UnsupportedFormat: return Error::fail(Error::Code::Unsupported, "load unsupported glTF format from", path); @@ -54,7 +54,7 @@ class Gltf final : public store::Codec { const std::filesystem::path path = paths(node_path).front(); try { const auto result = mesh::io::gltf::save_to_path(mesh, path); - if (!result.has_value()) { + if (!result) { switch (static_cast(result.error())) { case mesh::io::SaveMeshErrorKind::UnsupportedFormat: return Error::fail(Error::Code::Unsupported, "write unsupported glTF format to", path); diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index a1c42f2e..3ed4ea94 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -128,7 +128,7 @@ inline Expected read_store_metadata(const std::filesystem::path& const std::filesystem::path path = base_path / metadata_file_name; auto metadata = io::envelope::read_from_path(path); if (!metadata) { - return Error::propagate(std::move(metadata)); + return metadata; } if (auto valid = validate(*metadata); !valid) { return Error::fail(Error::Code::CorruptData, "invalid store metadata in " + path.string() + ": " + valid.error()); @@ -140,12 +140,12 @@ inline Expected> read_index_file(const std::fi { auto metadata = read_store_metadata(path.parent_path()); if (!metadata) { - return Error::propagate(std::move(metadata)); + return Error::propagate(std::move(metadata), "read octree store metadata"); } auto encoded_index = io::envelope::read_from_path(path); if (!encoded_index) { - return Error::propagate(std::move(encoded_index)); + return Error::propagate(std::move(encoded_index), "read octree store index \"" + path.string() + "\""); } auto index = decode_index(*encoded_index); if (!index) { diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index 236948ab..968bd6a6 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -61,7 +61,8 @@ Expected> open_folder(const std::filesyste if (index_exists) { auto indexed = store::open_index(index_path, format, payload_class, resolve_codec); if (!indexed) { - return Error::propagate(std::move(indexed)); + return Error::propagate( + std::move(indexed), "open indexed octree storage at \"" + base_path.string() + "\""); } return store::Storage(std::move(*indexed)); } @@ -70,7 +71,7 @@ Expected> open_folder(const std::filesyste std::string codec_selector = options.preferred_extension.value_or(std::move(default_codec_selector)); auto codec = resolve_codec(codec_selector); if (!codec) { - return Error::propagate(std::move(codec)); + return Error::propagate(std::move(codec), "resolve octree storage codec \"" + codec_selector + "\""); } return store::make_storage(base_path, format, mapping, std::move(payload_class), std::move(codec_selector), std::move(*codec)); diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h index 1fe469cd..f6ff379b 100644 --- a/src/terrainlib/sf/finalize_storage.h +++ b/src/terrainlib/sf/finalize_storage.h @@ -12,7 +12,7 @@ namespace sf { inline Expected finalize_storage(mesh::storage::Storage& storage) { auto save_result = storage.save_or_create_index(); - if (!save_result.has_value()) { + if (!save_result) { return save_result; } diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h index 8174a3bf..d1e99098 100644 --- a/src/terrainlib/store/Index.h +++ b/src/terrainlib/store/Index.h @@ -68,8 +68,8 @@ class Index { NodeStatus* parent_status = get_raw_unchecked(parent.value()); if (parent_status == nullptr) { auto parent_result = add(parent.value()); - if (!parent_result.has_value()) { - return Error::propagate(std::move(parent_result)); + if (!parent_result) { + return parent_result; } set_raw_unchecked(parent.value(), NodeStatus::Virtual); set_raw_unchecked(key, NodeStatus::Leaf); @@ -122,15 +122,15 @@ class Index { Expected is_absent(const Key& key) const { auto present = is_present(key); - if (!present.has_value()) { - return Error::propagate(std::move(present)); + if (!present) { + return present; } return !present.value(); } Expected is(const NodeStatus status, const Key& key) const { auto result = get(key); - if (!result.has_value()) { + if (!result) { return Error::propagate(std::move(result)); } return result.value() == status; diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index 1d30361c..cedce8c1 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -59,7 +59,7 @@ class RawStorage { Expected has(const Key& key) const { auto node_paths = paths(key); - if (!node_paths.has_value()) { + if (!node_paths) { return Error::propagate(std::move(node_paths)); } if (node_paths->empty()) { @@ -81,7 +81,7 @@ class RawStorage { Expected remove(const Key& key) const { auto node_paths = paths(key); - if (!node_paths.has_value()) { + if (!node_paths) { return Error::propagate(std::move(node_paths)); } bool removed = false; @@ -107,7 +107,7 @@ class RawStorage { return store::invalid_key_error(key); } auto source_exists = source.has(key); - if (!source_exists.has_value()) { + if (!source_exists) { return Error::propagate(std::move(source_exists)); } if (!source_exists.value()) { @@ -117,15 +117,15 @@ class RawStorage { const std::filesystem::path probe("__codec_probe__/node"); if (m_codec->paths(probe) != source.m_codec->paths(probe)) { auto loaded = source.m_codec->read(source.m_layout.node_path(key)); - if (!loaded.has_value()) { + if (!loaded) { return Error::propagate(std::move(loaded), "read source node while copying"); } const auto prepared = before_modify(); - if (!prepared.has_value()) { + if (!prepared) { return prepared; } auto written = m_codec->write(m_layout.node_path(key), loaded.value()); - if (!written.has_value()) { + if (!written) { return Error::propagate(std::move(written), "write target node while copying"); } return {}; @@ -140,7 +140,7 @@ class RawStorage { return {}; } const auto prepared = before_modify(); - if (!prepared.has_value()) { + if (!prepared) { return prepared; } for (size_t index = 0; index < source_paths.size(); ++index) { diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index b77473af..c102a34e 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -85,8 +85,9 @@ class Storage { } if (m_index.has_value()) { auto status = m_index->get(key); - if (!status.has_value()) { - return Error::propagate(std::move(status)); + if (!status) { + return Error::propagate(std::move(status), + "look up node " + Traits::key_to_string(key) + " in storage index before loading"); } if (!status->has_value() || status->value() == NodeStatus::Virtual) { return Error::fail(Error::Code::NotFound, "node " + Traits::key_to_string(key) + " is not physically present in the index"); @@ -101,8 +102,9 @@ class Storage { return store::invalid_key_error(key); } auto exists = has(key); - if (!exists.has_value()) { - return Error::propagate(std::move(exists)); + if (!exists) { + return Error::propagate( + std::move(exists), "check whether node " + Traits::key_to_string(key) + " exists before saving"); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); @@ -112,13 +114,14 @@ class Storage { } auto result = m_raw.save(key, data); - if (!result.has_value()) { + if (!result) { return result; } if (m_index.has_value()) { auto added = m_index->add(key); - if (!added.has_value()) { - return Error::propagate(std::move(added)); + if (!added) { + return Error::propagate( + std::move(added), "add saved node " + Traits::key_to_string(key) + " to storage index"); } m_dirty = m_dirty || added.value(); } @@ -131,8 +134,9 @@ class Storage { return store::invalid_key_error(key); } auto exists = has(key); - if (!exists.has_value()) { - return Error::propagate(std::move(exists)); + if (!exists) { + return Error::propagate( + std::move(exists), "check whether target node " + Traits::key_to_string(key) + " exists before copying"); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); @@ -145,21 +149,23 @@ class Storage { return {}; } auto removed = m_index->remove(key); - if (!removed.has_value()) { - return Error::propagate(std::move(removed)); + if (!removed) { + return Error::propagate( + std::move(removed), "remove overwritten node " + Traits::key_to_string(key) + " from storage index"); } m_dirty = m_dirty || removed.value(); return {}; }; auto result = m_raw.copy_from(key, source.m_raw, prepare_target); - if (!result.has_value()) { + if (!result) { return result; } if (m_index.has_value()) { auto added = m_index->add(key); - if (!added.has_value()) { - return Error::propagate(std::move(added)); + if (!added) { + return Error::propagate( + std::move(added), "add copied node " + Traits::key_to_string(key) + " to storage index"); } m_dirty = m_dirty || added.value(); } @@ -169,13 +175,14 @@ class Storage { Expected remove(const Key& key) { auto removed = m_raw.remove(key); - if (!removed.has_value()) { + if (!removed) { return removed; } if (m_index.has_value()) { auto index_removed = m_index->remove(key); - if (!index_removed.has_value()) { - return Error::propagate(std::move(index_removed)); + if (!index_removed) { + return Error::propagate( + std::move(index_removed), "remove node " + Traits::key_to_string(key) + " from storage index"); } m_dirty = m_dirty || index_removed.value(); } @@ -191,8 +198,9 @@ class Storage { return m_raw.has(key); } auto status = m_index->get(key); - if (!status.has_value()) { - return Error::propagate(std::move(status)); + if (!status) { + return Error::propagate( + std::move(status), "look up node " + Traits::key_to_string(key) + " in storage index"); } return status->has_value() && status->value() != NodeStatus::Virtual; } @@ -202,7 +210,7 @@ class Storage { Expected path_for(const Key& key) const { auto node_paths = paths(key); - if (!node_paths.has_value()) { + if (!node_paths) { return Error::propagate(std::move(node_paths)); } return node_paths->empty() ? m_raw.layout().node_path(key) : node_paths->front(); @@ -251,7 +259,7 @@ class Storage { m_persistence->codec_selector, }; auto result = m_persistence->format.write(m_persistence->index_path, metadata); - if (result.has_value()) { + if (result) { m_dirty = false; } return result; @@ -282,7 +290,7 @@ class Storage { return; } const auto result = save_index(); - if (!result.has_value()) { + if (!result) { LOG_ERROR("Failed to automatically save index: {}", result.error().to_string()); } } diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h index 8c8e1597..5ac295c9 100644 --- a/src/terrainlib/store/open.h +++ b/src/terrainlib/store/open.h @@ -17,8 +17,8 @@ Expected> open_index( { using Key = typename Traits::Key; auto metadata_result = format.read(index_path); - if (!metadata_result.has_value()) { - return Error::propagate(std::move(metadata_result)); + if (!metadata_result) { + return Error::propagate(std::move(metadata_result), "read storage index \"" + index_path.string() + "\""); } IndexMetadata metadata = std::move(metadata_result.value()); if (metadata.payload_class != expected_payload_class) { @@ -36,8 +36,8 @@ Expected> open_index( return Error::fail(Error::Code::Unsupported, "unknown storage layout: " + metadata.layout_id); } auto codec = resolve_codec(metadata.codec_selector); - if (!codec.has_value()) { - return Error::propagate(std::move(codec)); + if (!codec) { + return Error::propagate(std::move(codec), "resolve storage codec \"" + metadata.codec_selector + "\""); } const std::filesystem::path base_path = index_path.parent_path(); diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h index 85cc533c..d98eeef8 100644 --- a/src/terrainlib/store/traverse.h +++ b/src/terrainlib/store/traverse.h @@ -34,8 +34,9 @@ Expected traverse(const Index& index, return store::invalid_key_error(root); } auto root_status = index.get(root); - if (!root_status.has_value()) { - return Error::propagate(std::move(root_status)); + if (!root_status) { + return Error::propagate( + std::move(root_status), "read traversal root status for node " + Traits::key_to_string(root)); } if (!root_status.value().has_value()) { return {}; @@ -45,8 +46,9 @@ Expected traverse(const Index& index, std::function(const Key&)> depth_first; depth_first = [&](const Key& current) -> Expected { auto status = index.get(current); - if (!status.has_value()) { - return Error::propagate(std::move(status)); + if (!status) { + return Error::propagate(std::move(status), + "read node status during depth-first traversal for " + Traits::key_to_string(current)); } if (!status.value().has_value()) { return {}; @@ -58,7 +60,7 @@ Expected traverse(const Index& index, if (children.has_value()) { for (const Key& child : children.value()) { auto result = depth_first(child); - if (!result.has_value()) { + if (!result) { return result; } } @@ -76,8 +78,9 @@ Expected traverse(const Index& index, queue.pop(); auto status = index.get(current); - if (!status.has_value()) { - return Error::propagate(std::move(status)); + if (!status) { + return Error::propagate(std::move(status), + "read node status during breadth-first traversal for " + Traits::key_to_string(current)); } if (!status.value().has_value()) { continue; From 30d7ec302bc1c5ac8ed50ff81c69acb88fa2d529 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:32:26 +0200 Subject: [PATCH 35/43] Add contextual Error propagation overload --- src/sf_merger/NodeWriter.h | 5 ++--- src/terrainlib/Error.cpp | 5 +++++ src/terrainlib/Error.h | 3 +++ unittests/terrainlib/error.cpp | 12 ++++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index 0b846cbe..c39a7c89 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -70,9 +70,8 @@ class NodeWriter { std::move(traversal), "traverse source subtree rooted at " + id.to_string() + " while copying"); } if (error.has_value()) { - return std::unexpected { - std::move(error.value()).with_context("copy subtree rooted at " + id.to_string() + " to output"), - }; + return Error::propagate( + std::move(error.value()), "copy subtree rooted at " + id.to_string() + " to output"); } return {}; } diff --git a/src/terrainlib/Error.cpp b/src/terrainlib/Error.cpp index 33d3200d..64d9251c 100644 --- a/src/terrainlib/Error.cpp +++ b/src/terrainlib/Error.cpp @@ -129,6 +129,11 @@ std::unexpected Error::propagate(Error&& error, const std::source_locatio return std::unexpected { std::move(error).with_context("propagated", location) }; } +std::unexpected Error::propagate(Error&& error, std::string message, const std::source_location location) +{ + return std::unexpected { std::move(error).with_context(std::move(message), location) }; +} + Error Error::with_context(std::string message, const std::source_location location) && { Error result = std::move(*this); diff --git a/src/terrainlib/Error.h b/src/terrainlib/Error.h index bace3f3d..e5d40834 100644 --- a/src/terrainlib/Error.h +++ b/src/terrainlib/Error.h @@ -95,6 +95,9 @@ class Error { std::source_location location = std::source_location::current()); static std::unexpected propagate(Error&& error, std::source_location location = std::source_location::current()); + static std::unexpected propagate(Error&& error, + std::string message, + std::source_location location = std::source_location::current()); [[nodiscard]] Error with_context( std::string message, diff --git a/unittests/terrainlib/error.cpp b/unittests/terrainlib/error.cpp index 44756de3..c2b7a611 100644 --- a/unittests/terrainlib/error.cpp +++ b/unittests/terrainlib/error.cpp @@ -38,3 +38,15 @@ TEST_CASE("Error::propagate uses a supplied context as its call-site frame", "[e CHECK(description.find("unittests/terrainlib/error.cpp") != std::string::npos); CHECK(description.find("decode payload") != std::string::npos); } + +TEST_CASE("Error::propagate adds supplied context to an Error", "[error]") +{ + Error source = Error::make(Error::Code::Io, "open input"); + const Expected result = Error::propagate(std::move(source), "read input"); + + REQUIRE_FALSE(result.has_value()); + const std::string description = result.error().to_string(); + CHECK(description.find("read input") != std::string::npos); + CHECK(description.find("unittests/terrainlib/error.cpp") != std::string::npos); + CHECK(description.find("open input") != std::string::npos); +} From ea52a92a2a28d808d930432b39abe400e0226784 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:58:02 +0200 Subject: [PATCH 36/43] Simplify Error propagation API --- src/terrainlib/Error.cpp | 5 ----- src/terrainlib/Error.h | 24 ++++-------------------- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/src/terrainlib/Error.cpp b/src/terrainlib/Error.cpp index 64d9251c..19d34121 100644 --- a/src/terrainlib/Error.cpp +++ b/src/terrainlib/Error.cpp @@ -124,11 +124,6 @@ std::unexpected Error::fail(const Code code, return std::unexpected { make(code, operation, source, destination, cause, location) }; } -std::unexpected Error::propagate(Error&& error, const std::source_location location) -{ - return std::unexpected { std::move(error).with_context("propagated", location) }; -} - std::unexpected Error::propagate(Error&& error, std::string message, const std::source_location location) { return std::unexpected { std::move(error).with_context(std::move(message), location) }; diff --git a/src/terrainlib/Error.h b/src/terrainlib/Error.h index e5d40834..4777d29b 100644 --- a/src/terrainlib/Error.h +++ b/src/terrainlib/Error.h @@ -83,10 +83,7 @@ class Error { template static std::unexpected propagate(std::expected&& result, - std::source_location location = std::source_location::current()); - template - static std::unexpected propagate(std::expected&& result, - std::string message, + std::string message = "propagated", std::source_location location = std::source_location::current()); template static std::unexpected propagate(std::expected&& result, @@ -94,23 +91,16 @@ class Error { std::string message, std::source_location location = std::source_location::current()); static std::unexpected propagate(Error&& error, - std::source_location location = std::source_location::current()); - static std::unexpected propagate(Error&& error, - std::string message, + std::string message = "propagated", std::source_location location = std::source_location::current()); - [[nodiscard]] Error with_context( - std::string message, - std::source_location location = std::source_location::current()) &&; - [[nodiscard]] Error reclassified(Code code, - std::string message, - std::source_location location = std::source_location::current()) &&; - Code code() const { return m_code; } std::string to_string() const; private: Error(Code code, Frame frame); + [[nodiscard]] Error with_context(std::string message, std::source_location location) &&; + [[nodiscard]] Error reclassified(Code code, std::string message, std::source_location location) &&; Code m_code; std::vector m_frames; @@ -119,12 +109,6 @@ class Error { template using Expected = std::expected; -template -std::unexpected Error::propagate(std::expected&& result, const std::source_location location) -{ - return propagate(std::move(result).error(), location); -} - template std::unexpected Error::propagate( std::expected&& result, std::string message, const std::source_location location) From 29f8b71394a598f2a55a97937ce74a854d7ea8d9 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:34:12 +0200 Subject: [PATCH 37/43] docs: add project code style --- .clang-format | 192 +++++++++++++++++++++++++++++++++++++++++++++ AGENTS.md | 7 ++ README.md | 93 +--------------------- docs/code-style.md | 122 ++++++++++++++++++++++++++++ 4 files changed, 322 insertions(+), 92 deletions(-) create mode 100644 .clang-format create mode 100644 AGENTS.md create mode 100644 docs/code-style.md diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..51fc0aa5 --- /dev/null +++ b/.clang-format @@ -0,0 +1,192 @@ +--- +Language: Cpp +# BasedOnStyle: WebKit +AccessModifierOffset: -4 +AlignAfterOpenBracket: DontAlign +AlignArrayOfStructures: None +AlignConsecutiveMacros: None +AlignConsecutiveAssignments: None +AlignConsecutiveBitFields: None +AlignConsecutiveDeclarations: None +AlignEscapedNewlines: Right +AlignOperands: DontAlign +AlignTrailingComments: false +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortEnumsOnASingleLine: true +AllowShortBlocksOnASingleLine: Empty +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: Yes +AttributeMacros: + - __capability +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: All +BreakBeforeConceptDeclarations: true +BreakBeforeBraces: WebKit +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeComma +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 160 +CommentPragmas: '^ IWYU pragma:' +QualifierAlignment: Leave +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +PackConstructorInitializers: BinPack +BasedOnStyle: '' +ConstructorInitializerAllOnOneLineOrOnePerLine: false +AllowAllConstructorInitializersOnNextLine: true +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseLabels: false +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock +IndentRequires: false +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +LambdaBodyIndentation: Signature +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: Inner +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 4 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PenaltyIndentedWhitespace: 0 +PointerAlignment: Left +PPIndentWidth: -1 +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + BeforeNonEmptyParentheses: false +SpaceAroundPointerQualifiers: Default +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: Never +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +BitFieldColonSpacing: Both +Standard: Latest +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Never +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE + - NS_SWIFT_NAME + - CF_SWIFT_NAME +... + diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..ebcaef91 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,7 @@ +# Agent instructions + +This is the alpine-terrain-builder project. + +Follow the project [code style](docs/code-style.md). + +Before committing, run `git ls-files --eol --cached --others --exclude-standard` and verify that each changed text file's working-tree line endings (`w/...`) agree with the line endings required by its Git attributes (`attr/...`). Fix unintended mixed or mismatched line endings before committing. diff --git a/README.md b/README.md index 011b8cd6..3cbf664e 100644 --- a/README.md +++ b/README.md @@ -1,94 +1,3 @@ # alpine-terrain-builder -## Overview - -This repository contains the following tools: - -1. [tile_downloader](src/tile_downloader) for downloading orthophoto tiles, which are used for texturing the generated meshes. -2. [terrainbuilder](src/terrainbuilder/) for building the highest quality meshes from the downloaded orthophoto tiles and a heightmap dataset. -3. [terrainmerger](src/terrainmerger/) for building a LOD hierarchy from the highest quality meshes by merging them upwards. -4. [terrainconvert](src/terrainconvert/) for converting .tile files into .glb files. - -### Core Library -[terrainlib](src/terrainlib/) contains classes common to all the tools above. - -### Scripts -The tools typically only handle one tile per command which makes it infeasible to run them manually for a whole region due to the sheer amount of tiles. Therefore, we've created the following python scripts, which automate calling the relevant tools for a specified regions: - -1. [batch-builder.py](batch-builder.py) calls [terrainbuilder](src/terrainbuilder/) for all tiles in the specified region. -2. [batch-merger.py](batch-merger.py) calls [terrainmerger](src/terrainmerger/) for all tiles in the specified region. -3. [batch-converter.py](batch-converter.py) calls [terrainconvert](src/terrainconvert/) for all .tile files in the specified directory and its subdirectories. - -## Example Workflow - -In this example, we'll show how to build the hierarchy for Vienna's city center (Zoom: 13, X: 4468, Y:2840). - -### 1. Downloading tiles -Tile coordinates always use the Google/Mapbox/XYZ convention internally: the origin is north-west, X grows east, and Y grows south. Google Maps, Mapbox, OpenStreetMap, and most XYZ services use the common URL order `{zoom}/{x}/{y}`. - -The `basemap` and `gataki` providers select their complete URL pattern and Y direction automatically. Both currently use downward Y with the URL order `{zoom}/{y}/{x}`. The `basemap` provider downloads the basemap.at orthophoto, and the Gataki mirror uses: - -`https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg` - -Example for the root tile: https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/13/2840/4468.jpeg - -For another service, pass a quoted `--url` pattern containing `{zoom}`, `{x}`, and `{y}`. Placeholder placement selects URL coordinate order. Custom URLs default to downward Y; use `--url-y-direction up` for legacy TMS. Downloaded files always use the common Google/Mapbox layout `{zoom}/{x}/{y}.jpeg`, independently of the remote URL. - -The following command downloads the mirror's `{zoom}/{y}/{x}` URLs and writes the root tile to `./tiles/13/4468/2840.jpeg`: - -``` -./tile-downloader --provider gataki --zoom 13 --x 4468 --y 2840 --max-zoom-level 19 -``` - -For example, a custom Google/Mapbox URL can be selected with `--url 'https://example.test/{zoom}/{x}/{y}.jpeg'`. - -### 2. Download heightmap dataset -The meshes are built from a heightmap dataset, therefore we need to download one. For this example we'll only use a small part of the complete dataset for the whole of austria (available at https://gataki.cg.tuwien.ac.at/raw/Oe_2020/, 268 GB to 1.1 TB). The part we're gonna use contains Vienna's city center and it is available at https://gataki.cg.tuwien.ac.at/raw/vienna/innenstadt_gs_1m_mgi.tif (228 MB). - -![Vienna city center heightmap](readme_img/innenstadt_gs_1m_mgi.jpg) - -### 3. Building reference meshes -The following command will build reference meshes for the root tile `13/4468/2840` at the target zoom level `19` from the tile textures in `./tiles/` and the dataset `./innenstadt_gs_1m_mgi.tif` and output them in the folder `./meshes`. The SRS used is `EPSG:3857`, as the [demo renderer](https://github.com/polskus/renderer-alpinite) expects coordinates in the WebMercator (EPSG:3857) system. - -``` -python3 batch-builder.py --terrainbuilder ./terrainbuilder --tile 13 4468 2840 --target-zoom 19 --output-dir ./meshes -- --textures ./tiles/ --dataset ./innenstadt_gs_1m_mgi.tif --mesh-srs EPSG:3857 -``` - -### 4. Building tile hierarchy by merging and simplifying reference meshes -The following command will build the LOD hierarchy from the reference meshes generated in the previous step at `./meshes` and output simplified GLB/GLTF files at `./hierarchy/{zoom}/{Y}/{X}.simplified.glb`. - -> [!WARNING] -> This process is very time intensive and may end up taking hours depending on the region size! - -``` -python3 batch-merger.py --terrainmerger ./terrainmerger --root-tile 13 4468 2840 --max-zoom 19 --input-dir ./meshes --output-dir ./hierarchy -``` - -### 5. Converting missing meshes from .TILE to .GLB -Due to the separation of concerns in step `3. Building reference meshes` the tilebuilder does not output .GLB files, and in step `4. Building tile hierarchy by merging and simplifying reference meshes` the terrainmerger only outputs simplified GLB files for the next higher level. Therefore, there are no GLB files present for the highest zoom level `19`. Fortunately, there is the terrainconvert tool, which enables the conversion between .TILE and .GLB files without simplification. -The following command runs a script, which checks if there is a `*.simplified.glb` file for every `*.tile` present. - -> [!NOTE] -> Even though we name the converted GLB files using the suffix `.simplified.glb`, there is no simplification happening during the conversion. This naming is only used for consistency's sake, as the terrainmerger creates GLB files with this suffix and the [demo renderer](https://github.com/polskus/renderer-alpinite) expects a consistent naming scheme. You could also, in theory, rename the whole hierarchy to just use `*.glb`, make the relevant change in the [demo renderer](https://github.com/polskus/renderer-alpinite) and it would work too. - -``` -python3 batch-converter.py --terrainconverter ./terrainconvert --dir ./hierarchy/19 --suffix .simplified -``` - -### 6. Deployment -To make the tiles accessible to the [demo renderer](https://github.com/polskus/renderer-alpinite) just serve the final hierarchy via a http(s) server, which the computer running the [demo renderer](https://github.com/polskus/renderer-alpinite) can access. The default URL format used by the [demo renderer](https://github.com/polskus/renderer-alpinite) is `/{zoom}/{Y}/{X}.simplified.glb`. - -See [Modifying the tile server URL](https://github.com/polskus/renderer-alpinite/blob/main/README.md#modifying-the-tile-server-url) for information on how to change the server URL, file structure and suffix. - -## building -In order to build, you need to install: - -- fmt (https://fmt.dev/latest/index.html) -- GDAL 3 (https://gdal.org/) -- glm (https://github.com/g-truc/glm) -- catch2 (version 2.x) (https://github.com/catchorg/Catch2/tree/v2.x) -- FreeImage -- tbb (intel threading building blocks) - -sudo apt-get install libcgal-dev libopencv-dev libfmt-dev libglm-dev libgdal-dev catch2 libfreeimage-dev libtbb-dev libcurl4-openssl-dev -(libgmp-dev libmpfr-dev libsqlite3-dev) +See the project [code style](docs/code-style.md) before contributing. diff --git a/docs/code-style.md b/docs/code-style.md new file mode 100644 index 00000000..bed47211 --- /dev/null +++ b/docs/code-style.md @@ -0,0 +1,122 @@ +# Code style + +## Core code style + +- Class names use CamelCase. Method, function, and variable names use snake_case. +- Class data members have an `m_` prefix and are usually private. Struct data members have no prefix and are usually public. +- Use `void set_attribute(int value)` and `int attribute() const` for setters and getters; avoid a `get_` prefix. Follow the [Qt recommendations](https://wiki.qt.io/API_Design_Principles#Naming_Boolean_Getters,_Setters,_and_Properties) for boolean getters. +- Structs are usually small and simple, have few or no methods, and never use inheritance. +- Files use CamelCase when they contain a CamelCase class. Otherwise, files use snake_case and their contents belong to a matching snake_case namespace. +- Reflect the directory and file structure in namespaces. For example, `folder/structure.h` corresponds to `namespace folder::structure { ... }`. +- Indent with four spaces. Do not use tabs. +- Format new code, and only new code, with the repository's [`.clang-format`](../.clang-format) file. Do not reformat unrelated existing code. +- Follow the [Qt API design principles](https://wiki.qt.io/API_Design_Principles) and the [C++ Core Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines) where this guide does not specify a rule. + +## Avoid repeated type names + +Avoid repeating the same name across namespaces and types. For example, `io::envelope::Envelope` is undesirable. A supporting type such as `io::envelope::Version` reads well, but placing the main class in that namespace forces its longer, repetitive name. + +Prefer `io::Envelope`. Put supporting types either inside `io::Envelope` as public types or in an `io::envelope_details` namespace. + +## Pointers vs. references + +Avoid out parameters in new APIs when practical; prefer returning a small struct. When an out parameter is necessary, use a pointer rather than a non-const reference. The address-of operator at the call site makes it clear that the argument can be modified: + +```cpp +color.get_hsv(&hue, &saturation, &value); +``` + +## The art of naming + +Names are a central part of API design. Avoid abbreviations, even familiar ones, because consistent full words are easier to understand and remember. Choose precise names that preserve useful context without adding redundant words. If a concept is difficult to name or describe in one sentence, reconsider whether it belongs in the API. + +Use consistent names for related classes and operations. A function name should make clear whether the function has side effects. Give parameters descriptive names and use the same names in declarations, definitions, and documentation. Avoid repeating an enum type's name in scoped enumerators when the scope already provides the context. + +## Error handling + +Functions that can fail return `Expected`, which is an alias for `std::expected`. + +### Checking results + +Use the short boolean form in control flow: + +```cpp +if (!result) { + // Handle error. +} + +if (result) { + // Handle value. +} +``` + +### Creating errors + +Use `Error::fail` where an error originates instead of `std::unexpected`: + +```cpp +return Error::fail( + Error::Code::InvalidInput, + "invalid storage layout"); +``` + +Choose the most specific error code available. + +### Forwarding errors + +Generally, errors can be forwarded with `Error::propagate`: + +```cpp +auto result = operation(); // Expected +if (!result) { + // result can be either a std::expected or an Error. + return Error::propagate(std::move(result)); +} +// The current function returns Expected. +``` + +`propagate` adds the source file and line as additional context. In some cases, it can be useful to avoid that: + +```cpp +auto result = operation(); +if (!result) { + return result; +} +``` + +This is appropriate for transparent helpers, recursive calls, and simple delegation where another error frame would add no useful information. When the success types differ and direct return is impossible, use `propagate`. + +### Adding context + +At meaningful abstraction or operation boundaries, propagate with a concise context message: + +```cpp +auto result = write_payload(path); +if (!result) { + // result can be either a std::expected or an Error. + return Error::propagate(std::move(result), "write DAG metadata"); +} +``` + +Useful context identifies what the caller was trying to accomplish, such as: + +- The domain operation: `"gather relevant input leaves"`. +- The data role: `"read DAG clustering"`. +- The affected node: `"add saved node " + key + " to storage index"`. +- The higher-level phase: `"finalize merged output dataset"`. + +Avoid merely repeating details already supplied by the lower-level error. For example, if the lower layer records the exact path and I/O operation, the caller should add the file's semantic role rather than repeat the path unnecessarily. + +The error type can be changed (reclassified) as well: + +```cpp +auto result = write_payload(path); +if (!result) { + return Error::propagate( + std::move(result), + Error::Code::CorruptData, + "write DAG metadata"); +} +``` + +Reclassify an existing error only if the current abstraction genuinely changes its meaning. From b0d1b0784e82f7165a7012519d3135bea88b4f96 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:28:24 +0200 Subject: [PATCH 38/43] Port touched code to project error convention --- src/dag_builder/codec/Dag.h | 17 ++-- src/dag_builder/codec/DagFormat.h | 82 +++++++++---------- src/dag_convert_debug/main.cpp | 2 +- src/mesh_convert/main.cpp | 9 +- src/sf_builder/terrainbuilder.cpp | 11 ++- src/sf_merger/NodeWriter.h | 11 ++- src/sf_merger/cut.h | 6 +- src/terrainlib/mesh/codec/SfMesh.h | 9 +- src/terrainlib/mesh/codec/SfMeshFormat.h | 30 +++---- src/terrainlib/mesh/io.cpp | 66 +++++++++++---- src/terrainlib/mesh/io.h | 8 +- src/terrainlib/octree/storage/IndexFile.h | 46 ++++++----- src/tile_builder/alpine_raster.cpp | 6 +- src/tile_builder/image_writer.cpp | 21 +++-- src/tile_builder/image_writer.h | 22 ++--- .../dag_builder/storage_compatibility.cpp | 8 +- unittests/sf_merger/integration.cpp | 2 +- unittests/terrainlib/mesh_io.cpp | 31 +++++-- unittests/terrainlib/store_compatibility.cpp | 8 +- unittests/tilebuilder/dataset_reading.cpp | 29 +++---- unittests/tilebuilder/image_writer.cpp | 16 ++-- 21 files changed, 264 insertions(+), 176 deletions(-) diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h index 32e4485f..3342b2f4 100644 --- a/src/dag_builder/codec/Dag.h +++ b/src/dag_builder/codec/Dag.h @@ -42,13 +42,11 @@ class Dag final : public store::Codec { } auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); if (!clustering) { - return Error::fail( - Error::Code::CorruptData, "invalid DAG clustering in \"" + node_paths[0].string() + "\": " + clustering.error()); + return Error::propagate(std::move(clustering), "decode DAG clustering read from \"" + node_paths[0].string() + "\""); } auto metadata = dag::format::decode_metadata(std::move(*metadata_payload)); if (!metadata) { - return Error::fail( - Error::Code::CorruptData, "invalid DAG metadata in \"" + node_paths[1].string() + "\": " + metadata.error()); + return Error::propagate(std::move(metadata), "decode DAG metadata read from \"" + node_paths[1].string() + "\""); } dag::ClusterBatch batch { .metadata = std::move(*metadata), @@ -70,17 +68,14 @@ class Dag final : public store::Codec { } auto clustering = dag::format::encode_clustering(batch.clustering); if (!clustering) { - return Error::fail( - Error::Code::InvalidInput, "cannot encode DAG clustering for \"" + node_paths[0].string() + "\": " + clustering.error()); + return Error::propagate(std::move(clustering), "encode DAG clustering for \"" + node_paths[0].string() + "\""); } const dag::format::NodeMetadata metadata = dag::format::encode_metadata(batch.metadata); if (auto valid = dag::format::validate(*clustering); !valid) { - return Error::fail( - Error::Code::InvalidInput, "invalid DAG clustering for \"" + node_paths[0].string() + "\": " + valid.error()); + return Error::propagate(std::move(valid), "validate DAG clustering for \"" + node_paths[0].string() + "\""); } if (auto valid = dag::format::validate(metadata); !valid) { - return Error::fail( - Error::Code::InvalidInput, "invalid DAG metadata for \"" + node_paths[1].string() + "\": " + valid.error()); + return Error::propagate(std::move(valid), "validate DAG metadata for \"" + node_paths[1].string() + "\""); } auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); if (!data_written) { @@ -110,7 +105,7 @@ class MetadataView final : public store::Codec { } auto metadata = dag::format::decode_metadata(std::move(*payload)); if (!metadata) { - return Error::fail(Error::Code::CorruptData, "invalid DAG metadata in \"" + path.string() + "\": " + metadata.error()); + return Error::propagate(std::move(metadata), "decode DAG metadata read from \"" + path.string() + "\""); } return std::move(*metadata); } diff --git a/src/dag_builder/codec/DagFormat.h b/src/dag_builder/codec/DagFormat.h index 27453f39..afff1bc9 100644 --- a/src/dag_builder/codec/DagFormat.h +++ b/src/dag_builder/codec/DagFormat.h @@ -7,10 +7,10 @@ #include #include -#include #include #include +#include "Error.h" #include "cluster.h" #include "dag_node.h" #include "io/envelope.h" @@ -85,11 +85,11 @@ using Clustering = ClusteringSchema::latest_type; inline v1::OctreeId encode_id(const octree::Id& id) { return { .level = id.level(), .index = id.index_on_level() }; } -inline std::expected decode_id(const v1::OctreeId& id) +inline Expected decode_id(const v1::OctreeId& id) { auto result = octree::Id::try_make(static_cast(id.level), static_cast(id.index)); if (!result) { - return std::unexpected("DAG payload contains an invalid octree ID"); + return Error::fail(Error::Code::CorruptData, "DAG payload contains an invalid octree ID"); } return *result; } @@ -120,30 +120,30 @@ inline NodeMetadata encode_metadata(const dag::NodeMetadata& metadata) return result; } -inline std::expected validate(const NodeMetadata& metadata) +inline Expected validate(const NodeMetadata& metadata) { for (const std::uint32_t group : metadata.group_assignment) { if (group >= metadata.groups.size()) { - return std::unexpected("DAG metadata contains an invalid group assignment"); + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains an invalid group assignment"); } } for (const v1::Group& group : metadata.groups) { if (!std::isfinite(group.error)) { - return std::unexpected("DAG metadata contains a non-finite error"); + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains a non-finite error"); } for (const v1::Id& child : group.children) { if (!decode_id(child.source_batch)) { - return std::unexpected("DAG metadata contains an invalid child ID"); + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains an invalid child ID"); } } } return {}; } -inline std::expected decode_metadata(NodeMetadata metadata) +inline Expected decode_metadata(NodeMetadata metadata) { if (auto valid = validate(metadata); !valid) { - return std::unexpected(valid.error()); + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate decoded DAG metadata"); } dag::NodeMetadata result; result.group_assignment = std::move(metadata.group_assignment); @@ -161,7 +161,7 @@ inline std::expected decode_metadata(NodeMetadat for (const v1::Id& child : group.children) { auto source_batch = decode_id(child.source_batch); if (!source_batch) { - return std::unexpected(source_batch.error()); + return Error::propagate(std::move(source_batch), "decode DAG metadata child ID"); } decoded_group.children.push_back({ *source_batch, child.cluster_index }); } @@ -170,7 +170,7 @@ inline std::expected decode_metadata(NodeMetadat return result; } -inline std::expected encode_clustering(const ::Clustering& clustering) +inline Expected encode_clustering(const ::Clustering& clustering) { Clustering result { .vertex_count = clustering.positions.size(), @@ -181,7 +181,7 @@ inline std::expected encode_clustering(const ::Clusteri result.clusters.reserve(clustering.clusters.size()); for (const ::Cluster& cluster : clustering.clusters) { if (cluster.texture_id == (std::numeric_limits::max)()) { - return std::unexpected("DAG texture ID collides with the on-disk null sentinel"); + return Error::fail(Error::Code::InvalidInput, "DAG texture ID collides with the on-disk null sentinel"); } result.clusters.push_back({ .triangle_count = cluster.local_triangles.size(), @@ -200,12 +200,12 @@ inline std::expected encode_clustering(const ::Clusteri for (const cv::Mat& texture : clustering.textures) { auto jpeg = mesh::io::write_texture_to_encoded_buffer(texture, ".jpeg"); if (!texture.empty() && jpeg.empty()) { - return std::unexpected("could not encode DAG texture"); + return Error::fail(Error::Code::InvalidInput, "could not encode DAG texture"); } result.textures.push_back({ std::move(jpeg) }); } - } catch (const cv::Exception&) { - return std::unexpected("could not encode DAG texture"); + } catch (const cv::Exception& error) { + return Error::fail(Error::Code::InvalidInput, "could not encode DAG texture: " + error.msg); } return result; } @@ -215,77 +215,77 @@ inline bool count_fits(const std::uint64_t count, const std::size_t element_size return count <= std::numeric_limits::max() / element_size && count * element_size <= io::envelope::default_max_decompressed_size; } -inline std::expected validate(const Clustering& clustering) +inline Expected validate(const Clustering& clustering) { if (!count_fits(clustering.vertex_count, sizeof(glm::dvec3))) { - return std::unexpected("DAG position count exceeds the allocation limit"); + return Error::fail(Error::Code::InvalidInput, "DAG position count exceeds the allocation limit"); } if (clustering.vertex_count != 0 && clustering.encoded_positions.empty()) { - return std::unexpected("DAG position count and encoded data disagree"); + return Error::fail(Error::Code::InvalidInput, "DAG position count and encoded data disagree"); } std::size_t decoded_size = static_cast(clustering.vertex_count) * sizeof(glm::dvec3); for (const v1::Cluster& cluster : clustering.clusters) { if (!count_fits(cluster.triangle_count, sizeof(glm::uvec3)) || !count_fits(cluster.vertex_count, sizeof(std::uint32_t)) || !count_fits(cluster.uv_count, sizeof(glm::dvec2))) { - return std::unexpected("DAG cluster count exceeds the allocation limit"); + return Error::fail(Error::Code::InvalidInput, "DAG cluster count exceeds the allocation limit"); } const std::size_t cluster_size = static_cast(cluster.triangle_count) * sizeof(glm::uvec3) + static_cast(cluster.vertex_count) * sizeof(std::uint32_t) + static_cast(cluster.uv_count) * sizeof(glm::dvec2); if (cluster_size > io::envelope::default_max_decompressed_size - decoded_size) { - return std::unexpected("DAG decoded data exceeds the allocation limit"); + return Error::fail(Error::Code::InvalidInput, "DAG decoded data exceeds the allocation limit"); } decoded_size += cluster_size; if (cluster.uv_count != 0 && cluster.uv_count != cluster.vertex_count) { - return std::unexpected("DAG cluster UV count does not match its vertex count"); + return Error::fail(Error::Code::InvalidInput, "DAG cluster UV count does not match its vertex count"); } if ((cluster.triangle_count != 0 && cluster.encoded_triangles.empty()) || (cluster.vertex_count != 0 && cluster.encoded_vertex_indices.empty()) || (cluster.uv_count != 0 && cluster.encoded_uvs.empty())) { - return std::unexpected("DAG cluster count and encoded data disagree"); + return Error::fail(Error::Code::InvalidInput, "DAG cluster count and encoded data disagree"); } if (cluster.texture_id != (std::numeric_limits::max)() && cluster.texture_id >= clustering.textures.size()) { - return std::unexpected("DAG cluster contains an invalid texture reference"); + return Error::fail(Error::Code::InvalidInput, "DAG cluster contains an invalid texture reference"); } if (!std::isfinite(cluster.absolute_error)) { - return std::unexpected("DAG cluster contains a non-finite error"); + return Error::fail(Error::Code::InvalidInput, "DAG cluster contains a non-finite error"); } } return {}; } template -inline std::expected, std::string> decode_vertex_buffer( +inline Expected> decode_vertex_buffer( const std::uint64_t count, const std::vector& encoded, const std::string_view label) { if (!count_fits(count, sizeof(T))) { - return std::unexpected(std::string(label) + " count exceeds the allocation limit"); + return Error::fail(Error::Code::CorruptData, std::string(label) + " count exceeds the allocation limit"); } std::vector result(static_cast(count)); if (meshopt_decodeVertexBuffer(result.data(), result.size(), sizeof(T), encoded.data(), encoded.size()) != 0) { - return std::unexpected("could not decode " + std::string(label)); + return Error::fail(Error::Code::CorruptData, "could not decode " + std::string(label)); } return result; } -inline std::expected, std::string> decode_triangles(const std::uint64_t count, const std::vector& encoded) +inline Expected> decode_triangles(const std::uint64_t count, const std::vector& encoded) { if (!count_fits(count, sizeof(glm::uvec3))) { - return std::unexpected("DAG triangle count exceeds the allocation limit"); + return Error::fail(Error::Code::CorruptData, "DAG triangle count exceeds the allocation limit"); } std::vector result(static_cast(count)); if (meshopt_decodeIndexBuffer(result.data(), result.size() * 3, sizeof(std::uint32_t), encoded.data(), encoded.size()) != 0) { - return std::unexpected("could not decode DAG triangles"); + return Error::fail(Error::Code::CorruptData, "could not decode DAG triangles"); } return result; } -inline std::expected<::Clustering, std::string> decode_clustering(Clustering clustering) +inline Expected<::Clustering> decode_clustering(Clustering clustering) { if (auto valid = validate(clustering); !valid) { - return std::unexpected(valid.error()); + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate decoded DAG clustering"); } auto positions = decode_vertex_buffer(clustering.vertex_count, clustering.encoded_positions, "DAG positions"); if (!positions) { - return std::unexpected(positions.error()); + return Error::propagate(std::move(positions), "decode DAG positions"); } ::Clustering result; @@ -296,22 +296,22 @@ inline std::expected<::Clustering, std::string> decode_clustering(Clustering clu auto vertex_indices = decode_vertex_buffer(encoded.vertex_count, encoded.encoded_vertex_indices, "DAG vertex indices"); auto uvs = decode_vertex_buffer(encoded.uv_count, encoded.encoded_uvs, "DAG UVs"); if (!triangles) { - return std::unexpected(triangles.error()); + return Error::propagate(std::move(triangles), "decode DAG cluster triangles"); } if (!vertex_indices) { - return std::unexpected(vertex_indices.error()); + return Error::propagate(std::move(vertex_indices), "decode DAG cluster vertex indices"); } if (!uvs) { - return std::unexpected(uvs.error()); + return Error::propagate(std::move(uvs), "decode DAG cluster UVs"); } for (const std::uint32_t vertex : *vertex_indices) { if (vertex >= result.positions.size()) { - return std::unexpected("DAG cluster contains an invalid global vertex index"); + return Error::fail(Error::Code::CorruptData, "DAG cluster contains an invalid global vertex index"); } } for (const glm::uvec3 triangle : *triangles) { if (glm::any(glm::greaterThanEqual(triangle, glm::uvec3(static_cast(vertex_indices->size()))))) { - return std::unexpected("DAG cluster contains an invalid local triangle index"); + return Error::fail(Error::Code::CorruptData, "DAG cluster contains an invalid local triangle index"); } } result.clusters.push_back({ @@ -331,12 +331,12 @@ inline std::expected<::Clustering, std::string> decode_clustering(Clustering clu for (const v1::Texture& texture : clustering.textures) { cv::Mat image = mesh::io::read_texture_from_encoded_bytes(texture.jpeg); if (image.empty()) { - return std::unexpected("could not decode DAG texture"); + return Error::fail(Error::Code::CorruptData, "could not decode DAG texture"); } textures.push_back(std::move(image)); } - } catch (const cv::Exception&) { - return std::unexpected("could not decode DAG texture"); + } catch (const cv::Exception& error) { + return Error::fail(Error::Code::CorruptData, "could not decode DAG texture: " + error.msg); } result.textures = TextureSet(std::move(textures)); return result; diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index 005c054d..71c96cd0 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -32,7 +32,7 @@ void export_node(const cli::Args &args) { const auto save_result = mesh::io::save_to_path(mesh, args.output_path); if (!save_result) { - LOG_ERROR("Failed to save mesh to {}: {}", args.output_path, save_result.error().description()); + LOG_ERROR("Failed to save mesh to {}: {}", args.output_path, save_result.error().to_string()); } } diff --git a/src/mesh_convert/main.cpp b/src/mesh_convert/main.cpp index 2d812705..a2f742ab 100644 --- a/src/mesh_convert/main.cpp +++ b/src/mesh_convert/main.cpp @@ -1,7 +1,6 @@ #include #include -#include #include "mesh/SimpleMesh.h" #include "mesh/io.h" @@ -11,9 +10,9 @@ void run(const cli::Args& args) { LOG_INFO("Loading input mesh..."); - const std::expected load_result = mesh::io::load_from_path(args.input_path); + const auto load_result = mesh::io::load_from_path(args.input_path); if (!load_result) { - LOG_ERROR("Failed to load mesh: {}", load_result.error().description()); + LOG_ERROR("Failed to load mesh: {}", load_result.error().to_string()); return; } SimpleMesh mesh = load_result.value(); @@ -30,9 +29,9 @@ void run(const cli::Args& args) { } LOG_INFO("Writing output mesh..."); - const std::expected save_result = mesh::io::save_to_path(mesh, args.output_path); + const auto save_result = mesh::io::save_to_path(mesh, args.output_path); if (!save_result) { - LOG_ERROR("Failed to save mesh: {}", save_result.error().description()); + LOG_ERROR("Failed to save mesh: {}", save_result.error().to_string()); return; } } diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 9c33f956..6195b213 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -154,8 +154,9 @@ void build_and_save_patch( // metadata["texture_bounds"] = fmt::format( // "{{ \"min\": {{ \"x\": {}, \"y\": {} }}, \"max\": {{ \"x\": {}, \"y\": {} }} }}", // texture_bounds.min.x, texture_bounds.min.y, texture_bounds.max.x, texture_bounds.max.y); - if (!mesh::io::save_to_path(mesh, output_path, mesh::io::SaveOptions{.metadata = metadata})) { - LOG_ERROR("Failed to save mesh to file {}", output_path); + const auto saved = mesh::io::save_to_path(mesh, output_path, mesh::io::SaveOptions{.metadata = metadata}); + if (!saved) { + LOG_ERROR("Failed to save mesh to file {}: {}", output_path, saved.error().to_string()); std::exit(2); } LOG_DEBUG("Writing mesh took {}s", format_secs_since(start)); @@ -195,10 +196,8 @@ Expected build_all_patches( output_base_path, std::move(open_options)); if (!storage_result) { - LOG_ERROR_AND_EXIT( - "Failed to open output dataset {}: {}", - output_base_path, - storage_result.error().to_string()); + return Error::propagate( + std::move(storage_result), "open output terrain dataset \"" + output_base_path.string() + "\""); } mesh::storage::Storage raw_storage = std::move(storage_result.value()); raw_storage.settings().allow_overwrite = overwrite_existing; diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index c39a7c89..3bca914f 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -4,7 +4,9 @@ #include #include +#include +#include "Error.h" #include "NodeLoader.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" @@ -38,7 +40,14 @@ class NodeWriter { // change extension to .png p.replace_extension(".png"); if (mesh.texture.has_value()) { - cv::imwrite(p, mesh.texture.value()); + try { + if (!cv::imwrite(p, mesh.texture.value())) { + return Error::fail(Error::Code::Io, "write node texture to", p); + } + } catch (const cv::Exception& error) { + return Error::fail( + Error::Code::Io, "write node texture to \"" + p.string() + "\": " + error.what()); + } } return {}; } diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 4589012d..b06437f0 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -36,7 +36,11 @@ inline Expected cut_leaf_node( DEBUG_ASSERT(DEBUG_ASSERT_VAL( ctx.input.index().is(store::NodeStatus::Leaf, id)).value()); - const SimpleMesh mesh = DEBUG_ASSERT_VAL(ctx.input.load(id)).value(); + auto mesh_result = ctx.input.load(id); + if (!mesh_result) { + return Error::propagate(std::move(mesh_result), "read cut input node " + id.to_string()); + } + const SimpleMesh mesh = std::move(*mesh_result); LOG_TRACE("Cutting mesh at {} using mask with {} vertices and {} triangles", id, mask.vertex_count(), mask.face_count()); const Cow clipped = clip_on_mask(mesh, mask, ctx.keep_inside); diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h index db64bb02..37a88b5c 100644 --- a/src/terrainlib/mesh/codec/SfMesh.h +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -25,11 +25,12 @@ class SfMesh final : public store::Codec { return Error::propagate(std::move(payload), "read SF mesh"); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return Error::fail(Error::Code::CorruptData, "invalid SF mesh in \"" + path.string() + "\": " + valid.error()); + return Error::propagate( + std::move(valid), Error::Code::CorruptData, "validate SF mesh read from \"" + path.string() + "\""); } auto decoded = mesh::sf::decode_payload(std::move(*payload)); if (!decoded) { - return Error::fail(Error::Code::CorruptData, "could not decode SF mesh \"" + path.string() + "\""); + return Error::propagate(std::move(decoded), "decode SF mesh \"" + path.string() + "\""); } return std::move(*decoded); } @@ -44,10 +45,10 @@ class SfMesh final : public store::Codec { const std::filesystem::path path = paths(node_path).front(); auto payload = mesh::sf::encode_payload(mesh, options); if (!payload) { - return Error::fail(Error::Code::InvalidInput, "could not encode SF mesh \"" + path.string() + "\""); + return Error::propagate(std::move(payload), "encode SF mesh \"" + path.string() + "\""); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return Error::fail(Error::Code::InvalidInput, "invalid SF mesh for \"" + path.string() + "\": " + valid.error()); + return Error::propagate(std::move(valid), "validate SF mesh for \"" + path.string() + "\""); } auto result = ::io::envelope::write_to_path(*payload, path); if (!result) { diff --git a/src/terrainlib/mesh/codec/SfMeshFormat.h b/src/terrainlib/mesh/codec/SfMeshFormat.h index c2e17de8..d97de5cd 100644 --- a/src/terrainlib/mesh/codec/SfMeshFormat.h +++ b/src/terrainlib/mesh/codec/SfMeshFormat.h @@ -1,11 +1,9 @@ #pragma once #include -#include #include -#include - +#include "Error.h" #include "io/envelope.h" #include "mesh/EncodedMesh.h" #include "mesh/SimpleMesh.h" @@ -29,34 +27,34 @@ namespace v1 { using Schema = ::io::envelope::PayloadSchema<"mesh.SfMesh", ::io::envelope::Version<1, v1::Payload>>; using Payload = Schema::latest_type; -inline std::expected validate(const Payload& payload) +inline Expected validate(const Payload& payload) { if (payload.vertex_count > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Position)) { - return std::unexpected("SF mesh vertex count exceeds the allocation limit"); + return Error::fail(Error::Code::InvalidInput, "SF mesh vertex count exceeds the allocation limit"); } if (payload.face_count > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Triangle)) { - return std::unexpected("SF mesh face count exceeds the allocation limit"); + return Error::fail(Error::Code::InvalidInput, "SF mesh face count exceeds the allocation limit"); } if (payload.vertex_count == 0 && (!payload.positions.empty() || !payload.uvs.empty())) { - return std::unexpected("empty mesh contains vertex data"); + return Error::fail(Error::Code::InvalidInput, "empty mesh contains vertex data"); } if (payload.vertex_count != 0 && payload.positions.empty()) { - return std::unexpected("non-empty mesh contains no position data"); + return Error::fail(Error::Code::InvalidInput, "non-empty mesh contains no position data"); } if (payload.face_count == 0 && !payload.triangles.empty()) { - return std::unexpected("empty mesh contains triangle data"); + return Error::fail(Error::Code::InvalidInput, "empty mesh contains triangle data"); } if (payload.face_count != 0 && payload.triangles.empty()) { - return std::unexpected("non-empty mesh contains no triangle data"); + return Error::fail(Error::Code::InvalidInput, "non-empty mesh contains no triangle data"); } return {}; } -inline std::expected encode_payload(const mesh::Simple& mesh, const mesh::EncodeOptions options = {}) +inline Expected encode_payload(const mesh::Simple& mesh, const mesh::EncodeOptions options = {}) { auto encoded = mesh::encode(mesh, options); if (!encoded) { - return std::unexpected(encoded.error()); + return Error::fail(Error::Code::InvalidInput, "could not encode SF mesh payload"); } return Payload { .vertex_count = encoded->header.vertex_count, @@ -68,7 +66,7 @@ inline std::expected encode_payload(const mesh::Simp }; } -inline std::expected decode_payload(Payload payload) +inline Expected decode_payload(Payload payload) { const mesh::Encoded encoded{ .header = { @@ -83,7 +81,11 @@ inline std::expected decode_payload(Payload pay .uvs = std::move(payload.uvs), .texture = std::move(payload.texture), }; - return mesh::decode(encoded); + auto decoded = mesh::decode(encoded); + if (!decoded) { + return Error::fail(Error::Code::CorruptData, "could not decode SF mesh payload"); + } + return std::move(*decoded); } } // namespace mesh::sf diff --git a/src/terrainlib/mesh/io.cpp b/src/terrainlib/mesh/io.cpp index 7be3eee4..4f91ae50 100644 --- a/src/terrainlib/mesh/io.cpp +++ b/src/terrainlib/mesh/io.cpp @@ -6,27 +6,61 @@ namespace mesh::io { -std::expected load_from_path( +namespace { + +Error load_error(const LoadMeshError error, const std::filesystem::path& path) +{ + switch (static_cast(error)) { + case LoadMeshErrorKind::UnsupportedFormat: + return Error::make(Error::Code::Unsupported, "load unsupported glTF format from", path); + case LoadMeshErrorKind::FileNotFound: + return Error::make(Error::Code::NotFound, "open glTF file", path); + case LoadMeshErrorKind::InvalidFormat: + return Error::make(Error::Code::CorruptData, "decode invalid glTF file", path); + case LoadMeshErrorKind::OutOfMemory: + return Error::make(Error::Code::ResourceExhausted, "load glTF file: out of memory", path); + } + return Error::make(Error::Code::Internal, "unknown glTF load failure for \"" + path.string() + "\""); +} + +Error save_error(const SaveMeshError error, const std::filesystem::path& path) +{ + switch (static_cast(error)) { + case SaveMeshErrorKind::UnsupportedFormat: + return Error::make(Error::Code::Unsupported, "write unsupported glTF format to", path); + case SaveMeshErrorKind::OpenFile: + return Error::make(Error::Code::Io, "open glTF file for writing", path); + case SaveMeshErrorKind::WriteFile: + return Error::make(Error::Code::Io, "write glTF file", path); + case SaveMeshErrorKind::OutOfMemory: + return Error::make(Error::Code::ResourceExhausted, "write glTF file: out of memory", path); + } + return Error::make(Error::Code::Internal, "unknown glTF write failure for \"" + path.string() + "\""); +} + +} // namespace + +Expected load_from_path( const std::filesystem::path &path, const LoadOptions& options) { const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { - return gltf::load_from_path(path, options); + auto result = gltf::load_from_path(path, options); + if (!result) { + return Error::propagate(load_error(result.error(), path), "load glTF mesh"); + } + return std::move(*result); } else if (extension == ".sfmesh") { std::filesystem::path node_path = path; node_path.replace_extension(); const mesh::codec::SfMesh codec; - auto result = codec.read(node_path); - if (!result) { - return std::unexpected(LoadMeshErrorKind::InvalidFormat); - } - return std::move(*result); + return codec.read(node_path); } else { - return std::unexpected(LoadMeshErrorKind::UnsupportedFormat); + return Error::fail(Error::Code::Unsupported, "unsupported mesh input format: " + extension.string()); } } -std::expected save_to_path( +Expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options) { @@ -36,18 +70,18 @@ std::expected save_to_path( const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { - return gltf::save_to_path(mesh, path, options); + auto result = gltf::save_to_path(mesh, path, options); + if (!result) { + return Error::propagate(save_error(result.error(), path), "write glTF mesh"); + } + return {}; } else if (extension == ".sfmesh") { std::filesystem::path node_path = path; node_path.replace_extension(); const mesh::codec::SfMesh codec; - auto result = codec.write(node_path, mesh, mesh::EncodeOptions{.texture_format = options.texture_format}); - if (!result) { - return std::unexpected(SaveMeshErrorKind::WriteFile); - } - return {}; + return codec.write(node_path, mesh, mesh::EncodeOptions{.texture_format = options.texture_format}); } else { - return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); + return Error::fail(Error::Code::Unsupported, "unsupported mesh output format: " + extension.string()); } } diff --git a/src/terrainlib/mesh/io.h b/src/terrainlib/mesh/io.h index ea474c73..d05242ec 100644 --- a/src/terrainlib/mesh/io.h +++ b/src/terrainlib/mesh/io.h @@ -2,19 +2,17 @@ #include -#include - +#include "Error.h" #include "mesh/SimpleMesh.h" #include "mesh/io/options.h" -#include "mesh/io/error.h" namespace mesh::io { -std::expected load_from_path( +Expected load_from_path( const std::filesystem::path &path, const LoadOptions& options = {}); -std::expected save_to_path( +Expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options = {}); diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index 3ed4ea94..f724123b 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -43,16 +43,16 @@ using StoreIndexSchema = io::envelope::PayloadSchema<"octree.StoreIndex", io::en using StoreMetadata = StoreMetadataSchema::latest_type; using StoreIndex = StoreIndexSchema::latest_type; -inline std::expected validate(const StoreMetadata& metadata) +inline Expected validate(const StoreMetadata& metadata) { if (metadata.layout_id.empty()) { - return std::unexpected("layout ID is empty"); + return Error::fail(Error::Code::InvalidInput, "layout ID is empty"); } if (metadata.payload_class.empty()) { - return std::unexpected("payload class is empty"); + return Error::fail(Error::Code::InvalidInput, "payload class is empty"); } if (metadata.codec_selector.empty()) { - return std::unexpected("codec selector is empty"); + return Error::fail(Error::Code::InvalidInput, "codec selector is empty"); } return {}; } @@ -71,24 +71,24 @@ inline StoreIndex encode_index(const store::Index& index) return result; } -inline std::expected, std::string> decode_index(const StoreIndex& encoded) +inline Expected> decode_index(const StoreIndex& encoded) { store::Index result; std::unordered_set seen; for (const v1::IndexEntry& entry : encoded.entries) { const auto id = Id::try_make(static_cast(entry.level), static_cast(entry.index)); if (!id) { - return std::unexpected("index contains an invalid octree ID"); + return Error::fail(Error::Code::CorruptData, "index contains an invalid octree ID"); } if (!seen.insert(*id).second) { - return std::unexpected("index contains a duplicate octree ID"); + return Error::fail(Error::Code::CorruptData, "index contains a duplicate octree ID"); } if (entry.status > static_cast(store::NodeStatus::Virtual)) { - return std::unexpected("index contains an invalid node status"); + return Error::fail(Error::Code::CorruptData, "index contains an invalid node status"); } - const auto set = result.set_raw(*id, store::NodeStatus { static_cast(entry.status) }); + auto set = result.set_raw(*id, store::NodeStatus { static_cast(entry.status) }); if (!set) { - return std::unexpected("index contains an invalid octree ID"); + return Error::propagate(std::move(set), Error::Code::CorruptData, "add decoded node to store index"); } } @@ -97,27 +97,32 @@ inline std::expected, std::string> decode_index(const bool has_child = false; if (children) { for (const Id& child : *children) { - const auto child_status = result.get(child); + auto child_status = result.get(child); if (!child_status) { - return std::unexpected("index child lookup failed"); + return Error::propagate( + std::move(child_status), Error::Code::CorruptData, "look up child while validating store index"); } has_child = has_child || child_status->has_value(); } } if ((status == store::NodeStatus::Leaf && has_child) || ((status == store::NodeStatus::Inner || status == store::NodeStatus::Virtual) && !has_child)) { - return std::unexpected("index contains inconsistent node topology"); + return Error::fail(Error::Code::CorruptData, "index contains inconsistent node topology"); } const auto parent = StoreTraits::parent(id); if (!parent) { if (id != StoreTraits::root()) { - return std::unexpected("index contains a non-root node without a parent"); + return Error::fail(Error::Code::CorruptData, "index contains a non-root node without a parent"); } continue; } - const auto parent_status = result.get(*parent); - if (!parent_status || !parent_status->has_value() || parent_status->value() == store::NodeStatus::Leaf) { - return std::unexpected("index contains a node without a valid indexed parent"); + auto parent_status = result.get(*parent); + if (!parent_status) { + return Error::propagate( + std::move(parent_status), Error::Code::CorruptData, "look up parent while validating store index"); + } + if (!parent_status->has_value() || parent_status->value() == store::NodeStatus::Leaf) { + return Error::fail(Error::Code::CorruptData, "index contains a node without a valid indexed parent"); } } return result; @@ -131,7 +136,8 @@ inline Expected read_store_metadata(const std::filesystem::path& return metadata; } if (auto valid = validate(*metadata); !valid) { - return Error::fail(Error::Code::CorruptData, "invalid store metadata in " + path.string() + ": " + valid.error()); + return Error::propagate( + std::move(valid), Error::Code::CorruptData, "validate store metadata read from \"" + path.string() + "\""); } return std::move(*metadata); } @@ -149,7 +155,7 @@ inline Expected> read_index_file(const std::fi } auto index = decode_index(*encoded_index); if (!index) { - return Error::fail(Error::Code::CorruptData, "invalid store index in " + path.string() + ": " + index.error()); + return Error::propagate(std::move(index), "decode store index read from \"" + path.string() + "\""); } return store::IndexMetadata { std::move(*index), @@ -167,7 +173,7 @@ inline Expected write_index_file(const std::filesystem::path& path, const .codec_selector = metadata.codec_selector, }; if (auto valid = validate(store_metadata); !valid) { - return Error::fail(Error::Code::InvalidInput, "invalid store metadata: " + valid.error()); + return Error::propagate(std::move(valid), "validate store metadata for writing"); } const std::filesystem::path metadata_path = path.parent_path() / metadata_file_name; if (auto written = io::envelope::write_to_path(store_metadata, metadata_path); !written) { diff --git a/src/tile_builder/alpine_raster.cpp b/src/tile_builder/alpine_raster.cpp index f1e387d9..7e4b1a0c 100644 --- a/src/tile_builder/alpine_raster.cpp +++ b/src/tile_builder/alpine_raster.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -38,5 +39,8 @@ ParallelTileGenerator alpine_raster::make_generator(const std::string& input_dat void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const radix::Raster& heights) const { - image::save_image_as_png(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); + auto result = image::save_image_as_png(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); + if (!result) { + throw std::runtime_error(result.error().to_string()); + } } diff --git a/src/tile_builder/image_writer.cpp b/src/tile_builder/image_writer.cpp index 91ee199c..c0131610 100644 --- a/src/tile_builder/image_writer.cpp +++ b/src/tile_builder/image_writer.cpp @@ -23,24 +23,29 @@ #include "io/conversion.h" #include -#include -void image::save_image_as_png(const radix::Raster& input_image, const std::string& path) +Expected image::save_image_as_png(const radix::Raster& input_image, const std::string& path) { auto converted = io::conversion::to_mat(input_image); if (!converted) { - throw std::runtime_error("Failed to convert raster to OpenCV image"); + return Error::propagate(std::move(converted), "convert raster for PNG output"); } cv::Mat flipped; - cv::flip(*converted, flipped, 0); cv::Mat image; - cv::cvtColor(flipped, image, cv::COLOR_RGB2BGR); + try { + cv::flip(*converted, flipped, 0); + cv::cvtColor(flipped, image, cv::COLOR_RGB2BGR); + } catch (const cv::Exception& error) { + return Error::fail(Error::Code::Internal, "prepare PNG image: " + error.msg); + } try { - if (!cv::imwrite(path, image)) - throw std::runtime_error("Failed to write PNG image to " + path); + if (!cv::imwrite(path, image)) { + return Error::fail(Error::Code::Io, "write PNG image to \"" + path + "\""); + } } catch (const cv::Exception& error) { - throw std::runtime_error("Failed to write PNG image to " + path + ": " + error.what()); + return Error::fail(Error::Code::Io, "write PNG image to \"" + path + "\": " + error.msg); } + return {}; } diff --git a/src/tile_builder/image_writer.h b/src/tile_builder/image_writer.h index 37d697da..9c72598f 100644 --- a/src/tile_builder/image_writer.h +++ b/src/tile_builder/image_writer.h @@ -23,29 +23,31 @@ #include #include #include -#include #include #include #include +#include "Error.h" + namespace image { -void save_image_as_png(const radix::Raster& image, const std::string& path); +Expected save_image_as_png(const radix::Raster& image, const std::string& path); template -void debug_out(const radix::Raster& image, const std::string& path) +Expected debug_out(const radix::Raster& image, const std::string& path) { - if (image.buffer().empty()) - throw std::invalid_argument("Can't write an empty raster to " + path); + if (image.buffer().empty()) { + return Error::fail(Error::Code::InvalidInput, "cannot write an empty debug raster to \"" + path + "\""); + } const auto [min, max] = std::ranges::minmax(image); const auto range = float(max) - float(min); - save_image_as_png(radix::raster::transform(image, - [min, range](const auto value) { - const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); - return glm::u8vec3(intensity); - }), + return save_image_as_png(radix::raster::transform(image, + [min, range](const auto value) { + const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); + return glm::u8vec3(intensity); + }), path); } diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 941ed1dd..51c18a85 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -109,7 +109,9 @@ TEST_CASE("versioned DAG payload validation is free-standing") .group_assignment = { 0 }, .groups = {}, }; - CHECK_FALSE(dag::format::validate(invalid_metadata).has_value()); + const auto validated_metadata = dag::format::validate(invalid_metadata); + REQUIRE_FALSE(validated_metadata.has_value()); + CHECK(validated_metadata.error().code() == Error::Code::InvalidInput); dag::format::Clustering oversized_clustering { .vertex_count = std::numeric_limits::max(), @@ -117,7 +119,9 @@ TEST_CASE("versioned DAG payload validation is free-standing") .clusters = {}, .textures = {}, }; - CHECK_FALSE(dag::format::validate(oversized_clustering).has_value()); + const auto validated_clustering = dag::format::validate(oversized_clustering); + REQUIRE_FALSE(validated_clustering.has_value()); + CHECK(validated_clustering.error().code() == Error::Code::InvalidInput); } TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") diff --git a/unittests/sf_merger/integration.cpp b/unittests/sf_merger/integration.cpp index faa25e0d..bb7e68e0 100644 --- a/unittests/sf_merger/integration.cpp +++ b/unittests/sf_merger/integration.cpp @@ -115,7 +115,7 @@ TEST_CASE("SF builders preserve geometry at regular and point-touching mask bord const auto regression_mesh = mesh::io::load_from_path(regression_path); const std::string regression_error = regression_mesh.has_value() ? std::string() - : regression_mesh.error().description(); + : regression_mesh.error().to_string(); INFO("Point-touching mask-border merge: " << regression_error); REQUIRE(regression_mesh.has_value()); CHECK_FALSE(regression_mesh->is_empty()); diff --git a/unittests/terrainlib/mesh_io.cpp b/unittests/terrainlib/mesh_io.cpp index 3100bbd3..0bd8c787 100644 --- a/unittests/terrainlib/mesh_io.cpp +++ b/unittests/terrainlib/mesh_io.cpp @@ -93,9 +93,9 @@ TEST_CASE("io roundtrip") { REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const std::expected result = mesh::io::load_from_path(mesh_path); + const auto result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { - FAIL(result.error().description()); + FAIL(result.error().to_string()); } std::filesystem::remove(mesh_path); const SimpleMesh roundtrip_mesh = result.value(); @@ -136,9 +136,9 @@ TEST_CASE("io roundtrip high precision") { REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const std::expected result = mesh::io::load_from_path(mesh_path); + const auto result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { - FAIL(result.error().description()); + FAIL(result.error().to_string()); } std::filesystem::remove(mesh_path); const SimpleMesh roundtrip_mesh = result.value(); @@ -179,9 +179,9 @@ TEST_CASE("io roundtrip no texture") { REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const std::expected result = mesh::io::load_from_path(mesh_path); + const auto result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { - FAIL(result.error().description()); + FAIL(result.error().to_string()); } // std::filesystem::remove(mesh_path); const SimpleMesh roundtrip_mesh = result.value(); @@ -216,9 +216,9 @@ TEST_CASE("io roundtrip no texture and uvs") { REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const std::expected result = mesh::io::load_from_path(mesh_path); + const auto result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { - FAIL(result.error().description()); + FAIL(result.error().to_string()); } std::filesystem::remove(mesh_path); const SimpleMesh roundtrip_mesh = result.value(); @@ -229,3 +229,18 @@ TEST_CASE("io roundtrip no texture and uvs") { } } } + +TEST_CASE("mesh IO reports project errors") +{ + const auto unsupported_input = mesh::io::load_from_path("mesh.unsupported"); + REQUIRE_FALSE(unsupported_input.has_value()); + CHECK(unsupported_input.error().code() == Error::Code::Unsupported); + + const auto missing_input = mesh::io::load_from_path("missing.glb"); + REQUIRE_FALSE(missing_input.has_value()); + CHECK(missing_input.error().code() == Error::Code::NotFound); + + const auto unsupported_output = mesh::io::save_to_path(mesh::Simple {}, "mesh.unsupported"); + REQUIRE_FALSE(unsupported_output.has_value()); + CHECK(unsupported_output.error().code() == Error::Code::Unsupported); +} diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index f8915de8..7c75ca74 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -70,7 +70,9 @@ TEST_CASE("versioned octree and SF payload validation is free-standing") .index = 0, .status = static_cast(store::NodeStatus::Leaf), } } }; - CHECK_FALSE(octree::storage::decode_index(invalid_index).has_value()); + const auto decoded_index = octree::storage::decode_index(invalid_index); + REQUIRE_FALSE(decoded_index.has_value()); + CHECK(decoded_index.error().code() == Error::Code::CorruptData); const mesh::sf::Payload oversized_mesh { .vertex_count = std::numeric_limits::max(), @@ -80,7 +82,9 @@ TEST_CASE("versioned octree and SF payload validation is free-standing") .uvs = {}, .texture = {}, }; - CHECK_FALSE(mesh::sf::validate(oversized_mesh).has_value()); + const auto validated_mesh = mesh::sf::validate(oversized_mesh); + REQUIRE_FALSE(validated_mesh.has_value()); + CHECK(validated_mesh.error().code() == Error::Code::InvalidInput); } TEST_CASE("storage hard-links matching payload formats") diff --git a/unittests/tilebuilder/dataset_reading.cpp b/unittests/tilebuilder/dataset_reading.cpp index d6f23416..2b505789 100644 --- a/unittests/tilebuilder/dataset_reading.cpp +++ b/unittests/tilebuilder/dataset_reading.cpp @@ -122,7 +122,7 @@ TEST_CASE("reading") const auto s = std::string("/austria/").length(); const auto l = dataset_name.length() - std::string(".tif").length() - s; const auto debug_file = fmt::format("./heights_{}_srs{}_{}.png", test_name, test_srs, dataset_name.substr(s, l)); - image::debug_out(heights, debug_file); + REQUIRE(image::debug_out(heights, debug_file).has_value()); } const auto heights = reader.read(srs_bounds, 100, 111); @@ -170,8 +170,9 @@ TEST_CASE("reading") require_projection_available(*ref_dataset, geodetic_srs); const auto ref_reader = DatasetReader(ref_dataset, geodetic_srs, 1); const auto ref_heights = ref_reader.read(ref_bounds, render_width, render_height); - if (ALP_UNITTESTS_DEBUG_IMAGES) - image::debug_out(ref_heights, fmt::format("./heights_ref.png")); + if (ALP_UNITTESTS_DEBUG_IMAGES) { + REQUIRE(image::debug_out(ref_heights, fmt::format("./heights_ref.png")).has_value()); + } for (std::string dataset_name : datasets) { const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR + std::string(dataset_name)).value(); @@ -183,12 +184,12 @@ TEST_CASE("reading") const auto l = dataset_name.length() - std::string(".tif").length() - s; if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debug_out(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))); + REQUIRE(image::debug_out(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))).has_value()); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(ref_heights.begin(), ref_heights.end(), heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); const auto path = fmt::format("./diffs_{}_{}.png", test_name, dataset_name.substr(s, l)); - image::debug_out(height_diffs, path); + REQUIRE(image::debug_out(height_diffs, path).has_value()); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(ref_heights.begin(), ref_heights.end(), heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -246,12 +247,12 @@ TEST_CASE("reading") fmt::print("low res time: {}; high res time: {}\n", double(low_res_time) / 1000.0, double(high_res_time) / 1000.0); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debug_out(low_res_heights, fmt::format("./low_res_heights.png")); - image::debug_out(high_res_heights, fmt::format("./high_res_heights.png")); + REQUIRE(image::debug_out(low_res_heights, fmt::format("./low_res_heights.png")).has_value()); + REQUIRE(image::debug_out(high_res_heights, fmt::format("./high_res_heights.png")).has_value()); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debug_out(height_diffs, "./diff_low_res_high_res.png"); + REQUIRE(image::debug_out(height_diffs, "./diff_low_res_high_res.png").has_value()); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -288,12 +289,12 @@ TEST_CASE("reading") REQUIRE(high_res_heights.height() == render_height); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debug_out(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")); - image::debug_out(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")); + REQUIRE(image::debug_out(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")).has_value()); + REQUIRE(image::debug_out(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")).has_value()); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debug_out(height_diffs, "./ov_with_warping_diff_low_res_high_res.png"); + REQUIRE(image::debug_out(height_diffs, "./ov_with_warping_diff_low_res_high_res.png").has_value()); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -330,12 +331,12 @@ TEST_CASE("reading") REQUIRE(high_res_heights.height() == render_height); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debug_out(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")); - image::debug_out(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")); + REQUIRE(image::debug_out(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")).has_value()); + REQUIRE(image::debug_out(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")).has_value()); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debug_out(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png"); + REQUIRE(image::debug_out(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png").has_value()); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { diff --git a/unittests/tilebuilder/image_writer.cpp b/unittests/tilebuilder/image_writer.cpp index 3667e5e3..8fc548f2 100644 --- a/unittests/tilebuilder/image_writer.cpp +++ b/unittests/tilebuilder/image_writer.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -38,7 +37,7 @@ TEST_CASE("tile builder writes radix rasters as PNG images") raster.pixel({ 0, 1 }) = { 0, 0, 255 }; raster.pixel({ 1, 1 }) = { 255, 255, 255 }; - image::save_image_as_png(raster, output_path.string()); + REQUIRE(image::save_image_as_png(raster, output_path.string()).has_value()); const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); REQUIRE(decoded.rows == 2); @@ -53,7 +52,12 @@ TEST_CASE("tile builder writes radix rasters as PNG images") TEST_CASE("tile builder handles image writer edge cases") { - SECTION("empty debug raster is rejected") { CHECK_THROWS_AS(image::debug_out(radix::Raster {}, "empty.png"), std::invalid_argument); } + SECTION("empty debug raster is rejected") + { + const auto result = image::debug_out(radix::Raster {}, "empty.png"); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); + } SECTION("constant debug raster is written as black") { @@ -62,7 +66,7 @@ TEST_CASE("tile builder handles image writer edge cases") radix::Raster raster({ 2, 2 }); std::ranges::fill(raster, 42.F); - image::debug_out(raster, output_path.string()); + REQUIRE(image::debug_out(raster, output_path.string()).has_value()); const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); REQUIRE(decoded.rows == 2); @@ -78,6 +82,8 @@ TEST_CASE("tile builder handles image writer edge cases") std::filesystem::remove_all(missing_directory); const auto output_path = missing_directory / "image.png"; - CHECK_THROWS_AS(image::save_image_as_png(radix::Raster({ 1, 1 }), output_path.string()), std::runtime_error); + const auto result = image::save_image_as_png(radix::Raster({ 1, 1 }), output_path.string()); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Io); } } From 7288f1ea31d3c404d249759bd41f57eb4986de07 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:23:26 +0200 Subject: [PATCH 39/43] Port glTF IO to project error convention --- src/terrainlib/CMakeLists.txt | 1 - src/terrainlib/mesh/codec/Gltf.h | 37 +--- src/terrainlib/mesh/io.cpp | 46 +--- src/terrainlib/mesh/io/error.h | 142 ------------ src/terrainlib/mesh/io/gltf.cpp | 311 ++++++++++++++++++++------- src/terrainlib/mesh/io/gltf.h | 14 +- unittests/terrainlib/mesh_io.cpp | 21 +- unittests/terrainlib/store_codec.cpp | 4 +- 8 files changed, 279 insertions(+), 297 deletions(-) delete mode 100644 src/terrainlib/mesh/io/error.h diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index dc77c8fa..11c40f01 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -99,7 +99,6 @@ add_library(terrainlib mesh/igl/manifold.h mesh/igl/orient.h mesh/io.h mesh/io.cpp - mesh/io/error.h mesh/io/gltf.h mesh/io/gltf.cpp mesh/io/options.h mesh/io/texture.h mesh/io/texture.cpp diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h index 5a14f8e7..61e5aa35 100644 --- a/src/terrainlib/mesh/codec/Gltf.h +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -32,46 +33,28 @@ class Gltf final : public store::Codec { Expected read(const std::filesystem::path& node_path) const override { const std::filesystem::path path = paths(node_path).front(); - const auto result = mesh::io::gltf::load_from_path(path); + auto result = mesh::io::gltf::load_from_path(path); if (!result) { - switch (static_cast(result.error())) { - case mesh::io::LoadMeshErrorKind::UnsupportedFormat: - return Error::fail(Error::Code::Unsupported, "load unsupported glTF format from", path); - case mesh::io::LoadMeshErrorKind::FileNotFound: - return Error::fail(Error::Code::NotFound, "open glTF file", path); - case mesh::io::LoadMeshErrorKind::InvalidFormat: - return Error::fail(Error::Code::CorruptData, "decode invalid glTF file", path); - case mesh::io::LoadMeshErrorKind::OutOfMemory: - return Error::fail(Error::Code::ResourceExhausted, "load glTF file: out of memory", path); - } - std::terminate(); + return Error::propagate(std::move(result), "read glTF mesh \"" + path.string() + "\""); } - return std::move(result.value()); + return std::move(*result); } Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { const std::filesystem::path path = paths(node_path).front(); try { - const auto result = mesh::io::gltf::save_to_path(mesh, path); + auto result = mesh::io::gltf::save_to_path(mesh, path); if (!result) { - switch (static_cast(result.error())) { - case mesh::io::SaveMeshErrorKind::UnsupportedFormat: - return Error::fail(Error::Code::Unsupported, "write unsupported glTF format to", path); - case mesh::io::SaveMeshErrorKind::OpenFile: - return Error::fail(Error::Code::Io, "open glTF file for writing", path); - case mesh::io::SaveMeshErrorKind::WriteFile: - return Error::fail(Error::Code::Io, "write glTF file", path); - case mesh::io::SaveMeshErrorKind::OutOfMemory: - return Error::fail(Error::Code::ResourceExhausted, "write glTF file: out of memory", path); - } - std::terminate(); + return Error::propagate(std::move(result), "write glTF mesh \"" + path.string() + "\""); } return {}; + } catch (const std::bad_alloc&) { + return Error::fail(Error::Code::ResourceExhausted, "write glTF mesh \"" + path.string() + "\": out of memory"); } catch (const std::exception& error) { - return Error::fail(Error::Code::Internal, "write glTF file \"" + path.string() + "\": " + error.what()); + return Error::fail(Error::Code::Internal, "write glTF mesh \"" + path.string() + "\": " + error.what()); } catch (...) { - return Error::fail(Error::Code::Internal, "write glTF file \"" + path.string() + "\": unknown exception"); + return Error::fail(Error::Code::Internal, "write glTF mesh \"" + path.string() + "\": unknown exception"); } } diff --git a/src/terrainlib/mesh/io.cpp b/src/terrainlib/mesh/io.cpp index 4f91ae50..a80cc30f 100644 --- a/src/terrainlib/mesh/io.cpp +++ b/src/terrainlib/mesh/io.cpp @@ -6,50 +6,12 @@ namespace mesh::io { -namespace { - -Error load_error(const LoadMeshError error, const std::filesystem::path& path) -{ - switch (static_cast(error)) { - case LoadMeshErrorKind::UnsupportedFormat: - return Error::make(Error::Code::Unsupported, "load unsupported glTF format from", path); - case LoadMeshErrorKind::FileNotFound: - return Error::make(Error::Code::NotFound, "open glTF file", path); - case LoadMeshErrorKind::InvalidFormat: - return Error::make(Error::Code::CorruptData, "decode invalid glTF file", path); - case LoadMeshErrorKind::OutOfMemory: - return Error::make(Error::Code::ResourceExhausted, "load glTF file: out of memory", path); - } - return Error::make(Error::Code::Internal, "unknown glTF load failure for \"" + path.string() + "\""); -} - -Error save_error(const SaveMeshError error, const std::filesystem::path& path) -{ - switch (static_cast(error)) { - case SaveMeshErrorKind::UnsupportedFormat: - return Error::make(Error::Code::Unsupported, "write unsupported glTF format to", path); - case SaveMeshErrorKind::OpenFile: - return Error::make(Error::Code::Io, "open glTF file for writing", path); - case SaveMeshErrorKind::WriteFile: - return Error::make(Error::Code::Io, "write glTF file", path); - case SaveMeshErrorKind::OutOfMemory: - return Error::make(Error::Code::ResourceExhausted, "write glTF file: out of memory", path); - } - return Error::make(Error::Code::Internal, "unknown glTF write failure for \"" + path.string() + "\""); -} - -} // namespace - Expected load_from_path( const std::filesystem::path &path, const LoadOptions& options) { const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { - auto result = gltf::load_from_path(path, options); - if (!result) { - return Error::propagate(load_error(result.error(), path), "load glTF mesh"); - } - return std::move(*result); + return gltf::load_from_path(path, options); } else if (extension == ".sfmesh") { std::filesystem::path node_path = path; node_path.replace_extension(); @@ -70,11 +32,7 @@ Expected save_to_path( const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { - auto result = gltf::save_to_path(mesh, path, options); - if (!result) { - return Error::propagate(save_error(result.error(), path), "write glTF mesh"); - } - return {}; + return gltf::save_to_path(mesh, path, options); } else if (extension == ".sfmesh") { std::filesystem::path node_path = path; node_path.replace_extension(); diff --git a/src/terrainlib/mesh/io/error.h b/src/terrainlib/mesh/io/error.h deleted file mode 100644 index 40df1267..00000000 --- a/src/terrainlib/mesh/io/error.h +++ /dev/null @@ -1,142 +0,0 @@ -#pragma once - -#include - -namespace mesh::io { - -enum class LoadMeshErrorKind { - UnsupportedFormat, - FileNotFound, - InvalidFormat, - OutOfMemory -}; - -class LoadMeshError { -public: - LoadMeshError() = default; - constexpr LoadMeshError(LoadMeshErrorKind kind) - : kind(kind) {} - - operator LoadMeshErrorKind() const { - return this->kind; - } - constexpr bool operator==(LoadMeshError other) const { - return this->kind == other.kind; - } - constexpr bool operator!=(LoadMeshError other) const { - return this->kind != other.kind; - } - constexpr bool operator==(LoadMeshErrorKind other) const { - return this->kind == other; - } - constexpr bool operator!=(LoadMeshErrorKind other) const { - return this->kind != other; - } - - const char* description() const { - switch (kind) { - case LoadMeshErrorKind::UnsupportedFormat: - return "format not supported"; - case LoadMeshErrorKind::FileNotFound: - return "file not found"; - case LoadMeshErrorKind::InvalidFormat: - return "invalid file format"; - case LoadMeshErrorKind::OutOfMemory: - return "out of memory"; - default: - return "undefined error"; - } - } - -private: - LoadMeshErrorKind kind; -}; - -enum class SaveMeshErrorKind { - UnsupportedFormat, - OpenFile, - WriteFile, - OutOfMemory -}; - -class SaveMeshError { -public: - SaveMeshError() = default; - constexpr SaveMeshError(SaveMeshErrorKind kind) - : kind(kind) {} - - operator SaveMeshErrorKind() const { - return this->kind; - } - constexpr bool operator==(SaveMeshError other) const { - return this->kind == other.kind; - } - constexpr bool operator!=(SaveMeshError other) const { - return this->kind != other.kind; - } - constexpr bool operator==(SaveMeshErrorKind other) const { - return this->kind == other; - } - constexpr bool operator!=(SaveMeshErrorKind other) const { - return this->kind != other; - } - - const char* description() const { - switch (kind) { - case SaveMeshErrorKind::UnsupportedFormat: - return "format not supported"; - case SaveMeshErrorKind::OpenFile: - return "failed to open output file"; - case SaveMeshErrorKind::WriteFile: - return "failed to write to output file"; - case SaveMeshErrorKind::OutOfMemory: - return "out of memory"; - default: - return "undefined error"; - } - } - -private: - SaveMeshErrorKind kind; -}; - -} // namespace mesh::io - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const mesh::io::SaveMeshError &error, FormatContext &ctx) const { - return fmt::format_to(ctx.out(), "{}", error.description()); - } -}; - -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const mesh::io::LoadMeshError &error, FormatContext &ctx) const { - return fmt::format_to(ctx.out(), "{}", error.description()); - } -}; - -#include -#include -inline std::ostream &operator<<(std::ostream &os, const mesh::io::SaveMeshError &error) { - fmt::print(os, "{}", error); - return os; -} - -inline std::ostream &operator<<(std::ostream &os, const mesh::io::LoadMeshError &error) { - fmt::print(os, "{}", error); - return os; -} diff --git a/src/terrainlib/mesh/io/gltf.cpp b/src/terrainlib/mesh/io/gltf.cpp index 318716a6..25ed2c84 100644 --- a/src/terrainlib/mesh/io/gltf.cpp +++ b/src/terrainlib/mesh/io/gltf.cpp @@ -1,7 +1,11 @@ +#include +#include #include #include #include -#include +#include +#include +#include #define CGLTF_IMPLEMENTATION #define CGLTF_WRITE_IMPLEMENTATION @@ -10,13 +14,11 @@ #undef CGLTF_IMPLEMENTATION #undef CGLTF_WRITE_IMPLEMENTATION +#include "io/utils.h" #include "log.h" -#include "mesh/io/utils.h" #include "mesh/io/gltf.h" #include "mesh/io/texture.h" -using namespace mesh::io::utils; - namespace mesh::io::gltf { namespace { @@ -178,7 +180,7 @@ std::string data_uri_encode(unsigned char const *bytes_to_encode, unsigned int i return "data:application/octet-stream;base64," + base64_encode(bytes_to_encode, in_len); } -std::optional load_texture_from_material(const cgltf_material &material) { +Expected> load_texture_from_material(const cgltf_material &material) { if (!material.has_pbr_metallic_roughness || material.pbr_metallic_roughness.base_color_texture.texture == nullptr) { LOG_WARN("mesh material has no texture"); return std::nullopt; @@ -193,52 +195,184 @@ std::optional load_texture_from_material(const cgltf_material &material } const std::span raw_texture{cgltf_buffer_view_data(albedo_image.buffer_view), albedo_image.buffer_view->size}; - cv::Mat texture = mesh::io::read_texture_from_encoded_bytes(raw_texture); - return texture; -} - -#define GET_OR_INVALID_FORMAT(var, opt) \ - do { \ - if (!(opt).has_value()) { \ - return std::unexpected(LoadMeshErrorKind::InvalidFormat); \ - } else { \ - var = opt.value(); \ - } \ - } while (false) - -template -static std::optional> get_single_element(const char *name, cgltf_size count, T const *items) { - if (count == 0) { - LOG_ERROR("file contains no {}", name); - return std::nullopt; - } - if (count > 1) { - LOG_WARN("file contains more than one {}", name); - return std::nullopt; + try { + cv::Mat texture = mesh::io::read_texture_from_encoded_bytes(raw_texture); + if (texture.empty()) { + return Error::fail(Error::Code::CorruptData, "could not decode embedded glTF texture"); + } + return std::optional { std::move(texture) }; + } catch (const cv::Exception& error) { + if (error.code == cv::Error::StsNoMem) { + return Error::fail(Error::Code::ResourceExhausted, "decode embedded glTF texture: out of memory"); + } + return Error::fail(Error::Code::CorruptData, "could not decode embedded glTF texture: " + error.msg); } - - const T &ref = items[0]; - - return ref; } -LoadMeshError map_cgltf_error(cgltf_result result) { +Error load_cgltf_error(const cgltf_result result, const std::filesystem::path& path) { switch (result) { case cgltf_result::cgltf_result_data_too_short: case cgltf_result::cgltf_result_unknown_format: case cgltf_result::cgltf_result_invalid_json: case cgltf_result::cgltf_result_invalid_gltf: + return Error::make(Error::Code::CorruptData, "decode invalid glTF file", path); case cgltf_result::cgltf_result_legacy_gltf: - return LoadMeshErrorKind::InvalidFormat; + return Error::make(Error::Code::Unsupported, "decode legacy glTF file", path); case cgltf_result::cgltf_result_file_not_found: + return Error::make(Error::Code::NotFound, "open glTF file", path); case cgltf_result::cgltf_result_io_error: - return LoadMeshErrorKind::FileNotFound; + return Error::make(Error::Code::Io, "read glTF file", path); case cgltf_result::cgltf_result_out_of_memory: - return LoadMeshErrorKind::OutOfMemory; - default: - UNREACHABLE(); - break; + return Error::make(Error::Code::ResourceExhausted, "load glTF file: out of memory", path); + case cgltf_result::cgltf_result_invalid_options: + case cgltf_result::cgltf_result_success: + case cgltf_result::cgltf_result_max_enum: + return Error::make(Error::Code::Internal, "unexpected glTF loader result for \"" + path.string() + "\""); + } + return Error::make(Error::Code::Internal, "unknown glTF loader result for \"" + path.string() + "\""); +} + +Expected> serialize_json(const cgltf_options& options, const cgltf_data& data) +try { + const cgltf_size expected_size = cgltf_write(&options, nullptr, 0, &data); + if (expected_size == 0) { + return Error::fail(Error::Code::Internal, "glTF serialization produced no output"); + } + + std::vector json; + if (expected_size > json.max_size()) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the supported size"); } + json.resize(expected_size); + const cgltf_size actual_size = cgltf_write(&options, reinterpret_cast(json.data()), json.size(), &data); + if (actual_size != expected_size) { + return Error::fail(Error::Code::Internal, "glTF serialization size changed between passes"); + } + json.resize(actual_size - 1); + return json; +} catch (const std::bad_alloc&) { + return Error::fail(Error::Code::ResourceExhausted, "allocate glTF serialization output"); +} catch (const std::length_error&) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the supported size"); +} + +bool write_all(std::ofstream& file, std::span bytes) +{ + constexpr std::size_t max_write_size = std::size_t { 1 } << 30; + while (!bytes.empty()) { + const std::size_t write_size = (std::min)(bytes.size(), max_write_size); + file.write(reinterpret_cast(bytes.data()), static_cast(write_size)); + if (!file.good()) { + return false; + } + bytes = bytes.subspan(write_size); + } + return true; +} + +Expected write_serialized( + const std::filesystem::path& path, + const std::span json, + const cgltf_data& data, + const bool binary_output) +try { + constexpr std::uint32_t glb_magic = 0x46546C67; + constexpr std::uint32_t glb_version = 2; + constexpr std::uint32_t json_chunk_magic = 0x4E4F534A; + constexpr std::uint32_t binary_chunk_magic = 0x004E4942; + constexpr std::size_t header_size = 12; + constexpr std::size_t chunk_header_size = 8; + constexpr std::size_t glb_size_limit = (std::numeric_limits::max)(); + + std::size_t json_padding = 0; + std::size_t binary_size = 0; + std::size_t binary_padding = 0; + std::size_t padded_json_size = 0; + std::size_t padded_binary_size = 0; + std::size_t total_size = json.size(); + if (binary_output) { + json_padding = (4 - json.size() % 4) % 4; + binary_size = data.bin == nullptr ? 0 : data.bin_size; + binary_padding = (4 - binary_size % 4) % 4; + if (json.size() > glb_size_limit - 3 || binary_size > glb_size_limit - 3) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the GLB size limit"); + } + padded_json_size = json.size() + json_padding; + padded_binary_size = binary_size + binary_padding; + if (padded_json_size > glb_size_limit - header_size - chunk_header_size) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the GLB size limit"); + } + total_size = header_size + chunk_header_size + padded_json_size; + if (binary_size != 0) { + if (padded_binary_size > glb_size_limit - chunk_header_size + || chunk_header_size + padded_binary_size > glb_size_limit - total_size) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the GLB size limit"); + } + total_size += chunk_header_size + padded_binary_size; + } + } + + const std::error_code directory_error = ::io::utils::create_parent_directories(path); + if (directory_error) { + return Error::fail(Error::Code::Io, "create parent directories for", path, directory_error); + } + + std::ofstream file(path, std::ios::binary); + if (!file.is_open()) { + return Error::fail(Error::Code::Io, "open glTF file for writing", path); + } + + if (!binary_output) { + if (!write_all(file, json)) { + return Error::fail(Error::Code::Io, "write glTF JSON to", path); + } + } else { + const auto put_u32 = [](const std::span bytes, const std::size_t offset, const std::uint32_t value) { + bytes[offset] = static_cast(value); + bytes[offset + 1] = static_cast(value >> 8); + bytes[offset + 2] = static_cast(value >> 16); + bytes[offset + 3] = static_cast(value >> 24); + }; + std::array header {}; + put_u32(header, 0, glb_magic); + put_u32(header, 4, glb_version); + put_u32(header, 8, static_cast(total_size)); + put_u32(header, 12, static_cast(padded_json_size)); + put_u32(header, 16, json_chunk_magic); + constexpr std::array json_padding_bytes { 0x20, 0x20, 0x20 }; + constexpr std::array binary_padding_bytes {}; + + if (!write_all(file, header) + || !write_all(file, json) + || !write_all(file, std::span(json_padding_bytes).first(json_padding))) { + return Error::fail(Error::Code::Io, "write glTF JSON chunk to", path); + } + if (binary_size != 0) { + std::array binary_header {}; + put_u32(binary_header, 0, static_cast(padded_binary_size)); + put_u32(binary_header, 4, binary_chunk_magic); + const std::span binary_bytes(static_cast(data.bin), binary_size); + if (!write_all(file, binary_header) + || !write_all(file, binary_bytes) + || !write_all(file, std::span(binary_padding_bytes).first(binary_padding))) { + return Error::fail(Error::Code::Io, "write glTF binary chunk to", path); + } + } + } + + file.flush(); + if (!file.good()) { + return Error::fail(Error::Code::Io, "flush glTF file", path); + } + file.close(); + if (file.fail()) { + return Error::fail(Error::Code::Io, "close glTF file", path); + } + return {}; +} catch (const std::bad_alloc&) { + return Error::fail(Error::Code::ResourceExhausted, "allocate glTF writer state"); +} catch (const std::length_error&) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the supported size"); } /// Calculates the size of the data of a vector in bytes. @@ -265,60 +399,65 @@ static std::string image_ext_to_mime(std::string_view extension) { } } -std::expected load_raw_from_path(const std::filesystem::path &path) { +Expected load_raw_from_path(const std::filesystem::path &path) { cgltf_options options = {}; cgltf_data *data = NULL; const std::string path_str = path.string(); const char *path_ptr = path_str.c_str(); cgltf_result result = cgltf_parse_file(&options, path_ptr, &data); if (result != cgltf_result::cgltf_result_success) { - return std::unexpected(result); + return Error::propagate(load_cgltf_error(result, path), "parse glTF file"); } + RawMesh raw_mesh(data, cgltf_free); - result = cgltf_load_buffers(&options, data, path_ptr); + result = cgltf_load_buffers(&options, raw_mesh.get(), path_ptr); if (result != cgltf_result::cgltf_result_success) { - cgltf_free(data); - return std::unexpected(result); + return Error::propagate(load_cgltf_error(result, path), "load glTF buffers"); } - result = cgltf_validate(data); + result = cgltf_validate(raw_mesh.get()); if (result != cgltf_result_success) { - cgltf_free(data); - return std::unexpected(result); + return Error::propagate(load_cgltf_error(result, path), "validate glTF data"); } - return RawMesh(data, cgltf_free); + return raw_mesh; } -std::expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { +Expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { LOG_TRACE("Loading mesh from gltf data"); const cgltf_data &data = *raw; - const auto mesh_opt = get_single_element("mesh", data.meshes_count, data.meshes); - if (!mesh_opt.has_value()) { - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + if (data.meshes_count == 0) { + return Error::fail(Error::Code::CorruptData, "glTF file contains no mesh"); + } + if (data.meshes_count > 1) { + return Error::fail(Error::Code::Unsupported, "loading glTF files with multiple meshes is not supported"); } - const cgltf_mesh &mesh = mesh_opt.value(); + const cgltf_mesh &mesh = data.meshes[0]; - const auto mesh_primitive_opt = get_single_element("mesh primitive", mesh.primitives_count, mesh.primitives); - if (!mesh_primitive_opt.has_value()) { - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + if (mesh.primitives_count == 0) { + return Error::fail(Error::Code::CorruptData, "glTF mesh contains no primitive"); + } + if (mesh.primitives_count > 1) { + return Error::fail(Error::Code::Unsupported, "loading glTF meshes with multiple primitives is not supported"); } - const cgltf_primitive &mesh_primitive = mesh_primitive_opt.value(); + const cgltf_primitive &mesh_primitive = mesh.primitives[0]; if (mesh_primitive.type != cgltf_primitive_type::cgltf_primitive_type_triangles) { LOG_ERROR("mesh has invalid primitive type"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::Unsupported, "loading non-triangle glTF primitives is not supported"); } // indices if (mesh_primitive.indices == nullptr) { LOG_ERROR("mesh primitive has no indices"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::Unsupported, "loading non-indexed glTF primitives is not supported"); } cgltf_accessor &index_accessor = *mesh_primitive.indices; std::vector indices; - DEBUG_ASSERT(index_accessor.count % 3 == 0); + if (index_accessor.count % 3 != 0) { + return Error::fail(Error::Code::CorruptData, "glTF mesh index count is not divisible by three"); + } indices.resize(index_accessor.count / 3); cgltf_accessor_unpack_indices( &index_accessor, @@ -330,13 +469,13 @@ std::expected load_from_raw(const RawMesh &raw, const cgltf_attribute *position_attr = find_attribute_with_type(mesh_primitive.attributes, mesh_primitive.attributes_count, cgltf_attribute_type_position); if (position_attr == nullptr) { LOG_ERROR("mesh has no position attribute"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::CorruptData, "glTF mesh contains no position attribute"); } cgltf_accessor &position_accessor = *position_attr->data; if (position_accessor.type != cgltf_type_vec3) { LOG_WARN("mesh positions are not vec3"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::CorruptData, "glTF mesh positions are not three-component vectors"); } std::vector positions; positions.resize(position_accessor.count); @@ -351,7 +490,7 @@ std::expected load_from_raw(const RawMesh &raw, const cgltf_accessor &uv_accessor = *uv_attr->data; if (uv_accessor.type != cgltf_type_vec2) { LOG_WARN("mesh uvss are not vec2"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::CorruptData, "glTF mesh UVs are not two-component vectors"); } uvs.resize(uv_accessor.count); cgltf_accessor_unpack_floats(&uv_accessor, reinterpret_cast(uvs.data()), uvs.size() * 2); @@ -374,14 +513,18 @@ std::expected load_from_raw(const RawMesh &raw, const std::optional texture; if (mesh_primitive.material != nullptr) { - texture = load_texture_from_material(*mesh_primitive.material); + auto loaded_texture = load_texture_from_material(*mesh_primitive.material); + if (!loaded_texture) { + return Error::propagate(std::move(loaded_texture), "load glTF material texture"); + } + texture = std::move(*loaded_texture); } return SimpleMesh(indices, positionsd, uvsd, texture); } /// Saves the mesh as a .gltf or .glb file at the given path. -std::expected save_to_path( +Expected save_to_path( const SimpleMesh &terrain_mesh, const std::filesystem::path &path, const SaveOptions& options) { @@ -423,7 +566,23 @@ std::expected save_to_path( const bool has_texture = terrain_mesh.has_texture(); std::vector texture_bytes; if (has_texture) { - texture_bytes = mesh::io::write_texture_to_encoded_buffer(terrain_mesh.texture.value(), options.texture_format); + try { + if (!cv::haveImageWriter(options.texture_format)) { + return Error::fail(Error::Code::Unsupported, "unsupported glTF texture format: " + options.texture_format); + } + texture_bytes = mesh::io::write_texture_to_encoded_buffer(terrain_mesh.texture.value(), options.texture_format); + } catch (const cv::Exception& error) { + if (error.code == cv::Error::StsNoMem) { + return Error::fail(Error::Code::ResourceExhausted, "encode glTF texture: out of memory"); + } + if (error.code == cv::Error::StsUnsupportedFormat || error.code == cv::Error::StsNotImplemented) { + return Error::fail(Error::Code::Unsupported, "unsupported glTF texture encoding: " + error.msg); + } + return Error::fail(Error::Code::InvalidInput, "could not encode glTF texture: " + error.msg); + } + if (!terrain_mesh.texture->empty() && texture_bytes.empty()) { + return Error::fail(Error::Code::InvalidInput, "could not encode glTF texture"); + } } // Create a single buffer that holds all binary data (indices, vertices, textures) @@ -707,24 +866,32 @@ std::expected save_to_path( } // ********************* Save the GLTF data to a file ********************* // - create_parent_directories(path); cgltf_options gltf_options = {}; if (binary_output) { gltf_options.type = cgltf_file_type_glb; } - if (cgltf_write_file(&gltf_options, path.string().c_str(), &data) != cgltf_result_success) { - throw std::runtime_error("Failed to save GLTF file"); + auto serialized = serialize_json(gltf_options, data); + if (!serialized) { + return Error::propagate(std::move(serialized), "serialize glTF output"); + } + auto written = write_serialized(path, *serialized, data, binary_output); + if (!written) { + return Error::propagate(std::move(written), "write glTF output"); } return {}; } -std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { - std::expected raw_mesh = load_raw_from_path(path); +Expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { + auto raw_mesh = load_raw_from_path(path); if (!raw_mesh) { - return std::unexpected(map_cgltf_error(raw_mesh.error())); + return Error::propagate(std::move(raw_mesh), "load raw glTF data"); + } + auto mesh = load_from_raw(*raw_mesh, options); + if (!mesh) { + return Error::propagate(std::move(mesh), "decode glTF mesh \"" + path.string() + "\""); } - return load_from_raw(*raw_mesh, options); + return std::move(*mesh); } } // namespace mesh::io::gltf diff --git a/src/terrainlib/mesh/io/gltf.h b/src/terrainlib/mesh/io/gltf.h index 20333e93..02b5bf20 100644 --- a/src/terrainlib/mesh/io/gltf.h +++ b/src/terrainlib/mesh/io/gltf.h @@ -1,33 +1,31 @@ #pragma once #include -#include #include +#include #include -#include +#include "Error.h" #include "mesh/SimpleMesh.h" -#include "mesh/io/error.h" #include "mesh/io/options.h" namespace mesh::io::gltf { using RawMesh = std::unique_ptr; -std::expected load_from_path( +Expected load_from_path( const std::filesystem::path &path, const LoadOptions &options = {}); -std::expected load_from_raw( +Expected load_from_raw( const RawMesh &mesh, const LoadOptions &options = {}); -std::expected save_to_path( +Expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -// std::expected save_to_raw(const RawMesh &mesh, const SaveOptions &options = {}); -std::expected load_raw_from_path(const std::filesystem::path &path); +Expected load_raw_from_path(const std::filesystem::path &path); } // namespace mesh::io::gltf diff --git a/unittests/terrainlib/mesh_io.cpp b/unittests/terrainlib/mesh_io.cpp index 0bd8c787..f0186921 100644 --- a/unittests/terrainlib/mesh_io.cpp +++ b/unittests/terrainlib/mesh_io.cpp @@ -19,9 +19,11 @@ #include #include +#include #include "../catch2_helpers.h" #include "../opencv_helpers.h" +#include "../temporary_directory.h" #include "mesh/io.h" #include "mesh/encode.h" @@ -232,15 +234,32 @@ TEST_CASE("io roundtrip no texture and uvs") { TEST_CASE("mesh IO reports project errors") { + test::TemporaryDirectory temporary_directory("mesh-io-errors"); + const auto unsupported_input = mesh::io::load_from_path("mesh.unsupported"); REQUIRE_FALSE(unsupported_input.has_value()); CHECK(unsupported_input.error().code() == Error::Code::Unsupported); - const auto missing_input = mesh::io::load_from_path("missing.glb"); + const auto missing_input = mesh::io::load_from_path(temporary_directory.path() / "missing.glb"); REQUIRE_FALSE(missing_input.has_value()); CHECK(missing_input.error().code() == Error::Code::NotFound); + const std::filesystem::path corrupt_path = temporary_directory.path() / "corrupt.gltf"; + std::ofstream(corrupt_path) << "not glTF"; + const auto corrupt_input = mesh::io::load_from_path(corrupt_path); + REQUIRE_FALSE(corrupt_input.has_value()); + CHECK(corrupt_input.error().code() == Error::Code::CorruptData); + const auto unsupported_output = mesh::io::save_to_path(mesh::Simple {}, "mesh.unsupported"); REQUIRE_FALSE(unsupported_output.has_value()); CHECK(unsupported_output.error().code() == Error::Code::Unsupported); + + mesh::Simple mesh; + mesh.positions = { { 0, 0, 0 }, { 1, 0, 0 }, { 0, 1, 0 } }; + mesh.triangles = { { 0, 1, 2 } }; + const std::filesystem::path directory_output = temporary_directory.path() / "directory.glb"; + REQUIRE(std::filesystem::create_directory(directory_output)); + const auto failed_output = mesh::io::save_to_path(mesh, directory_output); + REQUIRE_FALSE(failed_output.has_value()); + CHECK(failed_output.error().code() == Error::Code::Io); } diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp index 06263375..8dbd4dfe 100644 --- a/unittests/terrainlib/store_codec.cpp +++ b/unittests/terrainlib/store_codec.cpp @@ -108,7 +108,7 @@ TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") SECTION("JSON glTF") { check_mesh_codec_reentrancy(mesh::codec::Gltf(mesh::codec::GltfContainer::Json), directory.path() / "gltf"); } } -TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") +TEST_CASE("runtime glTF codec reports writer IO errors", "[store][codec]") { TemporaryDirectory directory("gltf-error"); mesh::codec::Gltf codec(mesh::codec::GltfContainer::Binary); @@ -117,5 +117,5 @@ TEST_CASE("runtime glTF codec converts writer exceptions", "[store][codec]") const auto result = codec.write(node_path, triangle_mesh(0.0)); REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code() == Error::Code::Internal); + CHECK(result.error().code() == Error::Code::Io); } From 422c41385cbdfde484a966b0b9a0fe00a3db42ad Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:13:45 +0200 Subject: [PATCH 40/43] Simplify DAG codec API --- src/dag_builder/CMakeLists.txt | 5 +- .../{codec/DagFormat.h => codec.cpp} | 126 +++++++++++++++++- src/dag_builder/codec.h | 29 ++++ src/dag_builder/codec/Dag.h | 114 ---------------- src/dag_builder/codec/from_extension.h | 30 ----- src/dag_builder/storage.h | 8 +- src/dag_convert_debug/main.cpp | 4 +- .../dag_builder/storage_compatibility.cpp | 55 ++++---- 8 files changed, 181 insertions(+), 190 deletions(-) rename src/dag_builder/{codec/DagFormat.h => codec.cpp} (74%) create mode 100644 src/dag_builder/codec.h delete mode 100644 src/dag_builder/codec/Dag.h delete mode 100644 src/dag_builder/codec/from_extension.h diff --git a/src/dag_builder/CMakeLists.txt b/src/dag_builder/CMakeLists.txt index 5479a9a2..5acf10af 100644 --- a/src/dag_builder/CMakeLists.txt +++ b/src/dag_builder/CMakeLists.txt @@ -8,6 +8,7 @@ add_library(dagbuilderlib centroids.h cluster.h clusterize.h + codec.h codec.cpp compact.h correspondence.h dag_id.h @@ -41,10 +42,6 @@ add_library(dagbuilderlib atlas/rect/create.h atlas/reprojection.h - codec/Dag.h - codec/DagFormat.h - codec/from_extension.h - merge/clusterings.h merge/clusterings.cpp merge/clusters.h merge/weld.h diff --git a/src/dag_builder/codec/DagFormat.h b/src/dag_builder/codec.cpp similarity index 74% rename from src/dag_builder/codec/DagFormat.h rename to src/dag_builder/codec.cpp index afff1bc9..8bd73390 100644 --- a/src/dag_builder/codec/DagFormat.h +++ b/src/dag_builder/codec.cpp @@ -1,23 +1,27 @@ -#pragma once - #include #include #include #include +#include #include #include #include #include +#include "codec.h" + #include "Error.h" #include "cluster.h" #include "dag_node.h" #include "io/envelope.h" +#include "io/envelope_file.h" #include "mesh/io/texture.h" #include "meshopt.h" -namespace dag::format { +namespace dag::codec { +namespace { +namespace format { namespace v1 { @@ -342,4 +346,118 @@ inline Expected<::Clustering> decode_clustering(Clustering clustering) return result; } -} // namespace dag::format +} // namespace format + +Expected validate_batch(const dag::ClusterBatch& batch, const Error::Code code) +{ + if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { + return Error::fail(code, "DAG metadata group-assignment count does not match clustering cluster count"); + } + return {}; +} + +Expected read_metadata(const std::filesystem::path& path) +{ + auto payload = io::envelope::read_from_path(path); + if (!payload) { + return Error::propagate(std::move(payload), "read DAG metadata"); + } + auto metadata = format::decode_metadata(std::move(*payload)); + if (!metadata) { + return Error::propagate(std::move(metadata), "decode DAG metadata read from \"" + path.string() + "\""); + } + return std::move(*metadata); +} + +} // namespace + +std::vector ClusterBatch::paths(const std::filesystem::path& node_path) const +{ + return { + add_extension(node_path, ".dag"), + add_extension(node_path, ".dagmeta"), + }; +} + +Expected ClusterBatch::read(const std::filesystem::path& node_path) const +{ + const auto node_paths = paths(node_path); + auto clustering_payload = io::envelope::read_from_path(node_paths[0]); + if (!clustering_payload) { + return Error::propagate(std::move(clustering_payload), "read DAG clustering"); + } + auto metadata = read_metadata(node_paths[1]); + if (!metadata) { + return Error::propagate(std::move(metadata)); + } + auto clustering = format::decode_clustering(std::move(*clustering_payload)); + if (!clustering) { + return Error::propagate(std::move(clustering), "decode DAG clustering read from \"" + node_paths[0].string() + "\""); + } + dag::ClusterBatch batch { + .metadata = std::move(*metadata), + .clustering = std::move(*clustering), + }; + if (auto valid = validate_batch(batch, Error::Code::CorruptData); !valid) { + return Error::propagate(std::move(valid), + "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); + } + return batch; +} + +Expected ClusterBatch::write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const +{ + const auto node_paths = paths(node_path); + if (auto valid = validate_batch(batch, Error::Code::InvalidInput); !valid) { + return Error::propagate(std::move(valid), + "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); + } + auto clustering = format::encode_clustering(batch.clustering); + if (!clustering) { + return Error::propagate(std::move(clustering), "encode DAG clustering for \"" + node_paths[0].string() + "\""); + } + const format::NodeMetadata metadata = format::encode_metadata(batch.metadata); + if (auto valid = format::validate(*clustering); !valid) { + return Error::propagate(std::move(valid), "validate DAG clustering for \"" + node_paths[0].string() + "\""); + } + if (auto valid = format::validate(metadata); !valid) { + return Error::propagate(std::move(valid), "validate DAG metadata for \"" + node_paths[1].string() + "\""); + } + auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); + if (!data_written) { + return Error::propagate(std::move(data_written), "write DAG clustering"); + } + auto metadata_written = io::envelope::write_to_path(metadata, node_paths[1]); + if (!metadata_written) { + return Error::propagate(std::move(metadata_written), "write DAG metadata"); + } + return {}; +} + +std::vector Metadata::paths(const std::filesystem::path& node_path) const +{ + return { add_extension(node_path, ".dagmeta") }; +} + +Expected Metadata::read(const std::filesystem::path& node_path) const +{ + return read_metadata(paths(node_path).front()); +} + +Expected>> cluster_batch_from_extension(const std::string_view extension) +{ + if (extension == ".dag") { + return std::make_unique(); + } + return Error::fail(Error::Code::Unsupported, "unsupported DAG codec selector: " + std::string(extension)); +} + +Expected>> metadata_from_extension(const std::string_view extension) +{ + if (extension == ".dag") { + return std::make_unique(); + } + return Error::fail(Error::Code::Unsupported, "unsupported DAG metadata codec selector: " + std::string(extension)); +} + +} // namespace dag::codec diff --git a/src/dag_builder/codec.h b/src/dag_builder/codec.h new file mode 100644 index 00000000..8510cac7 --- /dev/null +++ b/src/dag_builder/codec.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include + +#include "dag_node.h" +#include "store/Codec.h" + +namespace dag::codec { + +class ClusterBatch final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override; + Expected read(const std::filesystem::path& node_path) const override; + Expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override; +}; + +class Metadata final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override; + Expected read(const std::filesystem::path& node_path) const override; +}; + +Expected>> cluster_batch_from_extension(std::string_view extension); +Expected>> metadata_from_extension(std::string_view extension); + +} // namespace dag::codec diff --git a/src/dag_builder/codec/Dag.h b/src/dag_builder/codec/Dag.h deleted file mode 100644 index 3342b2f4..00000000 --- a/src/dag_builder/codec/Dag.h +++ /dev/null @@ -1,114 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "codec/DagFormat.h" -#include "io/envelope_file.h" -#include "store/Codec.h" - -namespace dag::codec { - -inline Expected validate_batch(const dag::ClusterBatch& batch, const Error::Code code) -{ - if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { - return Error::fail(code, "DAG metadata group-assignment count does not match clustering cluster count"); - } - return {}; -} - -class Dag final : public store::Codec { -public: - std::vector paths(const std::filesystem::path& node_path) const override - { - return { - add_extension(node_path, ".dag"), - add_extension(node_path, ".dagmeta"), - }; - } - - Expected read(const std::filesystem::path& node_path) const override - { - const auto node_paths = paths(node_path); - auto clustering_payload = io::envelope::read_from_path(node_paths[0]); - if (!clustering_payload) { - return Error::propagate(std::move(clustering_payload), "read DAG clustering"); - } - auto metadata_payload = io::envelope::read_from_path(node_paths[1]); - if (!metadata_payload) { - return Error::propagate(std::move(metadata_payload), "read DAG metadata"); - } - auto clustering = dag::format::decode_clustering(std::move(*clustering_payload)); - if (!clustering) { - return Error::propagate(std::move(clustering), "decode DAG clustering read from \"" + node_paths[0].string() + "\""); - } - auto metadata = dag::format::decode_metadata(std::move(*metadata_payload)); - if (!metadata) { - return Error::propagate(std::move(metadata), "decode DAG metadata read from \"" + node_paths[1].string() + "\""); - } - dag::ClusterBatch batch { - .metadata = std::move(*metadata), - .clustering = std::move(*clustering), - }; - if (auto valid = validate_batch(batch, Error::Code::CorruptData); !valid) { - return Error::propagate(std::move(valid), - "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); - } - return batch; - } - - Expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override - { - const auto node_paths = paths(node_path); - if (auto valid = validate_batch(batch, Error::Code::InvalidInput); !valid) { - return Error::propagate(std::move(valid), - "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); - } - auto clustering = dag::format::encode_clustering(batch.clustering); - if (!clustering) { - return Error::propagate(std::move(clustering), "encode DAG clustering for \"" + node_paths[0].string() + "\""); - } - const dag::format::NodeMetadata metadata = dag::format::encode_metadata(batch.metadata); - if (auto valid = dag::format::validate(*clustering); !valid) { - return Error::propagate(std::move(valid), "validate DAG clustering for \"" + node_paths[0].string() + "\""); - } - if (auto valid = dag::format::validate(metadata); !valid) { - return Error::propagate(std::move(valid), "validate DAG metadata for \"" + node_paths[1].string() + "\""); - } - auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); - if (!data_written) { - return Error::propagate(std::move(data_written), "write DAG clustering"); - } - auto metadata_written = io::envelope::write_to_path(metadata, node_paths[1]); - if (!metadata_written) { - return Error::propagate(std::move(metadata_written), "write DAG metadata"); - } - return {}; - } -}; - -class MetadataView final : public store::Codec { -public: - std::vector paths(const std::filesystem::path& node_path) const override - { - return { add_extension(node_path, ".dagmeta") }; - } - - Expected read(const std::filesystem::path& node_path) const override - { - const std::filesystem::path path = paths(node_path).front(); - auto payload = io::envelope::read_from_path(path); - if (!payload) { - return Error::propagate(std::move(payload), "read DAG metadata"); - } - auto metadata = dag::format::decode_metadata(std::move(*payload)); - if (!metadata) { - return Error::propagate(std::move(metadata), "decode DAG metadata read from \"" + path.string() + "\""); - } - return std::move(*metadata); - } -}; - -} // namespace dag::codec diff --git a/src/dag_builder/codec/from_extension.h b/src/dag_builder/codec/from_extension.h deleted file mode 100644 index 76546828..00000000 --- a/src/dag_builder/codec/from_extension.h +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "codec/Dag.h" -#include "dag_node.h" - -namespace dag::codec { - -inline Expected>> from_extension(const std::string_view extension) -{ - if (extension == ".dag") { - return std::make_unique(); - } - return Error::fail(Error::Code::Unsupported, "unsupported DAG codec selector: " + std::string(extension)); -} - -inline Expected>> metadata_from_extension(const std::string_view extension) -{ - if (extension == ".dag") { - return std::make_unique(); - } - return Error::fail(Error::Code::Unsupported, "unsupported DAG metadata codec selector: " + std::string(extension)); -} - -} // namespace dag::codec diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index ba8555d4..76b89b5f 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -3,7 +3,7 @@ #include #include -#include "codec/from_extension.h" +#include "codec.h" #include "octree/StoreTraits.h" #include "octree/storage/IndexFile.h" #include "octree/storage/open_runtime.h" @@ -26,7 +26,7 @@ inline Expected open_index( path, octree::storage::index_format(), payload_class, - dag::codec::from_extension); + dag::codec::cluster_batch_from_extension); } inline Expected open_folder( @@ -36,7 +36,7 @@ inline Expected open_folder( path, std::string(payload_class), ".dag", - dag::codec::from_extension, + dag::codec::cluster_batch_from_extension, std::move(options)); } @@ -46,7 +46,7 @@ open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) path, std::string(payload_class), ".dag", - dag::codec::from_extension, + dag::codec::cluster_batch_from_extension, std::move(options)); } diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index 71c96cd0..b0f8c8c5 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -2,7 +2,7 @@ #include #include "cli.h" -#include "codec/Dag.h" +#include "codec.h" #include "dag_node.h" #include "log.h" #include "mesh/io.h" @@ -21,7 +21,7 @@ void export_node(const cli::Args &args) { } std::filesystem::path node_path = args.input_path; node_path.replace_extension(); - const dag::codec::Dag codec; + const dag::codec::ClusterBatch codec; const auto load_result = codec.read(node_path); if (!load_result) { LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error().to_string()); diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 51c18a85..8d80f397 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -1,11 +1,10 @@ #include #include -#include #include #include "build.h" -#include "codec/Dag.h" +#include "codec.h" #include "dag_node.h" #include "mesh/SimpleMesh.h" #include "mesh/storage.h" @@ -51,7 +50,7 @@ TEST_CASE("runtime DAG envelope codec is reentrant") const dag::ClusterBatch batch = sample_batch(); TemporaryDirectory output; - dag::codec::Dag codec; + dag::codec::ClusterBatch codec; std::vector> operations; for (int index = 0; index < 8; ++index) { operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { @@ -68,10 +67,10 @@ TEST_CASE("runtime DAG envelope codec is reentrant") } } -TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") +TEST_CASE("DAG codec rejects invalid batches and inconsistent files") { TemporaryDirectory output; - dag::codec::Dag codec; + dag::codec::ClusterBatch codec; const std::filesystem::path node_path(output.path() / "node"); const auto node_paths = codec.paths(node_path); @@ -88,13 +87,26 @@ TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") CHECK_FALSE(std::filesystem::exists(node_paths[1])); } + SECTION("write invalid metadata") + { + auto batch = sample_batch(); + batch.metadata.group_assignment = { 1 }; + + const auto result = codec.write(node_path, batch); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); + CHECK_FALSE(std::filesystem::exists(node_paths[0])); + CHECK_FALSE(std::filesystem::exists(node_paths[1])); + } + SECTION("read") { REQUIRE(codec.write(node_path, sample_batch()).has_value()); - dag::NodeMetadata metadata = sample_batch().metadata; - metadata.group_assignment.clear(); - const auto encoded_metadata = dag::format::encode_metadata(metadata); - REQUIRE(io::envelope::write_to_path(encoded_metadata, node_paths[1]).has_value()); + const std::filesystem::path empty_node_path(output.path() / "empty-node"); + const auto empty_node_paths = codec.paths(empty_node_path); + REQUIRE(codec.write(empty_node_path, dag::ClusterBatch {}).has_value()); + REQUIRE(std::filesystem::copy_file(empty_node_paths[1], node_paths[1], std::filesystem::copy_options::overwrite_existing)); const auto result = codec.read(node_path); @@ -103,30 +115,9 @@ TEST_CASE("DAG codec rejects inconsistent cross-file cluster counts") } } -TEST_CASE("versioned DAG payload validation is free-standing") -{ - dag::format::NodeMetadata invalid_metadata { - .group_assignment = { 0 }, - .groups = {}, - }; - const auto validated_metadata = dag::format::validate(invalid_metadata); - REQUIRE_FALSE(validated_metadata.has_value()); - CHECK(validated_metadata.error().code() == Error::Code::InvalidInput); - - dag::format::Clustering oversized_clustering { - .vertex_count = std::numeric_limits::max(), - .encoded_positions = {}, - .clusters = {}, - .textures = {}, - }; - const auto validated_clustering = dag::format::validate(oversized_clustering); - REQUIRE_FALSE(validated_clustering.has_value()); - CHECK(validated_clustering.error().code() == Error::Code::InvalidInput); -} - TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") { - const auto batch_codec = dag::codec::from_extension(".dag"); + const auto batch_codec = dag::codec::cluster_batch_from_extension(".dag"); REQUIRE(batch_codec.has_value()); CHECK(batch_codec.value()->paths("node") == std::vector { "node.dag", "node.dagmeta" }); @@ -137,7 +128,7 @@ TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[stor REQUIRE_FALSE(write_result.has_value()); CHECK(write_result.error().code() == Error::Code::Unsupported); - const auto unknown = dag::codec::from_extension(".unknown"); + const auto unknown = dag::codec::cluster_batch_from_extension(".unknown"); REQUIRE_FALSE(unknown.has_value()); CHECK(unknown.error().code() == Error::Code::Unsupported); } From 78dee9ccd0477fbd6d1645877d721def54e37b0a Mon Sep 17 00:00:00 2001 From: Adam Celarek <5292991+adam-ce@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:26:25 +0200 Subject: [PATCH 41/43] remove obsolete refactor docs --- docs/raster-store/before-refactor/README.md | 440 ------- docs/raster-store/envelope-migration-plan.md | 348 ------ docs/raster-store/refactor-plan.md | 1132 ------------------ docs/raster-store/refactor-status.md | 237 ---- docs/raster-store/status-quo.md | 287 ----- 5 files changed, 2444 deletions(-) delete mode 100644 docs/raster-store/before-refactor/README.md delete mode 100644 docs/raster-store/envelope-migration-plan.md delete mode 100644 docs/raster-store/refactor-plan.md delete mode 100644 docs/raster-store/refactor-status.md delete mode 100644 docs/raster-store/status-quo.md diff --git a/docs/raster-store/before-refactor/README.md b/docs/raster-store/before-refactor/README.md deleted file mode 100644 index 5a5e146d..00000000 --- a/docs/raster-store/before-refactor/README.md +++ /dev/null @@ -1,440 +0,0 @@ -# SF storage and merge architecture before the 2D/3D refactor - -> **Historical report:** this file describes the pre-refactor tree and keeps -> its original names, paths, diagrams, and conclusions for review history. -> Some linked source paths were intentionally removed by the completed -> migration. For the implemented boundary and verification results, see -> [refactor-status.md](../refactor-status.md); for current public names and -> examples, see [architecture.md](../architecture.md#implementation-status). - -This report describes the existing Structura Fundamentalis (SF) builder, -sparse octree index, storage stack, and mask-based dataset merger. Its purpose -is to identify the mechanisms that can be generalised from 3D octrees to a -shared 2D/3D hierarchy without carrying mesh- and ECEF-specific behaviour into -the common storage layer. - -The central conclusion is: - -> Generalise the sparse hierarchy, storage, traversal, merge decisions, and -> unchanged-subtree reuse. Keep GDAL ingestion, raster processing, mesh -> construction, spatial transforms, and mask geometry in dimension-specific -> adapters. - -The current storage templates are already payload-generic, but they are not -hierarchy-generic: `octree::Id` is embedded throughout indexing, traversal, -caches, disk paths, and serialization. - -## Current architecture - -```mermaid -flowchart TB - subgraph Applications - SFB[sf-builder] - SFM[sf-merger] - end - - subgraph Input - GDALR[GDAL raster datasource] - GDALV[GDAL vector mask] - OLD[Base SF dataset] - NEW[New SF dataset] - end - - subgraph Domain3D["3D and mesh policy"] - SPACE[octree::Space and ECEF bounds] - MESHBUILD[Height raster to SimpleMesh] - MESHMASK[Polygon to spherical extruded MeshMask] - MESHMERGE[Clip, combine, and texture atlas] - FALLBACK[Reconstruct child by clipping ancestor mesh] - end - - subgraph GenericCandidate["Mostly generalisable mechanisms"] - INDEX[IndexMap and NodeStatus] - WALK[Depth-first and breadth-first traversal] - DRIVER[Merger and merge result actions] - STORAGE[Storage and IndexedStorage] - RAW[RawStorage] - CODEC[Codec policy] - LAYOUT[Layout strategy] - LINK[Hard-link unchanged payload] - end - - GDALR --> SFB - SFB --> SPACE - SFB --> MESHBUILD - MESHBUILD --> STORAGE - - OLD --> SFM - NEW --> SFM - GDALV --> MESHMASK - MESHMASK --> SFM - SFM --> DRIVER - DRIVER --> FALLBACK - DRIVER --> MESHMERGE - DRIVER --> STORAGE - - STORAGE --> INDEX - STORAGE --> RAW - RAW --> CODEC - RAW --> LAYOUT - RAW --> LINK - WALK --> INDEX - DRIVER --> WALK -``` - -The significant code boundaries are: - -- hierarchy identity: [`octree::Id`](../../../src/terrainlib/octree/Id.h); -- sparse topology: [`IndexMap`](../../../src/terrainlib/octree/IndexMap.h); -- traversal: [`octree::traverse`](../../../src/terrainlib/octree/traverse.h); -- logical storage: - [`Storage_`](../../../src/terrainlib/octree/storage/Storage.h); -- raw file and hard-link storage: - [`RawStorage_`](../../../src/terrainlib/octree/storage/RawStorage.h); -- disk paths: [`disk::Layout`](../../../src/terrainlib/octree/disk/Layout.h); -- merge driver: [`Merger`](../../../src/sf_merger/merge.h). - -## Sparse index model - -The index has four logical states: - -| State | Physical payload | Indexed descendants | -|---|---:|---:| -| `Missing` | no | no | -| `Virtual` | no | yes | -| `Leaf` | yes | no | -| `Inner` | yes | yes | - -`Missing` is represented by absence from `IndexMap`. The other states are -defined by -[`NodeStatus`](../../../src/terrainlib/octree/NodeStatus.h). - -```mermaid -stateDiagram-v2 - [*] --> Missing - - Missing --> Leaf: add physical root or node - Missing --> Virtual: add deeper descendant - - Virtual --> Inner: add physical payload here - Leaf --> Inner: add physical descendant - - Inner --> Leaf: remove final descendant - Leaf --> Missing: remove payload and no descendants - Virtual --> Missing: remove final descendant - Inner --> Virtual: remove payload but keep descendants -``` - -This state machine is dimension-independent. What is currently 3D-specific is -how a node finds its parent and children: `octree::Id` uses three -Morton-interleaved coordinates and eight children. - -Traversal is also reusable in concept. It: - -1. looks up the root in the sparse index; -2. visits only entries that exist; -3. enumerates children through the concrete ID type; and -4. supports depth-first or breadth-first order plus a refinement predicate. - -The only reason -[`traverse()`](../../../src/terrainlib/octree/traverse.h) -is 3D is its dependency on `octree::Id` and `octree::Id::children()`. - -## Creating an SF dataset from a GDAL datasource - -The current SF builder creates physical mesh nodes at one requested octree -level. - -```mermaid -sequenceDiagram - participant CLI as sf-builder CLI - participant DS as Dataset and GDAL - participant Space as octree::Space - participant Build as terrainbuilder - participant Reader as RawDatasetReader - participant Raster as Raster and RasterMask - participant Mesh as mesh operations - participant Store as Storage - participant Index as IndexMap and terrain.index - - CLI->>DS: Open raster datasource - DS-->>CLI: SRS, 2D bounds, and height range - - CLI->>Space: Transform bounds to ECEF - Space-->>CLI: Smallest enclosing octree node - - loop Intersecting children until target level - CLI->>Space: Node bounds and intersection tests - end - - loop Each target-level node - CLI->>Build: build_patch using node bounds - Build->>Reader: Map SRS bounds to source pixels - Reader->>DS: RasterIO band 1 as float - Reader-->>Raster: Height raster - Raster->>Raster: Build NoData validity mask - Raster->>Mesh: Generate positions and triangle grid - Mesh->>Mesh: Clip to node volume - Mesh->>Mesh: Generate UVs and transform output SRS - Mesh-->>Store: SimpleMesh with optional texture - Store->>Store: MeshCodec writes node file - end - - Store->>Index: Scan output paths - Index->>Index: Add leaves and virtual ancestors - Index->>Store: Write terrain.index -``` - -### Current components - -| Responsibility | Current component | -|---|---| -| Own and open a GDAL datasource | [`Dataset`](../../../src/terrainlib/Dataset.h) | -| One-time GDAL registration | [`initialize_gdal_once()`](../../../src/terrainlib/init.cpp) | -| ECEF octree geometry | [`octree::Space`](../../../src/terrainlib/octree/Space.h) | -| Batch enumeration and orchestration | [`build_all_patches()`](../../../src/sf_builder/terrainbuilder.cpp) | -| Direct source-window reading | [`RawDatasetReader`](../../../src/sf_builder/raw_dataset_reader.h) | -| Height-to-mesh conversion | [`build_reference_mesh_patch()`](../../../src/sf_builder/mesh_builder.cpp) | -| Optional imagery | [`TileProvider`](../../../src/sf_builder/tile_provider.h) and [`texture_assembler.h`](../../../src/sf_builder/texture_assembler.h) | -| Mesh persistence | `Storage_` | -| Sparse topology | `IndexMap` | -| Index creation and serialization | `save_or_create_index()` and `terrain.index` | - -For a new output, `build_all_patches()` opens unindexed storage. Individual -saves therefore do not build the index incrementally. The final -`save_or_create_index()` recursively scans the output directory, parses node -paths, and calls `IndexMap::add()`. - -### More suitable 2D GDAL path - -The separate -[`DatasetReader`](../../../src/tile_builder/DatasetReader.h) -is closer to what a 2D SF or raster-store builder needs: - -- it accepts requested target-SRS bounds and output dimensions; -- it creates a GDAL warped VRT; -- it reprojects into the target grid; and -- it returns `radix::Raster`. - -[`Tiler`](../../../src/tile_builder/Tiler.h) and -[`ParallelTiler`](../../../src/tile_builder/ParallelTiler.h) -already enumerate `radix::tile::Id` values over dataset bounds. - -Their grid calculations are useful, but `ParallelTileGenerator` is a -delivery-file writer rather than an SF snapshot builder. - -## Mask-based SF dataset merging - -```mermaid -flowchart TD - START[Open base and new datasets as IndexedStorage] - MASK[Read vector mask through GDAL and OGR] - PREP[Polygon repair, sphere projection, triangulation, radial extrusion] - ROOT[Start at octree root] - STATUS[Read left and right NodeStatus] - POLICY[Masked visitor evaluates node] - RECURSE[Clip mask to node bounds and recurse into 8 children] - LEFT[Keep base subtree] - RIGHT[Keep new subtree] - MERGE[Clip base outside mask and new inside mask] - ATLAS[Combine meshes and rebuild texture atlas] - WRITE[Write new physical node] - COPY[Traverse unchanged source subtree] - LINK[Hard-link each physical payload] - INDEX[Scan and save new output index] - - START --> ROOT - MASK --> PREP --> POLICY - ROOT --> STATUS --> POLICY - - POLICY -->|virtual or refinement required| RECURSE - RECURSE --> STATUS - - POLICY -->|unchanged left| LEFT --> COPY - POLICY -->|unchanged right| RIGHT --> COPY - COPY --> LINK - - POLICY -->|boundary node| MERGE --> ATLAS --> WRITE - POLICY -->|no retained data| INDEX - - LINK --> INDEX - WRITE --> INDEX -``` - -### Merge components - -- [`Merger`](../../../src/sf_merger/merge.h) is the recursive dispatcher. -- [`NodeLoader`](../../../src/sf_merger/NodeLoader.h) reports status and loads - payloads. -- When an exact payload is missing, `NodeLoader` searches physical ancestors - and reconstructs the requested child by clipping the ancestor mesh. That - reconstruction is 3D-specific. -- [`NodeData`](../../../src/sf_merger/merge/NodeData.h) lazily exposes a node - payload to a visitor. -- [`Result`](../../../src/sf_merger/merge/Result.h) gives the driver four - actions: recurse, ignore, preserve one source unchanged, or write a merged - payload. -- [`Masked`](../../../src/sf_merger/merge/visitor/Masked.h) implements - mesh-and-mask policy. -- [`NodeWriter`](../../../src/sf_merger/NodeWriter.h) writes changed nodes or - copies unchanged subtrees. - -The vector-mask pipeline in -[`mask.h`](../../../src/sf_merger/mask.h) -is almost entirely 3D and Earth-specific after OGR polygon loading: - -```text -OGR polygons - -> referenced 2D polygon mask - -> ECEF and spherical projection - -> triangulated surface - -> extrusion over an Earth-radius interval - -> closed 3D MeshMask -``` - -For 2D raster merging, only the OGR polygon loading and CRS transformation -ideas remain relevant. Triangulation, spherical projection, extrusion, 3D -clipping, and texture atlases should not enter the generic core. - -## Hard-link behaviour - -Hard links are implemented at the lowest storage layer in -[`RawStorage_::copy_from()`](../../../src/terrainlib/octree/storage/RawStorage.h): - -```mermaid -flowchart TD - COPY[copy_from node] - EXISTS{Source payload exists?} - EXT{Source and target extensions match?} - DECODE[Decode source payload] - ENCODE[Encode into target format] - REMOVE[Remove existing target] - DIRS[Create parent directories] - HARDLINK[Create hard link] - DONE[Add physical node to target index] - - COPY --> EXISTS - EXISTS -->|no| ERROR1[FileNotFound] - EXISTS -->|yes| EXT - EXT -->|no| DECODE --> ENCODE --> DONE - EXT -->|yes| REMOVE --> DIRS --> HARDLINK --> DONE -``` - -Properties: - -- Hard linking is payload-agnostic and fully generalisable. -- It happens only when source and target filename extensions match. -- Different formats cause decode and re-encode through the codec. -- There is no copy fallback if hard-link creation fails, including across - filesystems. -- Snapshot immutability is not enforced by `Storage_`; it is a caller-level - convention. -- The existing target file is removed before the link is created. -- The new output receives a fresh index; `terrain.index` itself is not linked. - -## Generalisation assessment - -| Mechanism | Assessment | Required change | -|---|---|---| -| `NodeStatus` state model | Reuse essentially unchanged | Move outside `octree` naming | -| `IndexMap` state transitions | Generalise | Template over node key and tree traits | -| Sparse traversal | Generalise | Obtain children through tree traits | -| `Storage_` | Good starting point | Also template over node key and layout | -| `RawStorage_` hard-link behaviour | Generalise | Key-neutral paths and explicit fallback policy | -| Codec concept | Reuse | No dimensional dependency | -| Cache interface | Generalise | Key type is currently `octree::Id` | -| Layout strategy | Generalise | Key-neutral path API | -| Index file | Replace or version | Record topology kind, format version, and validated key | -| Merge result algebra | Generalise | `Merged` must hold generic payload, not `SimpleMesh` | -| Recursive merge driver | Generalise | Tree traits, payload, loader/writer, and policy | -| Unchanged-subtree reuse | Reuse | Enumerate every physical state, including `Inner` | -| GDAL datasource ownership | Reuse or adapt | Typed, multi-band reads and explicit NoData | -| Mask loading through OGR | Reuse or adapt | Produce dimension-specific mask representation | -| `octree::Space` | Keep 3D | Add a sibling 2D grid or space policy | -| Mesh clipping, atlas, and UV code | Keep 3D | Do not place in generic storage | -| ECEF and spherical mask conversion | Keep 3D | 2D uses polygon classification or rasterisation | -| `MeshCodec` | Keep 3D | Add a raster chunk codec or container | - -## Existing `Inner` limitation - -The index supports `Inner`, but the SF merger effectively does not: - -- `Merger::call_merge()` dispatches only `Missing`, `Leaf`, and `Virtual` - combinations. -- `Masked::visit()` declares `Inner` unreachable. -- `NodeWriter::copy_subtree_to_output()` skips `Virtual` and asserts every - other visited node is `Leaf`. - -This matters because a physical coarse node coexisting with physical -descendants is exactly the `Inner` case. A general 2D/3D implementation should -make "has a physical payload" and "has indexed descendants" independent -properties and handle all four states throughout traversal and merging. - -## Recommended target architecture - -```mermaid -flowchart TB - subgraph Core["Dimension-neutral hierarchical store"] - TRAITS["TreeTraits<Key>
root, parent, children, validation"] - SI["SparseIndex<Key>"] - TR["Traversal<Key>"] - ST["Storage<Key, Payload, Codec>"] - DL["DiskLayout<Key>"] - ME["MergeEngine<Key, Payload, Policy>"] - SNAP[Snapshot and subtree reuse] - end - - subgraph D2["2D adapter"] - TILE[radix::tile::Id] - GRID[Web Mercator grid] - RASTER[RasterChunk] - GDAL[GDAL warped-window reader] - MASK2[2D polygon or raster mask policy] - RC[Raster codec and container] - end - - subgraph D3["3D adapter"] - OCT[octree::Id] - ECEF[ECEF octree space] - MESH[SimpleMesh] - MASK3[Extruded mesh-mask policy] - MC[MeshCodec] - end - - TILE --> TRAITS - OCT --> TRAITS - - RASTER --> ST - MESH --> ST - RC --> ST - MC --> ST - - MASK2 --> ME - MASK3 --> ME - - SI --> TR - SI --> ST - DL --> ST - TR --> ME - ST --> SNAP - ME --> SNAP -``` - -The central abstractions should therefore be: - -1. `TreeTraits`: root, parent, children, maximum depth, and key - validation. -2. `SparseIndex`: the current `IndexMap` algorithm. -3. `Traversal`: DFS, BFS, and refinement independent of child - count. -4. `DiskLayout`: key-to-path and path-to-key conversion. -5. `Storage`: logical storage and index maintenance. -6. `MergeEngine`: paired sparse-tree walking. -7. Generic merge actions such as `Recurse`, `Ignore`, `KeepLeft`, `KeepRight`, - and `Write`. -8. A snapshot copier that hard-links unchanged physical nodes without knowing - their payload type. -9. Separate 2D and 3D spatial, mask, and fallback policies. - -The GDAL builder and mask merger should be clients of this core, not part of -it. diff --git a/docs/raster-store/envelope-migration-plan.md b/docs/raster-store/envelope-migration-plan.md deleted file mode 100644 index 0767b6be..00000000 --- a/docs/raster-store/envelope-migration-plan.md +++ /dev/null @@ -1,348 +0,0 @@ -# Versioned storage-envelope migration plan - -Status: implemented in the `raster-store` worktree; awaiting commit. - -## Purpose - -This plan replaces the repository's remaining private raw ZPP Bits disk -formats with the generic versioned envelope in `terrainlib/io`. It also moves -dataset-wide format selection out of the octree index and into an independent -metadata file, splits DAG data from its cheaply readable metadata, and removes -domain transformations and validation from ZPP serialization callbacks. - -The migration intentionally breaks compatibility with the pre-envelope -`terrain.index`, `.terrain`, and `.bin` formats. Every changed format receives -a new filename ending. Once an enveloped format is introduced, later readers -must continue to read its older envelope payload versions and upgrade them to -the latest in-memory type. - -## Decisions already made - -- The generic `io::envelope` magic, class name, class version, bounded - decompression, checksum, and compression fields are used for every new - private binary format. -- Default zstd compression with its embedded checksum is used for indexes, - metadata, SF meshes, DAG data, and DAG metadata, including already encoded - meshoptimizer and JPEG byte streams. -- The new persistent names are: - - `octree.storemeta` for dataset-wide metadata; - - `octree.storeindex` for the sparse hierarchy index; - - `.sfmesh` for enveloped SF mesh nodes; - - `.dag` for enveloped DAG clustering data; and - - `.dagmeta` for enveloped DAG node metadata. -- Standard `.glb` and `.gltf` files remain unchanged. They are debugging - formats and are not wrapped in the private envelope. -- Filename endings continue to identify formats. A format change receives a - new ending; versions that remain readable by the same schema retain the - ending. -- The dataset metadata file, not the index, is authoritative for the layout - ID, payload class, and codec selector. It remains readable when the index is - missing or malformed. -- The index continues to persist the complete `store::Index` state, including - `Leaf`, `Inner`, and `Virtual`. Deterministic byte order is not required. -- A DAG logical node consists of two files. The writable batch codec writes - both `.dag` and `.dagmeta`; the read-only metadata codec reads only - `.dagmeta`. -- Hard-link behavior remains path-list based. Identical new formats use the - same endings and can be linked; changing format or compression policy uses - a different ending and therefore decodes and rewrites. -- Domain-specific ZPP serialization callbacks are removed. ZPP serializes - versioned aggregate wire types mechanically. Explicit free functions encode, - validate, and decode those wire types. -- The old `io/serialize.h` and `io/serialize.inl` API is removed after all - production and test callers migrate. -- Avoidable copies are removed with spans and moves. Compression necessarily - requires owned uncompressed and compressed buffers; zero-copy decoding is - not a goal. - -## Goals - -1. Give every private binary file envelope identity, versioning, checksum, - compression, bounded allocation, and complete-input validation. -2. Keep persistence schemas explicit and versioned without putting disk - concerns into processing types. -3. Make validation independently callable and testable. -4. Preserve cheap DAG metadata reads without reading or decompressing `.dag`. -5. Remove layout/codec guessing from directory contents. -6. Keep standard glTF behavior unchanged. -7. Preserve current application semantics, topology rules, hard-link reuse, - and error propagation apart from the deliberate format break. - -## Non-goals - -- Reading pre-envelope `terrain.index`, `.terrain`, or `.bin` files. -- Changing GLB or JSON glTF serialization. -- Defining raster-fundamentalis `.amort`, source-attribution, or raster index - payloads. -- Changing hierarchy algorithms, SF merge policy, DAG construction, mesh - processing, or snapshot publication. -- Making index serialization deterministic. -- Adding a hard-link compatibility registry or implicit transcoding between - formats with the same ending. -- Refactoring unrelated image, tile-builder, or downloader formats. - -## Format boundary - -```text -octree dataset root -├── octree.storemeta envelope: octree.StoreMetadata -├── octree.storeindex envelope: octree.StoreIndex -└── - ├── .sfmesh envelope: mesh.SfMesh - ├── .dag envelope: dag.Clustering - ├── .dagmeta envelope: dag.NodeMetadata - ├── .glb unchanged standard debug output - └── .gltf unchanged standard debug output -``` - -The files shown under one node are alternatives except that `.dag` and -`.dagmeta` form one logical DAG node. - -## Versioned payloads - -Each format defines `v1` aggregate payloads, a schema alias, a latest-type -alias, and small read/write wrappers. Future `v2` types add -`from_previous(v1::Type)` as demonstrated by the envelope tests. - -### Dataset metadata - -`octree::storage::v1::StoreMetadata` contains: - -- layout ID; -- payload class ID; and -- codec selector/format ending. - -Its validator rejects empty identifiers and unknown layouts at the adapter -boundary. Payload-specific open functions additionally require the expected -payload class and codec. - -### Sparse index - -The versioned index payload contains the complete runtime -`store::Index`. To remove the custom `octree::Id` ZPP -callback, the disk representation uses aggregate key/status entries and -explicit conversion to and from the runtime index. Conversion validates IDs, -status values, duplicates, and topology consistency. - -The conversion is allowed to retain the runtime index's semantic state and -non-deterministic iteration order. It must not silently normalize malformed -serialized status combinations. - -### SF mesh - -`mesh::sf::v1::Payload` contains fixed-width counts and encoded byte buffers -for triangles, positions, UVs, and the optional texture. The envelope version -replaces `mesh::Encoded::Header::version`. Dimension and component-type fields -that are invariant for `mesh::Simple3d` are not used for runtime dispatch. - -Free functions perform: - -```text -mesh::Simple - -> encode meshoptimizer buffers and image - -> validate v1 payload - -> envelope write - -envelope read - -> validate v1 payload - -> decode meshoptimizer buffers and image - -> mesh::Simple -``` - -### DAG clustering and metadata - -The `.dag` payload contains the encoded clustering only. Its aggregate DTOs -use fixed-width counts, primitive coordinate aggregates, encoded -meshoptimizer buffers, encoded JPEG byte vectors, IDs, texture references, -and errors. The `.dagmeta` payload independently contains group assignment, -groups, child IDs, bounds, and errors. - -The writable `Codec` returns both paths, encodes the two -payloads, and writes both envelopes. Its read path reads both envelopes and -combines their decoded latest types. `Codec` returns and -reads only `.dagmeta`, with writes unsupported. - -Validation rejects malformed encoded buffers, invalid octree IDs, invalid -cluster/group/texture references, inconsistent counts, and dimensions that -would overflow allocations before constructing processing objects. - -## Serialization and I/O cleanup - -One bounded ZPP byte API supports the envelope implementation. Public path -helpers read and write `io::envelope::Bytes`; path wrappers compose byte I/O -with `io::envelope::serialize` and `deserialize`. All error conversion occurs -at the format/codec boundary. - -The migration removes: - -- raw `io::write_to_bytes`, `read_from_bytes`, `write_to_path`, and - `read_from_path` serialization; -- the `octree::Id` custom serializer and its embedded `try_make` validation; -- `store::Index` and `NodeStatus` persistence hooks; -- `mesh::Encoded` and `ImageAndExt` persistence hooks; -- DAG GLM, ID, metadata, texture, cluster, clustering, and batch serializers; - and -- unused standalone clustering persistence helpers. - -`io::bytes` remains the filesystem byte layer. Envelope path helpers expose -its `uint8_t` buffers as `std::byte` spans without copying. - -## Opening and creation lifecycle - -Opening an existing private dataset requires both fixed root files: - -1. Read and validate `octree.storemeta`. -2. Resolve the layout and payload codec from metadata. -3. Read and validate `octree.storeindex`. -4. Construct indexed storage. - -An unreadable metadata or index file returns a typed open error. There is no -directory scan fallback. - -Creating a dataset receives an explicit layout and codec, creates an indexed -in-memory store immediately, and writes node payloads while updating that -index. Finalization writes dataset metadata and the index. The implementation -does not infer layout or codec from existing node filenames. - -## Implementation phases - -### Phase 0 — Plan and semantic baseline - -- Record this approved design. -- Retain semantic round-trip, store, merge, and builder tests rather than raw - legacy bytes. -- Confirm the clean raster-store worktree and current local commits. - -Exit criterion: format names, compatibility boundary, split DAG layout, and -metadata authority are explicit. - -### Phase 1 — Consolidate envelope I/O - -- Add envelope path read/write helpers and zero-copy byte-span bridging to the - filesystem I/O layer. -- Provide a latest-version serialization overload so ordinary writers do not - repeat the latest version number. -- Remove avoidable intermediate copies. -- Add path I/O and latest-version tests. - -Exit criterion: production codecs can use the envelope without the legacy raw -serialization API. - -### Phase 2 — Version dataset metadata and index - -- Add versioned dataset metadata and index DTOs and schemas. -- Add free validation/conversion functions. -- Write `octree.storemeta` and `octree.storeindex` through the envelope. -- Separate dataset metadata persistence from index persistence in the shared - storage plumbing. -- Remove directory discovery and index reconstruction by extension scanning. -- Start new outputs indexed in memory. - -Exit criterion: a new empty or populated store opens only through its metadata -and index, and corrupt/mismatched files return typed failures. - -### Phase 3 — Migrate SF mesh payloads - -- Add the versioned `.sfmesh` wire payload. -- Move meshoptimizer and texture work into explicit encode/decode functions. -- Add free validation for counts, buffers, texture data, and mesh semantics. -- Update the SF mesh codec resolver and CLI defaults to `.sfmesh`. -- Retain `.glb`/`.gltf` unchanged. - -Exit criterion: SF builder/merger/store tests round-trip `.sfmesh`; standard -debug codecs still behave as before; no `.terrain` production path remains. - -### Phase 4 — Split and migrate DAG payloads - -- Add versioned `.dag` clustering and `.dagmeta` metadata payloads. -- Move all DAG encoding/decoding out of ZPP callbacks. -- Add a two-file writable/readable batch codec. -- Add a separate read-only `.dagmeta` codec. -- Update DAG builder and debug converter callers. - -Exit criterion: full batches round-trip through two files, metadata reads touch -only `.dagmeta`, and malformed encoded data returns errors rather than -asserting. - -### Phase 5 — Remove legacy serialization and inference - -- Delete domain-specific serializers and `io/serialize.h/.inl`. -- Delete unused standalone clustering persistence. -- Remove `.terrain`, `.bin`, and `terrain.index` private-format selectors and - tests. -- Remove unindexed directory discovery and suffix-probe index scans. -- Update documentation, golden scripts, test expectations, and error text. - -Exit criterion: repository search finds no production raw ZPP persistence or -legacy private extension and no format inference from directory contents. - -### Phase 6 — Verification - -- Run focused envelope, index, codec, storage, SF, and DAG tests after their - phases. -- Run the complete terrainlib, DAG builder, SF builder finalization, and SF - merger suites. -- Build `sf-builder`, `sf-merger`, `sf-index-browser`, `dag-builder`, and - `dag-convert-debug` in the repository build configuration. -- Run the golden end-to-end workflow with new expected endings where its data - prerequisites are available. -- Inspect `git diff --check` and the final worktree. - -## Required tests - -- Every production schema round-trips v1 through default zstd. -- Each schema rejects the wrong envelope class and unsupported versions. -- Corrupt compressed data and malformed aggregate payloads return typed - errors. -- Dataset metadata remains readable when `octree.storeindex` is corrupt. -- Payload class, codec, and layout mismatches fail before node processing. -- Invalid octree IDs, node statuses, duplicates, and inconsistent topology are - rejected by free validation/conversion functions. -- SF mesh encode/decode preserves current semantic mesh equality and rejects - malformed meshoptimizer/image data. -- DAG full-batch reads require `.dag` and `.dagmeta`. -- DAG metadata reads succeed with only `.dagmeta` present and do not access - `.dag`. -- Same-format node copies hard-link both DAG files; differing formats - decode/re-encode. -- GLB/GLTF tests remain unchanged. -- New dataset construction never scans extensions to decide its format. - -## Risks and controls - -| Risk | Control | -|---|---| -| A future reader cannot load v1 | Versioned namespaces, schema conversion trail, and retained v1 tests | -| Dataset metadata and index disagree | Metadata is authoritative; index contains topology only | -| Broken index hides format information | Independent `octree.storemeta` is read and tested separately | -| DAG metadata reads regress to full data reads | Separate `.dagmeta` file and read-only codec tests | -| Custom ZPP callbacks continue to hide failures | Aggregate DTOs plus explicit expected-returning validation/decode | -| Meshoptimizer corruption aborts through assertions | Checked decode wrappers return typed errors | -| New compression adds excessive copies | Shared byte type, spans, moves, and copy-focused review | -| New format is accidentally linked to an old one | Every changed format uses a new ending | -| Standard debug formats are accidentally wrapped | Dedicated unchanged GLB/GLTF codecs and existing tests | -| Incomplete output becomes authoritative | Keep explicit finalization boundaries and write root metadata/index last | - -## Expected migration map - -| Current | Target | -|---|---| -| `terrain.index` raw ZPP DTO | `octree.storeindex` envelope schema | -| layout and preferred extension inside index | `octree.storemeta` envelope schema | -| `.terrain` raw encoded mesh | `.sfmesh` envelope schema | -| `.bin` DAG batch | `.dag` clustering plus `.dagmeta` metadata envelopes | -| `.bin` metadata-prefix view | read-only `.dagmeta` codec | -| extension/layout directory discovery | explicit metadata-driven open | -| unindexed save then recursive scan | indexed in-memory creation and final write | -| domain `serialize(Archive&, T&)` callbacks | aggregate DTO plus free encode/validate/decode | -| `io/serialize.h/.inl` | envelope path helpers | -| `.glb` / `.gltf` | unchanged | - -## Verification result - -The implementation builds the SF builder/merger, DAG builder/debug converter, -and SF index browser targets. The terrainlib, DAG builder, SF finalization, SF -merger, and SF builder/merger integration tests pass with the new formats. The -golden end-to-end script has been migrated to the new filenames and passes -shell syntax validation; its external-data workflow was not run as part of -this implementation. diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md deleted file mode 100644 index 82028b7a..00000000 --- a/docs/raster-store/refactor-plan.md +++ /dev/null @@ -1,1132 +0,0 @@ -# 2D/3D hierarchical store refactor plan - -Status: proposal for review. This document is an implementation plan, not a -record of completed work. - -## Purpose - -This plan describes how to extract the existing octree-specific index, -traversal, storage, and codec code into a shared 2D/3D store. The refactor -must preserve existing 3D datasets and prove, using test-only mappings and -codecs where necessary, that the shared mechanisms work with a 2D key. -Persistent raster-fundamentalis formats, adapters, and tools are later work. - -## Decisions already made - -- The shared implementation will live in `src/terrainlib/store` and use the - `store` namespace. -- The minimal 2D traits adapter used to exercise the shared hierarchy will - live in `src/terrainlib/raster_store` and use the `raster_store` namespace. -- Common Structura Fundamentalis validation and errors will live in - `src/terrainlib/sf` and use the `sf` namespace. -- Existing 3D Structura Fundamentalis datasets must remain readable and - writable without changing their on-disk contract. -- DAG datasets written by the current code when Phase 0 begins must remain - readable and writable throughout the refactor without changing their index - or payload serialization. Older DAG payload schemas are not supported. -- `Inner` is a valid shared topology state and is supported by DAG, - raster-fundamentalis, and tile-base datasets. It is not valid in Structura - Fundamentalis datasets. -- SF producer and processing boundaries validate indexed SF data and return a - typed `sf::InvalidTopology` error containing an offending key when `Inner` - is present. The diagnostic `sf_index_browser` is exempt. This is an SF data - invariant, not a generic store or octree-format rule. -- 3D compatibility includes both existing path layouts: - `flat` and `level_and_coordinate_directories`. -- A persistent 2D raster tile format is not defined by this refactor. - [architecture.md](architecture.md) and - [storage-format.md](storage-format.md) describe intended direction and - provisional requirements that will be finalized in later RF work. They must - not be retrofitted onto existing 3D datasets. -- A path layout strategy should contain a stable identifier and two - operations: key to extensionless `NodePath`, and `NodePath` to key. It - should not require an inheritance hierarchy, RTTI, global - self-registration, or heap allocation. -- A configured codec owns all filename endings and maps one `NodePath` to one - or more physical files. `Codec::paths()` must not need a payload. -- Codecs are stateful runtime objects behind a small interface. They may - support reading, writing, or both. Reading and writing return - `std::expected`; unsupported operations and other operational failures are - reported as error values. Reading and writing must be reentrant (callable - concurrently from different threads). -- The runtime glTF codec catches exceptions from the existing glTF mesh writer - and converts them to `CodecError`; changing the mesh I/O API is out of scope. -- `store::RawStorage` exclusively owns the configured - `std::unique_ptr>`. `store::Storage` owns the raw - storage and therefore owns the codec transitively. Storage consumers do not - receive a codec template parameter and application call sites do not manage - codec objects. -- This refactor does not add synchronization to storage, indexes, or caches - and does not change their concurrency guarantees. Any concurrency bug fix is - separate work. -- Application callers retain their existing storage lifecycle, - synchronization, and failure-handling behaviour unless explicitly changed - by this plan. -- Preserve `dag::ThreadSafeStorage` as the caller-side synchronization around - DAG output storage. DAG storage remains cacheless while shared-lock reads are - used. -- Legacy index metadata selects a codec through an explicit, caller-supplied - resolver supplied by the payload-domain opening function. The octree format - adapter does not contain a global codec registry or depend on mesh or DAG - payload types. Application-level storage consumers do not call the resolver - or handle the resulting codec object. -- Dimension-specific index persistence uses a small runtime - `store::IndexFormat` value containing ordinary function pointers. It - is not an inheritance hierarchy and does not use global registration. -- `copy_from()` hard-links every file when the input and output codecs return - the same path list for a common dummy `NodePath`. Otherwise it decodes with - the input codec and encodes with the output codec. -- Public store operations reject invalid hierarchy keys through - `std::expected`; invalid keys are not represented by assertions or generic - booleans. -- Preserve `StorageSettings::allow_overwrite`. It defaults to `false`; - rejected overwrites return `AlreadyExists` through `std::expected`, and - enabling it replaces existing payloads. -- The existing cache implementations are currently non-functional. Port their - public API where it remains useful, keep application call sites building, and - provide compile coverage only. Cache behaviour is not a compatibility - requirement of this refactor. -- The `.png` written beside changed meshes by `sf_merger::NodeWriter` is an - unmanaged debug artifact. It is not part of a logical node, is not returned - by `Codec::paths()`, and is not indexed or copied by storage. Its existing - application-local behaviour is preserved. -- 3D geometry, ECEF bounds, mesh codecs, mesh reconstruction, mask geometry, - and raster-specific processing remain outside the shared store. - -## Goals - -1. Use one sparse hierarchy implementation for `octree::Id` and - `radix::tile::Id`. -2. Make traversal, storage, cache, and the runtime codec boundary - dimension-neutral while keeping production format adapters 3D-only in this - refactor. -3. Replace the current layout-strategy class hierarchy with small path-mapping - values backed by function pairs. -4. Allow one logical node payload to consist of multiple files without making - layouts aware of those files. -5. Preserve all valid existing 3D index files and payload paths. -6. Prove the shared topology and traversal with `radix::tile::Id` without - defining a persistent 2D adapter or format. -7. Land the refactor in small, testable steps. Every phase should build and - pass tests before the next phase begins. - -## Non-goals - -- Changing `octree::Id`, `octree::Space`, `IdRect`, `OddLevelShifted`, or - other 3D spatial calculations. -- Defining or implementing GDAL ingestion, raster resampling, filtering, - source selection, or mask rasterisation. -- Defining or implementing a raster-fundamentalis index, payload format, - layout ID, codec, publication lifecycle, or persistent storage adapter. -- Implementing an `rf_builder`, `rf_merger`, tile-base generator, tile server, - snapshot-reuse operation, or other RF tool in this refactor. -- Defining the raster-fundamentalis merge policy or the final paired-hierarchy - walker/action algebra. That work is deferred until `rf_merger` requirements - are defined. -- Defining a shared subtree-copy abstraction, `Inner` subtree-copy behaviour, - or forced re-encoding policy for RF. Those decisions are deferred until - `rf_merger`. -- Adding merge semantics for `Inner` nodes in SF. Such nodes are invalid SF - input and must be rejected before merge dispatch. -- Changing the existing 3D hard-link policy by adding a silent file-copy - fallback. -- Refactoring unrelated octree, DAG, mesh, or tile-builder code. -- Changing the serialized schema of the current DAG `.bin` payloads. - -## Compatibility contract - -Before moving code, tests must lock down the following 3D behaviour: - -| Item | Required compatibility | -|---|---| -| Index filename | `terrain.index` | -| Index field order | layout ID, preferred extension, index map | -| Node-key encoding | existing `octree::Id` level/index serialization | -| Node-status encoding | `Leaf = 0`, `Inner = 1`, `Virtual = 2` | -| Valid SF statuses | `Leaf` and `Virtual`; reject `Inner` with `sf::InvalidTopology` | -| Valid DAG statuses | `Leaf`, `Inner`, and `Virtual` | -| Flat layout ID | `flat` | -| Flat path | `-` | -| Coordinate layout ID | `level_and_coordinate_directories` | -| Coordinate path | `///` | -| Default layout | existing level/coordinate layout | -| Layout detection | both existing layouts remain detectable | -| Mesh codec selection | legacy preferred extension selects terrain or configured glTF codec | -| DAG codec selection | legacy `.bin` preferred extension selects the ZPP Bits codec | -| DAG payload encoding | current `dag::ClusterBatch` ZPP Bits serialization: metadata, then clustering | -| Equal codec path lists | hard-link every file, or report an explicit error | -| Different codec path lists | decode with input codec and encode with output codec | -| Overwrite setting | `StorageSettings::allow_overwrite`, default `false`; enabled writes replace existing payloads | - -Compatibility means that each refactor phase can open the Phase 0 fixtures -written by the current code. DAG formats older than the Phase 0 baseline and -pre-refactor readers opening post-refactor output are not tested. Exact -byte-for-byte rewriting of an unordered index map is not required, but the -serialized schema and values must remain compatible during the migration. - -The 3D index disk type should remain a versioned 3D adapter. The shared store -must not add a topology field, new header, checksum, or compression layer to -`terrain.index`. - -## Proposed source boundary - -```text -src/terrainlib/ -├── store/ -│ ├── NodeStatus.h -│ ├── NodeStatusOrMissing.h -│ ├── Traits.h -│ ├── InvalidKey.h -│ ├── Index.h -│ ├── traverse.h -│ ├── NodePath.h -│ ├── PathMapping.h -│ ├── Layout.h -│ ├── IndexFormat.h -│ ├── Codec.h -│ ├── CodecError.h -│ ├── OpenError.h -│ ├── CopyError.h -│ ├── StorageSettings.h -│ ├── RawStorage.h -│ ├── Storage.h -│ ├── IndexedStorage.h -│ ├── cache/ -│ │ ├── Interface.h -│ │ ├── Dummy.h -│ │ └── Lru.h -│ └── codec/ -│ └── ZppBits.h -├── mesh/ -│ └── codec/ -│ ├── Terrain.h -│ └── Gltf.h -├── sf/ -│ ├── InvalidTopology.h -│ └── validate_index.h -├── octree/ -│ ├── Id.h -│ ├── StoreTraits.h -│ ├── store_layout/ -│ │ ├── Flat.h -│ │ ├── LevelAndCoordinateDirectories.h -│ │ └── Mappings.h -│ └── storage/ -│ ├── IndexFile.h -│ └── open.h -└── raster_store/ - └── StoreTraits.h -``` - -The exact file grouping may be collapsed if a file would only contain a few -lines. The important boundaries are: - -- `store` contains dimension- and payload-neutral mechanisms; -- `store::codec::ZppBits` is the reusable concrete codec for payload types - that provide ZPP Bits serialization; -- `mesh::codec` contains the separately configured terrain and glTF codecs; -- `sf` contains SF-specific topology validation and errors shared by - `sf_builder`, `sf_merger`, and `dag_builder`; -- `octree` contains the 3D format and key adapters; -- `raster_store` contains only the minimal 2D hierarchy traits adapter in this - refactor; and -- subdirectory names match their namespaces where a subnamespace is used. - -Shared topology and octree-format adapters accept `Inner`. The SF restriction -is enforced by `sf::validate_index()` at SF producer/consumer boundaries and -reported as `sf::InvalidTopology`. It must not be embedded in `store::Index`, -traversal, or the generic 3D disk adapter. `sf_index_browser` is a diagnostic -tool and intentionally does not apply SF validation, so it can display invalid -trees including `Inner`. - -`sf::validate_index()` returns -`std::expected`. SF application-level error types -must retain this error and `CopyError` when propagating failures; neither is -reduced to a log message, assertion, or generic boolean. - -For this refactor, `sf::validate_index()` checks only for `Inner`. It does not -validate other structural or disk/filesystem invariants. Possible future -extensions are recorded in [todo.md](todo.md). - -Temporary forwarding headers and aliases under `octree` are allowed during -migration. They must not contain a second implementation. - -DAG serialization remains owned by `dag_builder`. Consolidate the serializers -for `dag::Id`, `dag::Group`, `dag::NodeMetadata`, `dag::ClusterBatch`, -`radix::geometry::Aabb3d`, GLM vectors, `Clustering`, `Cluster`, and -`TextureSet` in `src/dag_builder/serialization.h`. The DAG storage adapter -includes that header explicitly so template instantiation does not depend on -caller include order. Preserve the current tuples exactly: - -- `ClusterBatch`: metadata, clustering; -- `NodeMetadata`: group assignment, groups; and -- `Group`: children, error, bounds. - -## Shared interfaces - -The names below are the intended shape, not signatures that must be copied -verbatim without testing. - -### Hierarchy traits - -The store should be parameterized by one traits type rather than assuming that -all key classes expose identical member functions: - -```cpp -template -concept HierarchyTraits = requires(typename Traits::Key key) { - typename Traits::Key; - typename Traits::Hasher; - { Traits::root() } -> std::same_as; - { Traits::parent(key) }; - { Traits::children(key) }; - { Traits::is_valid(key) } -> std::same_as; -}; -``` - -The concrete names should be: - -```cpp -store::Index -store::Index -``` - -`octree::StoreTraits` adapts the existing optional parent/children API without -changing `octree::Id`. - -`raster_store::StoreTraits` adapts `radix::tile::Id` and must: - -- treat zoom zero as the only root; -- never call `radix::tile::Id::parent()` at zoom zero, where it underflows; -- reject coordinates outside `[0, 2^zoom)`; -- accept zoom levels 0 through - `std::numeric_limits::digits`, inclusive; -- treat that maximum zoom as terminal because a child cannot be represented - by the `uint32_t` x/y coordinates; -- validate the maximum zoom without evaluating an overflowing - `uint32_t{1} << 32`; -- use `radix::tile::Id::Hasher`; and -- return children in the deterministic order produced by - `radix::tile::Id::children()`. - -This traits adapter defines only hierarchy operations. It does not define -persistent coordinates, a path layout, or an RF disk format. - -The shared code must obtain roots, parents, children, validation, and hashing -through the traits. It must not use dimension checks or specialize behaviour -on key types internally. - -Operations accepting a key validate it through `Traits::is_valid()`. Index -lookup and mutation, traversal with an explicit root, and storage operations -return an `std::expected` retaining an `InvalidKey` when validation fails. -Keys produced internally by `Traits::root()`, `Traits::parent()`, and -`Traits::children()` are trusted only after trait-specific tests establish that -they preserve validity. The child order affects traversal order and is locked -down by the 2D and 3D trait tests; it is not serialized as separate metadata. - -The API result shapes are: - -- index lookup returns `expected, InvalidKey>`; -- index mutation and predicates return `expected>`; -- traversal returns `expected>`; and -- storage `load`, `save`, `has`, `remove`, path lookup, and `copy_from` return - operation-specific `expected` types which retain `InvalidKey` and any - applicable codec, filesystem, missing-source, or overwrite error. - -During Phase 1, a forwarding `octree::IndexMap` compatibility wrapper preserves -the old optional/bool API for existing 3D callers with already-valid -`octree::Id` values. It delegates to `store::Index`, is not -a second implementation, and is removed after callers migrate. - -### Sparse index and traversal - -Move the existing four-state model to: - -```cpp -store::NodeStatus -store::NodeStatusOrMissing -store::Index -store::traverse(index, visitor, refine, root, order) -``` - -The index algorithm remains the current one: - -- adding a physical descendant creates virtual ancestors; -- adding a payload to a virtual node makes it `Inner`; -- removing a payload from an `Inner` node makes it `Virtual`; -- removing the final descendant collapses virtual ancestors; and -- a physical parent becomes `Leaf` after its last descendant is removed. - -Traversal must follow only indexed nodes and use `Traits::children`. Child -order is the order supplied by the traits and is therefore deterministic per -hierarchy, not universally fixed by `store`. - -### Node paths and path mappings - -`store::NodePath` is an extensionless logical location for one hierarchy -node. For example: - -```text -octree flat 12-123456 -octree coordinates 12/34/56/78 -``` - -It does not necessarily name a physical file. Replace -`octree::disk::layout::Strategy` and -`octree::disk::layout::StrategyRegister` with a value similar to: - -```cpp -template -struct store::path_layout::Mapping { - std::string_view id; - NodePath (*key_to_node_path)(const Key&); - std::optional (*node_path_to_key)(const NodePath&); -}; -``` - -`store::path_layout::Resolver` owns the base directory and one `store::path_layout::Mapping`. It -does not own a preferred extension. A configured codec expands the -extensionless `NodePath` into the physical file or files. - -The stable ID is format metadata, not a third strategy operation. The -dimension adapters provide ordinary lookup functions: - -```cpp -octree::store_layout::flat() -octree::store_layout::level_and_coordinate_directories() -octree::store_layout::from_id(id) -octree::store_layout::all() -``` - -This retains runtime selection from an index file while removing virtual -dispatch, RTTI type-to-ID lookup, static registration, and ownership through -`unique_ptr`. - -Path parsers validate the complete logical `NodePath`, not a file ending. -Codec or format-adapter code removes and validates physical file endings -before asking the layout to recover a key. Invalid disk input returns an error -or `nullopt`; it must not trigger an assertion. - -### Codec interface - -A codec is a configured runtime object for one logical payload type. It owns -all physical filename endings and may map one `NodePath` to several files: - -```cpp -template -class store::Codec { -public: - virtual ~Codec() = default; - - virtual std::vector - paths(const NodePath& node_path) const = 0; - - virtual std::expected - read(const NodePath&) const { - return std::unexpected( - CodecError::unsupported_operation("read")); - } - - virtual std::expected write( - const NodePath&, - const NodeData&) const { - return std::unexpected( - CodecError::unsupported_operation("write")); - } -}; -``` - -Concrete codecs contain their configuration and are constructed before -storage use. `CodecError` is a payload-neutral operational error that records -the failed operation, an error category, and a diagnostic message. Concrete -codecs convert their domain errors to it. Unsupported read or write operations -may use the base implementation and return its `UnsupportedOperation` error. -Codec writes preserve the current directory-creation behaviour: they create -the parent directories required by their output paths before writing. The -storage hard-link path continues to create its target parent directories before -linking. - -`Codec::paths()` has the following contract: - -- it needs no NodeData payload and performs no filesystem access; -- it returns every physical file belonging to the logical node; -- results depend only on codec configuration and the supplied `NodePath`; -- result order is stable and pairs corresponding input/output files; -- two codecs returning the same path list for the same `NodePath` must produce - mutually compatible files; and -- different artifact counts or filename endings produce different lists. - -Examples: - -```text -Terrain codec - 12/34/56/78 - -> 12/34/56/78.terrain - -glTF codec configured for binary output - 12/34/56/78 - -> 12/34/56/78.glb - -glTF codec configured for JSON output - 12/34/56/78 - -> 12/34/56/78.gltf - -Multi-file test codec - 12/34/56/78 - -> 12/34/56/78.data - -> 12/34/56/78.metadata -``` - -The mesh side has separate terrain and glTF codecs because they use different -format implementations. Binary `.glb` and JSON `.gltf` remain configurations -of one glTF codec because both use the same `cgltf` implementation. - -The shared module also provides: - -```cpp -template -struct store::codec::ZppBits : store::Codec { .. }; -``` - -It uses the existing `io::read_from_path()` and `io::write_to_path()` -functions, maps one node to `.bin`, and converts `io::Error` to -`CodecError`. It contains no DAG-specific serialization logic. DAG payload -serialization remains in `dag_builder/serialization.h`, and its field order -and meshoptimizer/JPEG encoding remain unchanged. - -### Storage and format adapters - -Generalize storage over traits and NodeData. It owns a configured codec through -the runtime interface: - -```cpp -store::RawStorage -store::Storage -store::IndexedStorage -store::cache::Interface -``` - -`RawStorage` owns the -`std::unique_ptr>`. `Storage` owns `RawStorage`, and -`IndexedStorage` owns or derives from `Storage`; no other layer shares codec -ownership. Moving storage transfers ownership. Caches, layouts, index formats, -resolvers, and application consumers never own the codec. - -A move disarms the source so its destructor cannot save moved-out index state. -Move assignment must finalize a displaced dirty destination through the -existing destructor-save policy rather than silently discard it. Tests cover -the DAG builder's move into and release from `dag::ThreadSafeStorage`. - -Domain-specific mesh codecs remain under `mesh::codec`. The reusable ZPP Bits -codec remains under `store::codec`. RF codecs are deferred. - -Index serialization is not a responsibility of `store::Index`. Opening and -saving a dataset receives the following small runtime values: - -```cpp -template -struct store::IndexMetadata { - store::Index index; - std::string layout_id; - std::string codec_selector; -}; - -template -struct store::IndexFormat { - std::string_view index_filename; - - std::expected, IndexFormatError> - (*read)(const std::filesystem::path& index_path); - - std::expected - (*write)( - const std::filesystem::path& index_path, - const IndexMetadata& metadata); - - std::optional> - (*mapping_from_id)(std::string_view id); - - path_layout::Mapping - (*default_mapping)(); -}; -``` - -The value provides: - -- the index filename; -- index read/write conversion; -- mapping lookup by stable ID; -- the default mapping. - -Legacy directory discovery is an octree opening helper which composes the -octree index format, the supplied payload-domain codec resolver, and the known -octree mappings. It is not a generic `IndexFormat` operation. Neither the -format value nor discovery may reintroduce a layout class hierarchy or global -registration. - -For 3D, the adapter reads and writes the current `octree` index DTO unchanged. -Its `codec_selector` is exactly the legacy `preferred_extension`, including -the leading dot. Storage retains the selected `IndexFormat`, index path, -layout ID, and codec selector as its index-persistence state, so explicit and -destructor-triggered index saves can reproduce the same metadata. - -When opening indexed storage or discovering a legacy unindexed directory, it -passes the legacy `preferred_extension` to a caller-supplied codec resolver. -The resolver is an ordinary callable and returns -`std::expected>, CodecError>`. It is not -a global registry. - -The payload domains provide ordinary resolver functions: - -```text -mesh::codec::from_extension - .terrain -> terrain codec - .glb -> glTF codec with binary container - .gltf -> glTF codec with JSON container - -dag::codec::from_extension - .bin -> store::codec::ZppBits - -dag::codec::metadata_from_extension - .bin -> read-only dag::codec::MetadataView -``` - -An unknown extension returns an explicit `UnsupportedCodec` error. Opening a -new empty store receives an explicit legacy codec selector plus an already -constructed codec at the payload-domain opening boundary; the generic storage -does not infer persistent metadata from `Codec::paths()`. Mesh and DAG -convenience functions select or resolve the codec and pass ownership into raw -storage, so application storage consumers do not handle codec objects. -Convenience functions in `src/dag_builder/storage.h` supply the DAG resolvers. -Preserve `DagMetaStorage` and `IndexedDagMetaStorage` as independently readable -views of the metadata prefix in each `ClusterBatch` `.bin` file. Their codec -returns `UnsupportedOperation` from writes instead of terminating or replacing -a full batch with metadata alone. `dag::codec::MetadataView` implements this -domain-specific read-only prefix view over the generic ZPP Bits codec. - -Opening functions return their requested storage type through -`std::expected<..., OpenError>`. `OpenError` is a typed sum which retains the -failing path and the underlying error where applicable: - -- index I/O or malformed index metadata (`IndexFormatError`); -- filesystem failure; -- unknown layout ID; -- unsupported codec selector or codec construction failure (`CodecError`); and -- invalid hierarchy key (`InvalidKey`). - -Loading and saving likewise return storage-level expected errors which retain -an invalid key, an underlying `CodecError`, and `AlreadyExists` for a rejected -save. `CopyError` retains invalid-key, missing-source, overwrite, filesystem, -and codec failures. The SF call chains changed in Phase 4 retain these errors -to their application boundary. - -Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes -candidate endings by asking the supplied resolver, removes an accepted ending -to obtain a `NodePath`, and then invokes the selected layout parser. The -generic layout does not recover keys directly from codec-owned file paths. -Only a missing index triggers this discovery path. An unreadable or malformed -index, unknown layout, or unsupported codec selector returns `OpenError` and -must not silently fall back to directory discovery. - -### Copying one node and SF subtree reuse - -There are two separate responsibilities in the current implementation: - -1. `sf_merger::NodeWriter` traverses a source subtree and - `sf_merger::cut_leaf_node()` identifies an unchanged leaf. -2. `octree::Storage::copy_from()` delegates to - `octree::RawStorage::copy_from()`, where - `std::filesystem::create_hard_link()` performs the actual hard link and the - target index is updated on success. - -The filesystem hard-link implementation is therefore already in terrainlib. -Move the one-node storage operation into the shared store, but keep subtree -selection and traversal in `sf_merger`. There is no current DAG caller, and RF -subtree-copy requirements will be defined with `rf_merger`. - -The migrated call chains are: - -```text -sf_merger decides to keep a source subtree unchanged - -> NodeWriter traverses the source index - -> store::Storage::copy_from() - -> hard-link every codec path, or decode/encode - -sf_merger determines that a cut leaf is unchanged - -> cut_leaf_node() - -> store::Storage::copy_from() - -> hard-link every codec path, or decode/encode -``` - -`Storage::copy_from()` remains the operation for copying one logical node. -For one key, `copy_from()`: - -1. calls the input and output `Codec::paths()` with the same fixed dummy - `NodePath`; -2. when the lists are equal, calls both codecs again with their actual source - and target `NodePath` values and hard-links every source path to the - corresponding target path; -3. when the dummy lists differ, reads the payload with the input codec and - writes it with the output codec; and -4. updates the target index only after all links or the write complete. - -The dummy path must be fixed and collision-free, for example -`__codec_probe__/node`. Path lists are compared exactly, including count, -order, and filename endings. - -If overwrite is enabled for an indexed target, remove the target index entry -immediately before the first target file is modified. If linking several files -then fails partway through, return the error without a transactional rollback -guarantee; target links or old files may remain, but the logical node stays -unindexed. The copy operation stops immediately and propagates the failure -until the application aborts the overall operation. There is no journal, -rollback, cleanup guarantee, or silent copy fallback. An unsupported read or -write needed for re-encoding is returned through `CopyError`, retaining the -underlying `CodecError`. - -Hard-link rules: - -- never modify an existing linked payload in place; -- a matching codec path list hard-links every file; -- a different path list decodes with the input codec and encodes with the - output codec; -- hard-link failure is explicit; -- no silent file-copy fallback is introduced. - -#### SF-local subtree traversal - -`sf_merger::NodeWriter::copy_subtree_to_output()` remains in `sf_merger`. It -uses shared traversal and `Storage::copy_from()`, but it is not promoted to a -generic store API during this refactor. - -SF validation guarantees that its indexed inputs contain only `Leaf` and -`Virtual` nodes. The SF-local traversal skips `Virtual`, copies `Leaf`, and -does not define behaviour for `Inner`. An `Inner` node is rejected before -merge or cut processing with `sf::InvalidTopology` containing the offending -key. - -When `rf_merger` is designed, it can initially compose `store::traverse` and -`Storage::copy_from()`. At that point, the SF and RF implementations provide -enough evidence to decide whether a shared subtree copier is useful and how it -must handle RF `Inner` nodes. - -#### Error propagation - -The lower storage layer already represents ordinary copy failures, including -missing source files, directory creation failure, hard-link failure, decode -failure, and encode failure. The current `NodeWriter` consumes -`Storage::copy_from()` with `DEBUG_ASSERT_VAL`, as does the unchanged-leaf path -in `cut_leaf_node()`, while some overwrite paths terminate through -`LOG_ERROR_AND_EXIT()`. Their `void` call chains prevent the application from -reporting or handling these failures. - -Return failures from the SF-local subtree and cut functions through their -callers until the application boundary can report the affected key and path. -Codec, filesystem, unsupported-conversion, malformed-dataset, and overwrite -failures are propagated with `std::expected`; `CopyError` retains any -underlying `CodecError`. Assertions remain appropriate for internal -invariants, but operational failures are not assertion failures or -intentionally thrown exceptions. - -### Paired hierarchy walking is deferred - -This refactor does not extract `sf_merger::Merger` into a shared paired-tree -walker. SF only permits `Missing`, `Leaf`, and `Virtual`, and its current -recursion does not provide enough evidence to define the `Inner` behaviour -needed by raster-fundamentalis and tile-base merging. - -The previously proposed mutually exclusive actions `Recurse`, `Ignore`, -`KeepLeft`, `KeepRight`, and `Write` cannot express both an action for the -current physical payload and recursion into descendants. `Inner` merging may -require both. Whether the future interface uses a combined `WriteAndRecurse` -action or independent current-node and descendant decisions belongs to the -`rf_merger` design. - -For this refactor: - -- keep recursion, subtree traversal, and mesh policy in `sf_merger`; -- validate SF inputs and reject `Inner` through `std::expected`; -- move only the one-node `Storage::copy_from()` mechanism into `store`; and -- do not add a shared `store::merge` namespace. - -A future `rf_merger` task will define the paired-tree action algebra from the -2D requirements and may migrate `sf_merger` once both use cases are known. - -## Implementation phases - -Each phase should be one reviewable commit unless the tests and implementation -are clearer as two commits. Do not begin a later phase while the current phase -has failing tests. - -### Phase 0 — Capture current compatibility - -No production behaviour changes. - -1. Add golden SF fixtures created by the current code: - - one `terrain.index` using `flat`; - - one using `level_and_coordinate_directories`; - - across the fixtures, physical payload paths for a root, child, and deeper - descendant, without placing physical payloads at ancestor and descendant - keys in the same index; and - - index entries containing `Leaf` and `Virtual`, but no `Inner`. -2. Add one golden DAG dataset whose index selects `.bin`, whose payload - contains a valid serialized `dag::ClusterBatch`, and whose index contains - `Leaf`, `Virtual`, and `Inner`. Use the current metadata-then-clustering - format, include non-trivial group metadata, and prove that the same file can - be opened through the read-only `NodeMetadata` view. -3. Test that all fixtures open, resolve the expected IDs and extensions, and - traverse the expected sparse nodes. -4. Add path round-trip tests for boundary IDs and both layouts. -5. Add storage tests for: - - matching-extension hard links; - - different-extension decode/re-encode; - - overwrite-enabled replacement; - - indexed and unindexed opens; and - - final index creation by directory scan. -6. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, - `sf_index_browser`, `dag_builder`, and `dag_convert_debug`, including - `DagMetaStorage`, `IndexedDagMetaStorage`, and `dag::ThreadSafeStorage`. - -Exit criterion: the compatibility tests pass against the untouched -implementation and fail when any stable filename, layout ID, path encoding, -status value, index field order, or DAG payload serialization is deliberately -changed. - -### Phase 1 — Extract topology into `store` - -1. Move `NodeStatus` and `NodeStatusOrMissing` to `store`, preserving their - underlying values and serialization. -2. Introduce the hierarchy-traits concept and `octree::StoreTraits`. -3. Convert `IndexMap` into `store::Index`. -4. Convert traversal into `store::traverse`. -5. Add `raster_store::StoreTraits` for `radix::tile::Id`. -6. Run the same index-transition and DFS/BFS tests with both trait types, - including deterministic child order, invalid-key errors through - `std::expected`, maximum-depth children, and explicit traversal roots. -7. Provide the temporary forwarding `octree::IndexMap` wrapper and aliases so - downstream migration is separate; otherwise migrate downstream immediately. - -Exit criterion: 2D and 3D keys pass the same topology suite; existing 3D -callers still build through aliases; no filesystem code has changed. - -### Phase 2 — Introduce path mappings and runtime codecs - -This phase introduces the extensionless layout and runtime codec pieces -together. It does not cut production storage over to them yet. The existing -storage, layout strategies, and static codecs remain temporarily as the -working compatibility path until Phase 3 can replace the complete -layout-plus-codec path construction in one step. - -1. Add `store::NodePath`, `store::path_layout::Mapping`, and - `store::path_layout::Resolver`. -2. Add the stateful `store::Codec` interface with `paths()`, `read()`, - and `write()`, plus `CodecError`. -3. Add the runtime `store::codec::ZppBits`, preserving the existing `.bin` - path and serialized payload bytes. -4. Consolidate the DAG serialization functions in - `src/dag_builder/serialization.h` without changing their serialized field - order, meshoptimizer encoding, or JPEG texture encoding. -5. Add the terrain codec and one glTF codec configured for binary `.glb` or - JSON `.gltf`. Keep the current extension-dispatching `octree::MeshCodec` - only as temporary production compatibility glue until the Phase 3 cutover. - Test that glTF writer exceptions become `CodecError` values. -6. Port the two existing 3D layouts to ordinary function pairs without - changing stable IDs. The mappings return `level-index` and - `level/x/y/z` without file endings. -7. Add explicit `from_id()` and `all()` lookup functions in the 3D adapter. - Keep the singleton strategy registry only for the old production storage - path until Phase 3. -8. Compose each new 3D mapping with each applicable runtime codec in tests and - prove that they resolve all Phase 0 fixtures to identical physical payload - paths. -9. Add focused codec tests using single-file, multi-file, read/write, and - write-only test codecs. Test stable path ordering, unsupported operations, - directory creation, conversion of domain errors to `CodecError`, and - concurrent reads and writes. Exercise reentrancy of every production codec - used by the parallel DAG builder. - -Exit criterion: the extensionless mappings and runtime codecs together resolve -all Phase 0 fixtures to their existing physical payload paths; generic -`Layout` contains no extension; the new codec tests pass; existing production -storage and all callers still build unchanged through the temporary legacy -path. - -### Phase 3 — Generalize storage and index lifecycle - -1. Cut the production path construction over to the Phase 2 - `store::path_layout::Resolver` and runtime codecs. Move the legacy preferred extension - out of layout state and retain it as the 3D format adapter's codec selector. -2. Port legacy layout discovery so it recognizes codec endings through the - supplied resolver, strips the accepted ending, and then calls - `node_path_to_key()`. -3. Move copy error, raw storage, logical storage, and indexed storage into - `store`. Port the existing cache API where useful for source compatibility, - but do not require cache behaviour tests. -4. Make `RawStorage` exclusively own a configured - `std::unique_ptr>`. `Storage` owns it transitively through - raw storage; remove the codec template parameter from every storage type. -5. Replace every embedded `octree::Id` with `Traits::Key`. -6. Make every raw file operation obtain its complete file list through - `Codec::paths()`. `has()` requires every listed file, and `remove()` removes - every listed file. -7. Keep domain-specific mesh codecs outside the shared module under - `mesh::codec`. -8. Add the function-pointer-based `IndexFormat`, `IndexMetadata`, and - typed format/open errors. Split generic index maintenance from 3D index - serialization and legacy folder discovery. Test that only a missing index - starts discovery; all other index errors are returned. -9. Keep the current 3D `terrain.index` DTO and open functions as compatibility - adapters over the shared storage. Retain its exact preferred extension as - `codec_selector`, resolve it through the payload-domain mesh or DAG - resolver, and retain the format metadata required by automatic saving. -10. Add DAG storage convenience functions that supply the writable batch and - read-only metadata resolvers, and mesh storage convenience functions that - select the configured terrain or glTF codec. Preserve the DAG metadata - storage aliases. Migrate `dag_builder`, `dag_convert_debug`, and mesh - storage consumers without exposing codec objects at application call sites. -11. Migrate the existing octree storage aliases and all other application - callers, including adapting `dag::ThreadSafeStorage` to the new key and - expected-returning APIs. -12. Preserve the current 3D destructor-save behaviour until all callers have - explicit index finalization. Test move construction, move assignment over - dirty state, moved-from destruction, and the DAG move/release path. -13. Preserve `StorageSettings::allow_overwrite`. Replace process termination - on a rejected overwrite with `AlreadyExists` in the storage-level expected - error. Test that a rejected save returns `AlreadyExists` and that enabling - overwrite replaces the existing payload. -14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, writable - `ClusterBatch` `.bin`, and read-only `NodeMetadata` `.bin` dispatch, plus - explicit failure for an unknown preferred extension and metadata writes. -15. Instantiate the shared storage tests with `raster_store::StoreTraits` - using a test-only path mapping and codec. This proves the storage templates - contain no hidden `octree::Id` dependency without defining a stable RF - layout, codec, or disk format. -16. Delete the old strategy base class, strategy registry, concrete strategy - classes, static codec concept and codecs, and their temporary compatibility - glue once no call site uses them. - -Test that the Phase 0 DAG fixture opens through both new resolvers and that a -new deterministic `.bin` payload matches its golden bytes and remains readable -through the unchanged ZPP serialization functions. Test unknown layout IDs, -malformed index metadata, invalid hierarchy keys, and retained underlying -open/codec errors through `std::expected`. - -Exit criterion: all existing applications build and all Phase 0 fixtures pass -through the shared runtime codec and storage implementation. Phase 0 DAG -payload bytes and `.bin` paths remain unchanged. No extension-dispatching -mesh codec, layout inheritance, RTTI lookup, static registrar, owning strategy -pointer, or second storage implementation remains under `octree`. - -### Phase 4 — Harden node reuse and enforce SF topology - -1. Change `Storage::copy_from()` to compare input and output codec path lists - for the fixed dummy `NodePath`. -2. Hard-link all actual files when the lists match. A partially failed - multi-file operation returns an error without rolling back links already - created. -3. Decode with the input codec and encode with the output codec when lists - differ. -4. Test: - - one-file hard linking; - - multi-file hard linking; - - different path counts and endings; - - error propagation after a partially failed multi-file hard link; - - overwrite failure leaving an existing logical node unindexed even when - old or partial files remain; - - conversion between terrain and glTF; - - `copy_from()` overwrite rejection; - - `copy_from()` overwrite-enabled replacement; and - - missing-file, hard-link, decode, and encode error propagation. -5. Add `sf::validate_index()`, returning `sf::InvalidTopology` with the - offending key when it encounters `Inner`. Do not add other validation rules - in this refactor. -6. Apply the validator to SF-merger merge and cut inputs and the DAG builder's - SF input before processing. Apply it to SF-builder and SF-merger output - after the completed `terrain.index` has been written. If output validation - fails, retain the written index and payloads for diagnosis, propagate - `sf::InvalidTopology` to the command line, and report that the output is - invalid. Do not apply the validator when opening DAG datasets, through - generic octree/store adapters, or in the diagnostic `sf_index_browser`. - Extract an SF-builder finalization boundary into `sfbuilderlib` which writes - the index, validates it, and returns the typed result so output validation - and command-line propagation are directly testable. -7. Keep SF recursion, subtree traversal, and mesh policy in `sf_merger`. - Change its subtree and cut call chains to propagate validation and - `copy_from()` failures through `std::expected` to the application boundary. -8. Add integration tests proving: - - valid `Leaf`/`Virtual` SF merge behaviour is unchanged; - - an SF input containing `Inner` fails validation before merge dispatch; - - invalid SF output writes `terrain.index` for diagnosis and returns - `sf::InvalidTopology` to the application boundary; - - an unchanged SF subtree is hard-linked and a changed boundary node is - newly written; and - - an unchanged leaf in the SF cut path is hard-linked while a clipped leaf - is newly written. - -Exit criterion: one-node copying works through `Codec::paths()`, SF consumers -reject `Inner` with a typed error before processing, existing valid SF merge -and cut behaviour is preserved, invalid SF output remains inspectable with a -written `terrain.index`, and neither a shared subtree copier nor a paired-tree -walker has been introduced. - -### Phase 5 — Cleanup and documentation - -1. Remove temporary forwarding headers that no repository caller needs. -2. Remove obsolete files under `octree/disk` and the old generic - implementation under `octree/storage`. -3. Keep only 3D key, format, codec, and compatibility adapters under - `octree`. -4. Update includes, CMake source lists, and precompiled-header includes. -5. Update [architecture.md](architecture.md), - [storage-format.md](storage-format.md), [status-quo.md](status-quo.md), and - the before-refactor report with links to the implemented boundary. State - clearly that RF formats and tools remain future work and that architecture - requirements are not acceptance criteria for this refactor. Preserve the - before-refactor report as history; do not rewrite it as if it described the - new code. -6. Document the final public names, a legacy 3D opening example, and an - in-memory `radix::tile::Id` topology example. Do not document a persistent - RF format or opening API. - -Exit criterion: repository search finds no generic implementation tied to -`octree::Id`; all tests pass; the old layout strategy hierarchy is gone. - -### Phase 6 — Remove migration-only compatibility fixtures - -1. Delete the Phase 0 golden SF and DAG datasets and the tests whose only - purpose is opening or byte-comparing data written before the completed - refactor. -2. Keep the format round-trip, path, resolver, topology, storage, error, and - application integration tests which exercise the final implementation - without pre-refactor fixture files. - -Exit criterion: no pre-refactor dataset fixture remains, and the retained test -suite passes against data produced by the final implementation. - -## Test and verification plan - -Generic store tests should live in the existing `unittests_terrainlib` target. -Suggested files: - -```text -unittests/terrainlib/store_index.cpp -unittests/terrainlib/store_traverse.cpp -unittests/terrainlib/store_layout.cpp -unittests/terrainlib/store_codec.cpp -unittests/terrainlib/store_storage.cpp -unittests/terrainlib/store_compatibility.cpp # temporary through Phase 5 -unittests/terrainlib/sf_validate_index.cpp -``` - -The temporary DAG payload-compatibility fixture and resolver integration test -belong in `unittests_dagbuilder`, because `terrainlib` must not depend on -`dag::ClusterBatch` or its serializers. Phase 6 removes the fixture and its -byte-comparison test while retaining resolver tests built from current data. - -The validator's unit tests belong in `unittests_terrainlib`. Boundary tests -belong with their consumers: SF-builder output validation in -`unittests_sfbuilder`, merge and cut validation/error propagation in -`unittests_sfmerger`, and DAG-builder SF-input validation in -`unittests_dagbuilder`. `sf_index_browser` remains unvalidated by design. - -Cache migration has compile coverage only. Existing cache application call -sites must continue to build where the API remains meaningful, but no cache -behaviour or compatibility test is required. - -During implementation: - -1. Build in `$source_dir/build/$config_name`. -2. Run unit tests from that build directory. -3. Run the focused store tests after each edit. -4. Run the full `unittests_terrainlib` target at every phase boundary. -5. Run `unittests_dagbuilder` after the ZPP codec, DAG serialization header, or - DAG resolver changes. -6. Configure with `ALP_BUILD_SF_BUILDER`, `ALP_BUILD_SF_MERGER`, - `ALP_BUILD_SF_INDEX_BROWSER`, `ALP_BUILD_DAG_BUILDER`, and - `ALP_BUILD_DAG_CONVERT_DEBUG` enabled, then build `sf-builder`, `sf-merger`, - `sf-index-browser`, `dag-builder`, and `dag-convert-debug` after their - storage aliases move. -7. Run `unittests_sfbuilder`, `unittests_sfmerger`, and - `unittests_dagbuilder`, plus any existing merger integration fixture, after - the Phase 4 validation changes. -8. Inspect `git diff --check` and the final worktree before each commit. - -No formatting-only pass or unrelated refactor belongs in these commits. - -## Expected migration map - -| Current code | Target | -|---|---| -| `octree/NodeStatus.h` | `store/NodeStatus.h` plus temporary alias | -| `octree/NodeStatusOrMissing.h` | `store/NodeStatusOrMissing.h` plus temporary alias | -| `octree/IndexMap.*` | `store/Index.h` | -| `octree/traverse.h` | `store/traverse.h` | -| complete node paths embedded in layouts | extensionless `store/NodePath.h` plus codec endings | -| `octree/disk/Layout.h` | `store/path_layout.h` | -| `octree/disk/layout/Strategy.h` | `store/path_layout.h` | -| `StrategyRegister.h` | explicit dimension-adapter lookup functions | -| `strategy/Flat.h` | `octree/store_layout/Flat.h` | -| `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | -| `octree/storage/cache/*` | `store/cache/*` where useful; compile compatibility only | -| `octree/storage/codec/Codec.h` | runtime `store/Codec.h` | -| `octree/storage/codec/DefaultCodec.h` | runtime `store/codec/ZppBits.h` | -| `octree/storage/codec/MeshCodec.h` | `mesh/codec/Terrain.h` and configured `mesh/codec/Gltf.h` | -| `octree/storage/codec/ReadOnlyCodec.h` metadata specialization | `dag::codec::MetadataView` | -| `octree/storage/RawStorage.h` | `store/RawStorage.h` | -| `octree/storage/Storage.h` | `store/Storage.h` | -| `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | -| `octree::StorageSettings::allow_overwrite` | `store::StorageSettings::allow_overwrite`, preserving default and enabled behaviour | -| `octree/storage/helpers.*` | generic scan helpers plus 3D format adapter | -| `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | -| DAG serializers in `dag_node.h`, `dag_id.h`, `metadata.h`, `encoded.h`, and `zpp_bits_glm.h` | `dag_builder/serialization.h` | -| `dag_builder/storage.h` aliases | Batch and read-only metadata storage aliases plus codec resolver convenience functions | -| `dag_builder/thread_safe_storage.h` | Adapted caller-side wrapper over shared storage | -| `sf_merger::NodeWriter` subtree loop | remains in `sf_merger`; return copy failures through `std::expected` | -| `sf_merger::NodeWriter` auxiliary `.png` write | remains an unmanaged, application-local debug artifact | -| `sf_merger::cut_leaf_node()` copy path | remains in `sf_merger`; return copy failures through `std::expected` | -| SF merger `Inner` `UNREACHABLE()` path | `sf::validate_index()` returning `sf::InvalidTopology` | - -## Risks and controls - -| Risk | Control | -|---|---| -| Existing indexes stop loading | Golden pre-refactor fixtures and unchanged 3D DTO | -| The refactor changes current DAG `.bin` bytes | Temporary golden DAG fixture, unchanged serializers, and explicit `.bin` resolvers | -| Octree format adapter gains DAG dependencies | Caller-supplied resolver owned by `dag_builder` | -| Unknown legacy extension silently selects the wrong codec | Return an explicit `UnsupportedCodec` error | -| Valid legacy paths are parsed differently | Characterization and round-trip tests before replacement | -| Template migration creates a large unreviewable diff | Compatibility aliases and phase-by-phase caller migration | -| `radix::tile::Id` root underflows | Traits intercept root parent lookup | -| Invalid `radix::tile::Id` values enter the shared index | Validate through `raster_store::StoreTraits` and test boundary zooms | -| Invalid `Inner` nodes reach SF merge dispatch | Validate every SF input first and return the offending key in a typed error | -| SF subtree copying is generalized before RF requirements exist | Keep it in `sf_merger`; reconsider extraction with `rf_merger` | -| A paired-walker API is fixed before RF semantics are known | Defer its action algebra until `rf_merger` requirements are defined | -| Multi-file hard linking fails partway through | Remove any existing index entry before modifying target files, stop immediately, leave the node unindexed, propagate the error, and abort the overall operation; old or partial files may remain | -| Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | -| Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | -| Shared code accumulates mesh or provisional RF policy | Dependency tests/review against the source boundary | -| The refactor accidentally fixes the future RF disk format | Do not add a persistent 2D format adapter, layout, or codec | - -## Deferred raster-fundamentalis work - -This refactor is a prerequisite for, not an implementation of, the system -described in [architecture.md](architecture.md). That document and -[storage-format.md](storage-format.md) remain useful design input, but their RF -details are not acceptance criteria for this refactor and are not declared -final here. - -A later RF design phase must resolve and test at least: - -- the persistent index filename and schema, and how they use the existing - generic serialization envelope and versioning support; -- persistent tile keys, coordinate convention, layout IDs, and node paths; -- tile and source-attribution payload formats and codecs; -- snapshot construction, validation, publication, and crash expectations; -- hard-link preflight and unchanged-tile reuse; -- `rf_merger` policy, paired-tree walking, and `Inner` copy behaviour; and -- RF builders, converters, debugging outputs, and other tools. - -None of those decisions blocks completion of this refactor. -The removed ideas are retained, without implementation-plan status, in the -explicitly draft [rf_builder notes](rf_builder.md) and -[rf_merger notes](rf_merger.md). diff --git a/docs/raster-store/refactor-status.md b/docs/raster-store/refactor-status.md deleted file mode 100644 index 3a82dc46..00000000 --- a/docs/raster-store/refactor-status.md +++ /dev/null @@ -1,237 +0,0 @@ -# Raster store refactor status - -This file is the resumable implementation log for -[refactor-plan.md](refactor-plan.md). - -## Baseline - -- Branch: `raster-store` -- Baseline commit: `d1309f54cfcd5756786735c6a2c849419c187ed7` -- Baseline tag: `refactor_before` -- Started: 2026-08-14 -- Worktree was clean before implementation began. -- `dev_driver.py doctor` rejects this repository because its root - `CMakeLists.txt` is not the alpine-renderer project. Builds therefore use - this repository's own CMake configuration and build directories. - -## Current position - -- Phase: 6 — Remove migration-only compatibility fixtures -- State: complete; completion commit pending. -- Next action: commit and tag Phase 6, then perform the final clean-worktree - and tag audit. - -## Phase log - -### Phase 0 — Capture current compatibility - -- Added pre-refactor SF fixtures for `flat` and - `level_and_coordinate_directories` layouts. -- Added a pre-refactor DAG fixture using `.bin`, with `Leaf`, `Virtual`, and - `Inner` index states and non-trivial metadata. -- Added characterization coverage for fixture opening, metadata-prefix reads, - payload byte stability, layout/path round trips, hard links, - decode/re-encode, enabled overwrites, indexed/unindexed opens, and index - creation by directory scan. -- Recorded pre-refactor public aliases: - - SF builder, SF merger, and SF index browser use `octree::MeshStorage` and - `octree::IndexedStorage` (the latter aliases `IndexedMeshStorage`). - - DAG builder and DAG convert debug use `octree::DagStorage`, - `octree::IndexedDagStorage`, `octree::DagMetaStorage`, and - `octree::IndexedDagMetaStorage`. - - DAG output synchronization uses - `dag::ThreadSafeStorage`. -- State: complete. - -### Phase 1 — Extract topology into `store` - -- Added shared node-status types, hierarchy traits, invalid-key errors, - sparse index, and traversal under `terrainlib/store`. -- Added `octree::StoreTraits` and `raster_store::StoreTraits`. -- Converted `octree::IndexMap` and `octree::traverse` to compatibility - wrappers over the shared implementation. -- Added common 2D/3D topology and traversal coverage plus 2D boundary and - invalid-key tests. -- State: complete. - -### Phase 2 — Introduce path mappings and runtime codecs - -- Added extensionless `NodePath`, `PathMapping`, and `Layout` values. -- Added the stateful runtime `Codec` interface and typed `CodecError`. -- Added runtime ZPP Bits, terrain, binary glTF, and JSON glTF codecs. -- Added ordinary `flat` and `level_and_coordinate_directories` mapping - function pairs plus explicit lookup. -- Consolidated DAG serialization in `dag_builder/serialization.h` and made - the legacy DAG storage adapter include it explicitly. -- Added mapping/codec composition, parser validation, single-/multi-file, - unsupported-operation, write-only, directory-creation, error-conversion, - and concurrent codec tests. -- State: complete. - -### Phase 3 — Generalize storage and index lifecycle - -- Added typed storage, filesystem, overwrite, and index-format errors. -- Added shared `RawStorage`, `Storage`, and `IndexedStorage` with exclusive - runtime-codec ownership, trait-keyed operations, multi-file `has`/`remove`, - overwrite handling, index persistence, and move finalization. -- Added shared storage instantiations for both 2D and 3D traits plus move, - overwrite, invalid-key, and multi-file tests. -- Shared storage core focused tests: 48 assertions in 6 test cases passed. -- Shared storage core build and `git diff --check`: passed. -- Added the legacy 3D `terrain.index` format adapter, explicit mesh and DAG - codec resolvers, folder discovery, and expected-returning opening APIs. -- Preserved independently readable, read-only DAG metadata views and migrated - DAG output synchronization to the shared storage types. -- Migrated SF builder, SF merger, SF index browser, DAG builder, and DAG - convert debug without exposing codec ownership at application call sites. -- Ported cache API aliases for compile compatibility. -- Deleted the legacy layout class hierarchy and registry, static codec stack, - and duplicate octree storage implementation. -- Added resolver, unsupported metadata write, unknown layout/codec, malformed - index no-fallback, and DAG synchronized move/release coverage. -- State: complete. - -### Phase 4 — Harden node reuse and enforce SF topology - -- Hardened `copy_from()` around the fixed codec probe path: matching path - lists hard-link every physical file, differing lists decode/re-encode, and - actual path-count mismatches return a typed codec error. -- Moved overwrite index removal to the mutation boundary. Missing-source and - decode failures leave the old logical target indexed; encode and partial - multi-file hard-link failures leave it unindexed for safety. -- Added one-file and multi-file hard-link, re-encode, overwrite, - missing-source, decode, encode, and partial-hard-link failure coverage. -- Added `sf::InvalidTopology`, `sf::validate_index()`, typed SF processing and - finalization errors, and an SF finalization boundary that writes the index - before validating it. -- Applied SF validation before SF merge/cut and DAG processing and after SF - builder/merger output finalization. The diagnostic index browser remains - exempt. -- Propagated SF merger save, copy, open, index, and validation failures through - `std::expected` to the command-line boundary. -- Added SF builder, SF merger, SF cut, and DAG builder boundary tests, - including invalid-input rejection, inspectable invalid output, unchanged - hard links, and newly written changed/clipped nodes. -- State: complete. - -### Phase 5 — Cleanup and documentation - -- Migrated repository callers from the temporary octree topology, traversal, - storage, and cache aliases to the final `store`, `mesh::storage`, and - `dag::storage` public names. -- Removed the temporary `octree::IndexMap`, status, traversal, mesh-storage, - and cache forwarding headers and their wrapper-only unit tests. -- Changed the unchanged legacy `terrain.index` DTO to serialize the shared - `store::Index` directly. -- Kept only 3D identity/geometry, traits, mappings, legacy index-format, and - opening adapters under `octree`. -- Updated the architecture, storage-format, status-quo, historical - before-refactor report, and raster-store index documentation. The docs now - distinguish the implemented shared boundary from future RF formats and - tools and include final public names plus 3D-open and in-memory 2D examples. -- State: complete. - -### Phase 6 — Remove migration-only compatibility fixtures - -- Removed the golden pre-refactor SF and DAG indexes and payload files. -- Removed fixture-only SF/DAG opening and byte-comparison assertions. -- Converted DAG codec reentrancy, metadata-prefix reading, synchronized - storage, SF-builder finalization, and DAG validation tests to generate - current data in temporary directories. -- Retained layout/path, resolver, topology, storage, error, merge/cut, and - producer/consumer boundary coverage. -- Repository search and filesystem inspection find no remaining pre-refactor - fixture reference or fixture file. -- State: complete. - -## Verification log - -- Baseline `unittests_terrainlib`: 23,676 assertions in 391 test cases passed. -- Baseline `unittests_dagbuilder`: 400 assertions in 69 test cases passed. -- Focused Phase 0 terrainlib compatibility tests: 51 assertions in 6 test - cases passed before warning cleanup. -- Focused Phase 0 DAG compatibility test: 23 assertions passed. -- Phase 0 `unittests_terrainlib`: 23,732 assertions in 397 test cases passed. -- Phase 0 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. -- Phase 0 `git diff --check`: passed. -- Phase 1 focused shared-store tests: 67 assertions in 7 test cases passed. -- Phase 1 pre-refactor terrain compatibility tests: 56 assertions in 6 test - cases passed. -- Phase 1 `unittests_terrainlib`: 23,799 assertions in 404 test cases passed. -- Phase 1 `unittests_dagbuilder`: 423 assertions in 70 test cases passed. -- Phase 1 application builds passed for `sf-builder`, `dag-builder`, and - `dag-convert-debug`. The current Debug configuration has `sf-merger` and - `sf-index-browser` disabled. -- The all-target build remains blocked by a pre-existing, unrelated call to - removed `radix::tile::Scheme` in `unittests/sf_builder/texture.cpp`; the - affected refactor targets and suites build independently. -- Phase 1 `git diff --check`: passed. -- Phase 2 focused shared-store tests: 160 assertions in 17 test cases passed. -- Phase 2 DAG golden fixture: 30 assertions passed, including runtime codec - output byte equality. -- Phase 2 concurrent DAG runtime codec: 10 assertions passed. -- Phase 2 `unittests_terrainlib`: 23,892 assertions in 414 test cases passed. -- Phase 2 `unittests_dagbuilder`: 440 assertions in 71 test cases passed. -- Phase 2 application builds passed for `sf-builder`, `dag-builder`, and - `dag-convert-debug`. -- Phase 2 `git diff --check`: passed. -- Phase 3 focused terrain store/open tests: 233 assertions in 25 test cases - passed. -- Phase 3 focused DAG store tests: 17 assertions in 2 test cases passed. -- Phase 3 `unittests_terrainlib`: 23,973 assertions in 422 test cases passed. -- Phase 3 `unittests_dagbuilder`: 459 assertions in 73 test cases passed. -- Phase 3 all-application configuration built `sf-builder`, `sf-merger`, - `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. -- Phase 3 `git diff --check`: passed. -- Phase 4 focused storage-copy tests: 70 assertions in 5 test cases passed. -- Phase 4 SF validator tests: 6 assertions in 2 test cases passed. -- Phase 4 DAG SF-input boundary test: 12 assertions passed. -- Phase 4 SF merger boundary tests: 66 assertions in 4 test cases passed. -- Phase 4 SF builder finalization tests: 19 assertions in 2 test cases passed. -- Phase 4 `unittests_terrainlib`: 24,049 assertions in 429 test cases passed. -- Phase 4 `unittests_dagbuilder`: 471 assertions in 74 test cases passed. -- Phase 4 `unittests_sfmerger`: 121 assertions in 8 test cases passed. -- Phase 4 all-application configuration built `sf-builder`, `sf-merger`, - `sf-index-browser`, `dag-builder`, and `dag-convert-debug` successfully. -- The pre-existing `unittests_sfbuilder` compile failure involving removed - `radix::tile::Scheme` remains isolated; the new finalization tests build and - pass in `unittests_sfbuilder_finalization`. -- Phase 4 `git diff --check`: passed. -- Phase 5 `unittests_terrainlib`: 24,017 assertions in 426 test cases passed. -- Phase 5 `unittests_dagbuilder`: 472 assertions in 74 test cases passed. -- Phase 5 `unittests_sfmerger`: 121 assertions in 8 test cases passed. -- Phase 5 SF builder finalization: 19 assertions in 2 test cases passed. -- Phase 5 application builds passed for `sf-builder`, `sf-merger`, - `sf-index-browser`, `dag-builder`, and `dag-convert-debug`. -- Phase 5 repository searches found no remaining caller of a removed octree - forwarding name and no `octree::Id` dependency under `terrainlib/store`. -- Phase 5 `git diff --check`: passed. -- Phase 6 `unittests_terrainlib`: 23,997 assertions in 425 test cases passed. -- Phase 6 `unittests_dagbuilder`: 439 assertions in 73 test cases passed. -- Phase 6 `unittests_sfmerger`: 121 assertions in 8 test cases passed. -- Phase 6 SF builder finalization: 15 assertions in 2 test cases passed. -- Phase 6 fixture-reference search, fixture-file inspection, and - `git diff --check`: passed. - -## Commit and tag log - -- `refactor_before` — annotated baseline tag at `d1309f5`. -- `75277f8` — Phase 0 compatibility tests, indexes, DAG payloads, and status - log. -- Phase 0 fixture-payload follow-up — force-adds the intentionally committed - `.terrain` golden payloads ignored by the repository's general artifact - rule. -- `refactor_phase_0` — annotated Phase 0 completion tag. -- `4c171c8` / `refactor_phase_1` — shared topology extraction and Phase 1 - completion tag. -- `fbc02d0` / `refactor_phase_2` — runtime layouts/codecs and Phase 2 - completion tag. -- `61acb74` — Phase 3 shared runtime-codec storage core. -- `4ffcdea` / `refactor_phase_3` — Phase 3 application migration and - completion tag. -- `3086336` / `refactor_phase_4` — hardened node reuse, SF topology - enforcement, and Phase 4 completion tag. -- `bca0b5c` / `refactor_phase_5` — removed migration shims, documented final - public boundaries, and tagged Phase 5 completion. -- Phase 6 completion commit / `refactor_phase_6` — removed migration-only - fixtures while retaining current-data coverage. diff --git a/docs/raster-store/status-quo.md b/docs/raster-store/status-quo.md deleted file mode 100644 index 461faf46..00000000 --- a/docs/raster-store/status-quo.md +++ /dev/null @@ -1,287 +0,0 @@ -# Status quo and reuse assessment - -> **Historical assessment:** this document records the repository before the -> shared-store refactor and is intentionally not rewritten as current code -> documentation. The completed implementation and verification record is in -> [refactor-status.md](refactor-status.md); the resulting public boundary is -> summarized in [architecture.md](architecture.md#implementation-status). - -The refactor chose the generalization option discussed below. Sparse topology, -traversal, layouts, runtime codecs, storage, index lifecycle, and cache -interfaces now live under `store` and are parameterized by hierarchy traits. -The legacy 3D format remains an adapter under `octree`, while -`raster_store::StoreTraits` proves the shared in-memory mechanisms with -`radix::tile::Id`. Persistent RF formats and RF tools remain future work. - -This document evaluates the current repository after commit `9cf9065` -(`Consolidate raster handling and use std::expected`). It distinguishes -reusable mechanisms from interfaces that encode assumptions unsuitable for -the raster store. - -## Summary - -The project already contains most low-level ingredients: - -- `radix::Raster` for contiguous typed raster memory; -- `radix::tile::Id` for quadtree addressing; -- Web Mercator grid calculations and GDAL reprojection in `tile_builder`; -- sparse topology, indexed traversal, codecs, layouts, and hard-link reuse in - the octree storage code; and -- the SF merger's snapshot-like reuse of unchanged subtrees. - -There is no existing component that should become the raster store unchanged. -The best path is to compose the Radix raster and tile primitives with a new -2D storage layer, while extracting or adapting selected octree-storage ideas. - -## Reuse matrix - -| Component | Assessment | Intended use | -|---|---|---| -| `radix::Raster` | Reuse directly | In-memory payloads and source maps | -| `radix::RasterMask` | Reuse directly | Temporary validity/selection masks | -| `radix::raster::transform` | Reuse directly | Typed pixel transformations | -| `radix::raster::generate_mipmap` | Do not use as general generator | Only a 2x2 component-wise box average | -| `radix::tile::Id` | Reuse after hardening or through an adapter | Persistent quadtree keys | -| `radix::quad_tree::Node` | Do not reuse for disk index | Dense in-memory ownership model | -| CTB `GlobalMercator` and grid bounds | Reuse initially | Tile bounds and Web Mercator resolution | -| `Dataset` and GDAL setup | Reuse/adapt | Input discovery and reprojection | -| `DatasetReader` | Adapt substantially | Windowed, typed, multi-band ingestion | -| `Tiler` / `ParallelTiler` | Reuse calculations, replace orchestration | Candidate chunk enumeration | -| `ParallelTileGenerator` | Do not use as store builder | Small-file writer with incompatible lifecycle | -| Octree `IndexMap` algorithm | Reuse design; generalize or port | Sparse physical/virtual topology | -| Octree `Storage_` / `RawStorage_` | Reuse design and selected code | Codec boundary, indexing, hard-link copy | -| Octree disk layouts | Do not use as-is | They encode `octree::Id` and 3D paths | -| SF merger visitors and geometry code | Do not reuse | Mesh- and ECEF-specific semantics | -| SF merger unchanged-subtree copy | Reuse design | Snapshot construction and hard links | -| `zpp_bits` serialization helpers | Reuse cautiously | Versioned metadata/index serialization | - -## Radix raster - -### What can be reused - -`extern/radix/src/radix/raster.h` now provides a shared, value-typed raster: - -- rectangular `glm::uvec2` dimensions; -- contiguous row-major `std::vector` storage; -- element and byte spans; -- typed pixel access; -- move construction from an existing vector; -- a contiguous byte-valued `RasterMask`; -- dimension-checked concatenation; and -- masked and unmasked transforms. - -These properties fit both store arrays: - -```cpp -radix::Raster payload; -radix::Raster source_map; -``` - -The class correctly remains independent of Web Mercator, tile IDs, source -metadata, compression, and file I/O. Those belong to higher layers. - -### What needs adaptation around it - -Large chunks and filtered generation will benefit from operations not -currently supplied by `Raster`: - -- non-owning raster views and subwindows; -- explicit row stride where external codecs require it; -- halo/window assembly across neighbouring chunks; -- checked construction that reports allocation/dimension errors without - relying on assertions; and -- streaming or block processing when a full set of input chunks would exceed - the memory budget. - -These should be introduced only when required. They are not reasons to embed -store concepts into `Raster`. - -### What cannot be reused for final filtering - -`radix::raster::generate_mipmap` requires a square, power-of-two raster and -reduces each 2x2 group by component-wise averaging. It does not provide: - -- a selectable reconstruction or low-pass filter; -- the phase difference between vertex pixels and area pixels; -- halo samples across tile boundaries; -- linear-light colour and premultiplied-alpha handling; -- NoData-aware normalization; -- categorical reduction; or -- source-contribution tracking. - -It is therefore a useful simple raster utility, not the raster-store pyramid -generator. - -## Radix tile addressing - -### What can be reused - -`radix::tile::Id` already contains the required 2D identity: - -- zoom level; -- `x/y` coordinates; -- parent and four-child relationships; -- TMS/Slippy conversion; and -- hashing and ordering support. - -### Required hardening - -Persistent storage needs stronger invariants than the current convenience -type supplies: - -- Calling `parent()` at zoom zero currently underflows. -- Construction does not reject coordinates outside `[0, 2^z)`. -- Shifting `1u << zoom_level` limits valid conversion at high zooms. -- The scheme participates in identity, so the same spatial tile in TMS and - Slippy form becomes two keys. -- There is no persistent serialization contract or format version. - -The store should choose one canonical scheme and normalize all IDs at its API -boundary. Whether the hardening belongs in Radix or in a checked store adapter -is open. - -`radix::quad_tree::Node` is not a replacement for the index. It owns a -fully allocated group of four children whenever refined, represents no -missing child within such a group, and has no persistence or physical/virtual -status. The store requires a sparse map keyed by tile ID. - -## Tile builder and GDAL path - -### Reusable foundations - -The current tile builder already demonstrates: - -- opening GDAL raster datasets; -- reading dataset bounds; -- selecting Web Mercator or geodetic CTB grids; -- transforming arbitrary source SRS data during reads; -- calculating tile bounds and resolutions; -- enumerating intersecting `radix::tile::Id` values; and -- parallel per-tile processing. - -`ctb::GlobalMercator`, `Tiler::tile_for`, and `ParallelTiler` are useful -references and may be reused initially for grid math. - -### Required changes - -`DatasetReader` currently reads one band into a float raster, uses cubic GDAL -warping, and constructs a warped VRT for a requested output rectangle. The -store needs typed and multi-band reads, explicit alpha/NoData handling, -controlled resampling, source metadata, and deterministic alignment with the -store chunk grid. - -`Tiler` models delivery-oriented dimensions through `Border::Yes/No` and a -south/east extra pixel. Store chunks have no rendering overlap, and generated -vertex pixels require an explicit global sampling/filtering model. The border -boolean should not define store geometry. - -`ParallelTileGenerator` writes individual image files for explicitly -enumerated tiles. It is not suitable as the snapshot builder because it lacks: - -- source-map generation; -- old-snapshot reuse; -- atomic chunk containers; -- index transactions and publication; -- resumability validation; and -- bounded enumeration/streaming for very large tile sets. - -Its parallel work pattern and progress reporting may still inform the new -builder. - -## Octree index and storage - -### Reusable design - -The octree index captures the required topology semantics: - -- `Leaf`: physical payload without indexed descendants; -- `Inner`: physical payload with indexed descendants; -- `Virtual`: no physical payload, but indexed descendants; and -- absence from the map: missing node and subtree. - -Adding a physical node creates virtual ancestors and promotes a physical -ancestor from leaf to inner. Removing nodes collapses unused virtual chains. -Traversal follows only present index entries. - -The storage stack also has useful separation between: - -- logical indexed storage; -- raw path-based storage; -- disk layout; -- payload codec; and -- optional cache. - -`RawStorage_::copy_from` demonstrates hard-link reuse when source and target -extensions match. The SF merger demonstrates copying unchanged physical -subtrees and writing a new output index. - -### Why it cannot be reused unchanged - -The entire stack uses concrete `octree::Id` types. `IndexMap`, cache APIs, -layouts, filesystem path parsing, traversal, storage, serialization, and -formatting all embed this type. The coordinate-directory layout is -`level/x/y/z`, and the default codec is a mesh codec. - -The existing index file also records no tree kind. Reinterpreting its -serialized `(level, index)` octree IDs as web tiles would be unsafe. - -A 2D implementation can either: - -1. generalize the hierarchy/storage stack over an ID and layout policy; or -2. create a raster-store-specific 2D implementation using the same algorithms. - -Generalization avoids duplicate infrastructure but has a larger blast radius -in mature octree code. A separate implementation is initially safer but risks -long-term duplication. This requires an explicit decision before coding. - -### Behaviours that should not be copied blindly - -- Hard-link failure currently has no copy fallback. -- Existing indexes are trusted rather than reconciled against filesystem - contents when opened. -- An unindexed output discovers files through a final recursive directory - scan. -- Some error paths use assertions or process termination. -- The binary index lacks an explicit magic/version/topology header suitable - for a new durable format. - -The raster store should retain the useful topology and codec boundaries while -specifying stronger snapshot, validation, and error-handling rules. - -## SF builder and merger - -The SF builder's ECEF octree placement, mesh construction, texture atlases, -mask clipping, and mesh visitors are not reusable for a Web Mercator raster -store. - -The reusable ideas are architectural: - -- transform source data into a canonical spatial system during build; -- preserve physical ancestors as fallback representations; -- use an index to avoid per-node filesystem probing; -- stop refinement when a coherent representation is sufficient; and -- hard-link unchanged chunks into a new output dataset. - -Those ideas should be reimplemented against Radix tile IDs and raster payloads -rather than adapted through mesh abstractions. - -## Recommended component boundary - -The current components suggest the following dependency direction: - -```text -Radix - Raster, RasterMask, tile::Id, geometry - ↓ -Terrain library - GDAL dataset access, Web Mercator grid math, filtering primitives - ↓ -Raster store - source catalog, chunk container, sparse index, snapshots - ↓ -Builders and generators - ingestion, source selection, texture pyramids, height/geometry pyramids -``` - -The raster store should consume `radix::Raster`; Radix should not depend on -the store's source catalog, file format, or snapshot lifecycle. From a694040752079d5ec4d1f76d0e864f117bb27533 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:21:51 +0200 Subject: [PATCH 42/43] Fix resource exhaustion and temp directory ownership --- src/dag_builder/codec.cpp | 14 ++++++++++++-- unittests/temporary_directory.h | 11 +++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/dag_builder/codec.cpp b/src/dag_builder/codec.cpp index 8bd73390..d2fc979e 100644 --- a/src/dag_builder/codec.cpp +++ b/src/dag_builder/codec.cpp @@ -21,6 +21,12 @@ namespace dag::codec { namespace { + +Error::Code classify_texture_error(const cv::Exception& error, const Error::Code fallback) +{ + return error.code == cv::Error::StsNoMem ? Error::Code::ResourceExhausted : fallback; +} + namespace format { namespace v1 { @@ -209,7 +215,9 @@ inline Expected encode_clustering(const ::Clustering& clustering) result.textures.push_back({ std::move(jpeg) }); } } catch (const cv::Exception& error) { - return Error::fail(Error::Code::InvalidInput, "could not encode DAG texture: " + error.msg); + return Error::fail( + classify_texture_error(error, Error::Code::InvalidInput), + "could not encode DAG texture: " + error.msg); } return result; } @@ -340,7 +348,9 @@ inline Expected<::Clustering> decode_clustering(Clustering clustering) textures.push_back(std::move(image)); } } catch (const cv::Exception& error) { - return Error::fail(Error::Code::CorruptData, "could not decode DAG texture: " + error.msg); + return Error::fail( + classify_texture_error(error, Error::Code::CorruptData), + "could not decode DAG texture: " + error.msg); } result.textures = TextureSet(std::move(textures)); return result; diff --git a/unittests/temporary_directory.h b/unittests/temporary_directory.h index 837e77fb..10f2b60f 100644 --- a/unittests/temporary_directory.h +++ b/unittests/temporary_directory.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -25,6 +26,11 @@ class TemporaryDirectory { REQUIRE(std::filesystem::create_directories(m_path)); } + TemporaryDirectory(const TemporaryDirectory&) = delete; + TemporaryDirectory& operator=(const TemporaryDirectory&) = delete; + TemporaryDirectory(TemporaryDirectory&&) = delete; + TemporaryDirectory& operator=(TemporaryDirectory&&) = delete; + ~TemporaryDirectory() { std::error_code error; @@ -37,4 +43,9 @@ class TemporaryDirectory { std::filesystem::path m_path; }; +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(!std::is_move_constructible_v); +static_assert(!std::is_move_assignable_v); + } // namespace test From ec46555355b86e9ab9c62dbfacb958e02d2ff831 Mon Sep 17 00:00:00 2001 From: adam-ce <5292991+adam-ce@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:40:22 +0200 Subject: [PATCH 43/43] Format new C++ files --- src/dag_builder/codec.cpp | 641 +++++++++--------- src/terrainlib/Error.cpp | 48 +- src/terrainlib/Error.h | 52 +- src/terrainlib/io/compression.cpp | 12 +- src/terrainlib/io/conversion.h | 9 +- src/terrainlib/io/envelope.h | 3 +- src/terrainlib/mesh/codec/SfMesh.h | 13 +- src/terrainlib/octree/storage/IndexFile.h | 9 +- src/terrainlib/octree/storage/open_runtime.h | 9 +- src/terrainlib/sf/validate_index.h | 5 +- src/terrainlib/store/Codec.h | 5 +- src/terrainlib/store/RawStorage.h | 5 +- src/terrainlib/store/Storage.h | 32 +- src/terrainlib/store/Traits.h | 3 +- src/terrainlib/store/traverse.h | 9 +- .../dag_builder/storage_compatibility.cpp | 2 +- unittests/sf_builder/storage_finalization.cpp | 2 +- unittests/sf_merger/storage_boundaries.cpp | 2 +- unittests/terrainlib/store_codec.cpp | 11 +- unittests/terrainlib/store_compatibility.cpp | 2 +- unittests/terrainlib/store_storage.cpp | 17 +- 21 files changed, 389 insertions(+), 502 deletions(-) diff --git a/src/dag_builder/codec.cpp b/src/dag_builder/codec.cpp index d2fc979e..f5a5badf 100644 --- a/src/dag_builder/codec.cpp +++ b/src/dag_builder/codec.cpp @@ -22,95 +22,95 @@ namespace dag::codec { namespace { -Error::Code classify_texture_error(const cv::Exception& error, const Error::Code fallback) -{ - return error.code == cv::Error::StsNoMem ? Error::Code::ResourceExhausted : fallback; -} - -namespace format { - -namespace v1 { - - struct OctreeId { - std::uint8_t level; - std::uint64_t index; - }; - - struct Id { - OctreeId source_batch; - std::uint32_t cluster_index; - }; - - struct Vec3d { - double x; - double y; - double z; - }; - - struct Bounds { - Vec3d min; - Vec3d max; - }; - - struct Group { - std::vector children; - double error; - Bounds bounds; - }; - - struct NodeMetadata { - std::vector group_assignment; - std::vector groups; - }; - - struct Texture { - std::vector jpeg; - }; - - struct Cluster { - std::uint64_t triangle_count; - std::uint64_t vertex_count; - std::uint64_t uv_count; - std::vector encoded_triangles; - std::vector encoded_vertex_indices; - std::vector encoded_uvs; - std::uint32_t id; - std::uint32_t texture_id; - double absolute_error; - }; - - struct Clustering { - std::uint64_t vertex_count; - std::vector encoded_positions; - std::vector clusters; - std::vector textures; - }; - -} // namespace v1 - -using MetadataSchema = io::envelope::PayloadSchema<"dag.NodeMetadata", io::envelope::Version<1, v1::NodeMetadata>>; -using ClusteringSchema = io::envelope::PayloadSchema<"dag.Clustering", io::envelope::Version<1, v1::Clustering>>; -using NodeMetadata = MetadataSchema::latest_type; -using Clustering = ClusteringSchema::latest_type; - -inline v1::OctreeId encode_id(const octree::Id& id) { return { .level = id.level(), .index = id.index_on_level() }; } - -inline Expected decode_id(const v1::OctreeId& id) -{ - auto result = octree::Id::try_make(static_cast(id.level), static_cast(id.index)); - if (!result) { - return Error::fail(Error::Code::CorruptData, "DAG payload contains an invalid octree ID"); - } - return *result; -} + Error::Code classify_texture_error(const cv::Exception& error, const Error::Code fallback) + { + return error.code == cv::Error::StsNoMem ? Error::Code::ResourceExhausted : fallback; + } + + namespace format { + + namespace v1 { + + struct OctreeId { + std::uint8_t level; + std::uint64_t index; + }; + + struct Id { + OctreeId source_batch; + std::uint32_t cluster_index; + }; + + struct Vec3d { + double x; + double y; + double z; + }; + + struct Bounds { + Vec3d min; + Vec3d max; + }; + + struct Group { + std::vector children; + double error; + Bounds bounds; + }; + + struct NodeMetadata { + std::vector group_assignment; + std::vector groups; + }; + + struct Texture { + std::vector jpeg; + }; + + struct Cluster { + std::uint64_t triangle_count; + std::uint64_t vertex_count; + std::uint64_t uv_count; + std::vector encoded_triangles; + std::vector encoded_vertex_indices; + std::vector encoded_uvs; + std::uint32_t id; + std::uint32_t texture_id; + double absolute_error; + }; + + struct Clustering { + std::uint64_t vertex_count; + std::vector encoded_positions; + std::vector clusters; + std::vector textures; + }; + + } // namespace v1 + + using MetadataSchema = io::envelope::PayloadSchema<"dag.NodeMetadata", io::envelope::Version<1, v1::NodeMetadata>>; + using ClusteringSchema = io::envelope::PayloadSchema<"dag.Clustering", io::envelope::Version<1, v1::Clustering>>; + using NodeMetadata = MetadataSchema::latest_type; + using Clustering = ClusteringSchema::latest_type; + + inline v1::OctreeId encode_id(const octree::Id& id) { return { .level = id.level(), .index = id.index_on_level() }; } + + inline Expected decode_id(const v1::OctreeId& id) + { + auto result = octree::Id::try_make(static_cast(id.level), static_cast(id.index)); + if (!result) { + return Error::fail(Error::Code::CorruptData, "DAG payload contains an invalid octree ID"); + } + return *result; + } -inline NodeMetadata encode_metadata(const dag::NodeMetadata& metadata) -{ - NodeMetadata result; - result.group_assignment = metadata.group_assignment; - result.groups.reserve(metadata.groups.size()); - for (const dag::Group& group : metadata.groups) { - v1::Group encoded_group = { + inline NodeMetadata encode_metadata(const dag::NodeMetadata& metadata) + { + NodeMetadata result; + result.group_assignment = metadata.group_assignment; + result.groups.reserve(metadata.groups.size()); + for (const dag::Group& group : metadata.groups) { + v1::Group encoded_group = { .children = {}, .error = group.error, .bounds = { @@ -118,48 +118,48 @@ inline NodeMetadata encode_metadata(const dag::NodeMetadata& metadata) .max = {group.bounds.max.x, group.bounds.max.y, group.bounds.max.z}, }, }; - encoded_group.children.reserve(group.children.size()); - for (const dag::Id& child : group.children) { - encoded_group.children.push_back({ - .source_batch = encode_id(child.source_batch), - .cluster_index = child.cluster_index, - }); + encoded_group.children.reserve(group.children.size()); + for (const dag::Id& child : group.children) { + encoded_group.children.push_back({ + .source_batch = encode_id(child.source_batch), + .cluster_index = child.cluster_index, + }); + } + result.groups.push_back(std::move(encoded_group)); + } + return result; } - result.groups.push_back(std::move(encoded_group)); - } - return result; -} -inline Expected validate(const NodeMetadata& metadata) -{ - for (const std::uint32_t group : metadata.group_assignment) { - if (group >= metadata.groups.size()) { - return Error::fail(Error::Code::InvalidInput, "DAG metadata contains an invalid group assignment"); - } - } - for (const v1::Group& group : metadata.groups) { - if (!std::isfinite(group.error)) { - return Error::fail(Error::Code::InvalidInput, "DAG metadata contains a non-finite error"); - } - for (const v1::Id& child : group.children) { - if (!decode_id(child.source_batch)) { - return Error::fail(Error::Code::InvalidInput, "DAG metadata contains an invalid child ID"); + inline Expected validate(const NodeMetadata& metadata) + { + for (const std::uint32_t group : metadata.group_assignment) { + if (group >= metadata.groups.size()) { + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains an invalid group assignment"); + } + } + for (const v1::Group& group : metadata.groups) { + if (!std::isfinite(group.error)) { + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains a non-finite error"); + } + for (const v1::Id& child : group.children) { + if (!decode_id(child.source_batch)) { + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains an invalid child ID"); + } + } } + return {}; } - } - return {}; -} -inline Expected decode_metadata(NodeMetadata metadata) -{ - if (auto valid = validate(metadata); !valid) { - return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate decoded DAG metadata"); - } - dag::NodeMetadata result; - result.group_assignment = std::move(metadata.group_assignment); - result.groups.reserve(metadata.groups.size()); - for (v1::Group& group : metadata.groups) { - dag::Group decoded_group = { + inline Expected decode_metadata(NodeMetadata metadata) + { + if (auto valid = validate(metadata); !valid) { + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate decoded DAG metadata"); + } + dag::NodeMetadata result; + result.group_assignment = std::move(metadata.group_assignment); + result.groups.reserve(metadata.groups.size()); + for (v1::Group& group : metadata.groups) { + dag::Group decoded_group = { .children = {}, .error = group.error, .bounds = { @@ -167,217 +167,212 @@ inline Expected decode_metadata(NodeMetadata metadata) {group.bounds.max.x, group.bounds.max.y, group.bounds.max.z}, }, }; - decoded_group.children.reserve(group.children.size()); - for (const v1::Id& child : group.children) { - auto source_batch = decode_id(child.source_batch); - if (!source_batch) { - return Error::propagate(std::move(source_batch), "decode DAG metadata child ID"); + decoded_group.children.reserve(group.children.size()); + for (const v1::Id& child : group.children) { + auto source_batch = decode_id(child.source_batch); + if (!source_batch) { + return Error::propagate(std::move(source_batch), "decode DAG metadata child ID"); + } + decoded_group.children.push_back({ *source_batch, child.cluster_index }); + } + result.groups.push_back(std::move(decoded_group)); } - decoded_group.children.push_back({ *source_batch, child.cluster_index }); + return result; } - result.groups.push_back(std::move(decoded_group)); - } - return result; -} -inline Expected encode_clustering(const ::Clustering& clustering) -{ - Clustering result { - .vertex_count = clustering.positions.size(), - .encoded_positions = meshopt::encode_vertex_buffer(clustering.positions), - .clusters = {}, - .textures = {}, - }; - result.clusters.reserve(clustering.clusters.size()); - for (const ::Cluster& cluster : clustering.clusters) { - if (cluster.texture_id == (std::numeric_limits::max)()) { - return Error::fail(Error::Code::InvalidInput, "DAG texture ID collides with the on-disk null sentinel"); - } - result.clusters.push_back({ - .triangle_count = cluster.local_triangles.size(), - .vertex_count = cluster.vertex_indices.size(), - .uv_count = cluster.uvs.size(), - .encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles), - .encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices), - .encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs), - .id = cluster.id, - .texture_id = cluster.texture_id.value_or((std::numeric_limits::max)()), - .absolute_error = cluster.absolute_error, - }); - } - result.textures.reserve(clustering.textures.size()); - try { - for (const cv::Mat& texture : clustering.textures) { - auto jpeg = mesh::io::write_texture_to_encoded_buffer(texture, ".jpeg"); - if (!texture.empty() && jpeg.empty()) { - return Error::fail(Error::Code::InvalidInput, "could not encode DAG texture"); + inline Expected encode_clustering(const ::Clustering& clustering) + { + Clustering result { + .vertex_count = clustering.positions.size(), + .encoded_positions = meshopt::encode_vertex_buffer(clustering.positions), + .clusters = {}, + .textures = {}, + }; + result.clusters.reserve(clustering.clusters.size()); + for (const ::Cluster& cluster : clustering.clusters) { + if (cluster.texture_id == (std::numeric_limits::max)()) { + return Error::fail(Error::Code::InvalidInput, "DAG texture ID collides with the on-disk null sentinel"); + } + result.clusters.push_back({ + .triangle_count = cluster.local_triangles.size(), + .vertex_count = cluster.vertex_indices.size(), + .uv_count = cluster.uvs.size(), + .encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles), + .encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices), + .encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs), + .id = cluster.id, + .texture_id = cluster.texture_id.value_or((std::numeric_limits::max)()), + .absolute_error = cluster.absolute_error, + }); } - result.textures.push_back({ std::move(jpeg) }); + result.textures.reserve(clustering.textures.size()); + try { + for (const cv::Mat& texture : clustering.textures) { + auto jpeg = mesh::io::write_texture_to_encoded_buffer(texture, ".jpeg"); + if (!texture.empty() && jpeg.empty()) { + return Error::fail(Error::Code::InvalidInput, "could not encode DAG texture"); + } + result.textures.push_back({ std::move(jpeg) }); + } + } catch (const cv::Exception& error) { + return Error::fail(classify_texture_error(error, Error::Code::InvalidInput), "could not encode DAG texture: " + error.msg); + } + return result; } - } catch (const cv::Exception& error) { - return Error::fail( - classify_texture_error(error, Error::Code::InvalidInput), - "could not encode DAG texture: " + error.msg); - } - return result; -} - -inline bool count_fits(const std::uint64_t count, const std::size_t element_size) -{ - return count <= std::numeric_limits::max() / element_size && count * element_size <= io::envelope::default_max_decompressed_size; -} -inline Expected validate(const Clustering& clustering) -{ - if (!count_fits(clustering.vertex_count, sizeof(glm::dvec3))) { - return Error::fail(Error::Code::InvalidInput, "DAG position count exceeds the allocation limit"); - } - if (clustering.vertex_count != 0 && clustering.encoded_positions.empty()) { - return Error::fail(Error::Code::InvalidInput, "DAG position count and encoded data disagree"); - } - std::size_t decoded_size = static_cast(clustering.vertex_count) * sizeof(glm::dvec3); - for (const v1::Cluster& cluster : clustering.clusters) { - if (!count_fits(cluster.triangle_count, sizeof(glm::uvec3)) || !count_fits(cluster.vertex_count, sizeof(std::uint32_t)) - || !count_fits(cluster.uv_count, sizeof(glm::dvec2))) { - return Error::fail(Error::Code::InvalidInput, "DAG cluster count exceeds the allocation limit"); - } - const std::size_t cluster_size = static_cast(cluster.triangle_count) * sizeof(glm::uvec3) - + static_cast(cluster.vertex_count) * sizeof(std::uint32_t) + static_cast(cluster.uv_count) * sizeof(glm::dvec2); - if (cluster_size > io::envelope::default_max_decompressed_size - decoded_size) { - return Error::fail(Error::Code::InvalidInput, "DAG decoded data exceeds the allocation limit"); - } - decoded_size += cluster_size; - if (cluster.uv_count != 0 && cluster.uv_count != cluster.vertex_count) { - return Error::fail(Error::Code::InvalidInput, "DAG cluster UV count does not match its vertex count"); - } - if ((cluster.triangle_count != 0 && cluster.encoded_triangles.empty()) || (cluster.vertex_count != 0 && cluster.encoded_vertex_indices.empty()) - || (cluster.uv_count != 0 && cluster.encoded_uvs.empty())) { - return Error::fail(Error::Code::InvalidInput, "DAG cluster count and encoded data disagree"); - } - if (cluster.texture_id != (std::numeric_limits::max)() && cluster.texture_id >= clustering.textures.size()) { - return Error::fail(Error::Code::InvalidInput, "DAG cluster contains an invalid texture reference"); + inline bool count_fits(const std::uint64_t count, const std::size_t element_size) + { + return count <= std::numeric_limits::max() / element_size && count * element_size <= io::envelope::default_max_decompressed_size; } - if (!std::isfinite(cluster.absolute_error)) { - return Error::fail(Error::Code::InvalidInput, "DAG cluster contains a non-finite error"); - } - } - return {}; -} -template -inline Expected> decode_vertex_buffer( - const std::uint64_t count, const std::vector& encoded, const std::string_view label) -{ - if (!count_fits(count, sizeof(T))) { - return Error::fail(Error::Code::CorruptData, std::string(label) + " count exceeds the allocation limit"); - } - std::vector result(static_cast(count)); - if (meshopt_decodeVertexBuffer(result.data(), result.size(), sizeof(T), encoded.data(), encoded.size()) != 0) { - return Error::fail(Error::Code::CorruptData, "could not decode " + std::string(label)); - } - return result; -} - -inline Expected> decode_triangles(const std::uint64_t count, const std::vector& encoded) -{ - if (!count_fits(count, sizeof(glm::uvec3))) { - return Error::fail(Error::Code::CorruptData, "DAG triangle count exceeds the allocation limit"); - } - std::vector result(static_cast(count)); - if (meshopt_decodeIndexBuffer(result.data(), result.size() * 3, sizeof(std::uint32_t), encoded.data(), encoded.size()) != 0) { - return Error::fail(Error::Code::CorruptData, "could not decode DAG triangles"); - } - return result; -} - -inline Expected<::Clustering> decode_clustering(Clustering clustering) -{ - if (auto valid = validate(clustering); !valid) { - return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate decoded DAG clustering"); - } - auto positions = decode_vertex_buffer(clustering.vertex_count, clustering.encoded_positions, "DAG positions"); - if (!positions) { - return Error::propagate(std::move(positions), "decode DAG positions"); - } - - ::Clustering result; - result.positions = std::move(*positions); - result.clusters.reserve(clustering.clusters.size()); - for (const v1::Cluster& encoded : clustering.clusters) { - auto triangles = decode_triangles(encoded.triangle_count, encoded.encoded_triangles); - auto vertex_indices = decode_vertex_buffer(encoded.vertex_count, encoded.encoded_vertex_indices, "DAG vertex indices"); - auto uvs = decode_vertex_buffer(encoded.uv_count, encoded.encoded_uvs, "DAG UVs"); - if (!triangles) { - return Error::propagate(std::move(triangles), "decode DAG cluster triangles"); - } - if (!vertex_indices) { - return Error::propagate(std::move(vertex_indices), "decode DAG cluster vertex indices"); - } - if (!uvs) { - return Error::propagate(std::move(uvs), "decode DAG cluster UVs"); + inline Expected validate(const Clustering& clustering) + { + if (!count_fits(clustering.vertex_count, sizeof(glm::dvec3))) { + return Error::fail(Error::Code::InvalidInput, "DAG position count exceeds the allocation limit"); + } + if (clustering.vertex_count != 0 && clustering.encoded_positions.empty()) { + return Error::fail(Error::Code::InvalidInput, "DAG position count and encoded data disagree"); + } + std::size_t decoded_size = static_cast(clustering.vertex_count) * sizeof(glm::dvec3); + for (const v1::Cluster& cluster : clustering.clusters) { + if (!count_fits(cluster.triangle_count, sizeof(glm::uvec3)) || !count_fits(cluster.vertex_count, sizeof(std::uint32_t)) + || !count_fits(cluster.uv_count, sizeof(glm::dvec2))) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster count exceeds the allocation limit"); + } + const std::size_t cluster_size = static_cast(cluster.triangle_count) * sizeof(glm::uvec3) + + static_cast(cluster.vertex_count) * sizeof(std::uint32_t) + static_cast(cluster.uv_count) * sizeof(glm::dvec2); + if (cluster_size > io::envelope::default_max_decompressed_size - decoded_size) { + return Error::fail(Error::Code::InvalidInput, "DAG decoded data exceeds the allocation limit"); + } + decoded_size += cluster_size; + if (cluster.uv_count != 0 && cluster.uv_count != cluster.vertex_count) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster UV count does not match its vertex count"); + } + if ((cluster.triangle_count != 0 && cluster.encoded_triangles.empty()) || (cluster.vertex_count != 0 && cluster.encoded_vertex_indices.empty()) + || (cluster.uv_count != 0 && cluster.encoded_uvs.empty())) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster count and encoded data disagree"); + } + if (cluster.texture_id != (std::numeric_limits::max)() && cluster.texture_id >= clustering.textures.size()) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster contains an invalid texture reference"); + } + if (!std::isfinite(cluster.absolute_error)) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster contains a non-finite error"); + } + } + return {}; } - for (const std::uint32_t vertex : *vertex_indices) { - if (vertex >= result.positions.size()) { - return Error::fail(Error::Code::CorruptData, "DAG cluster contains an invalid global vertex index"); + + template + inline Expected> decode_vertex_buffer(const std::uint64_t count, const std::vector& encoded, const std::string_view label) + { + if (!count_fits(count, sizeof(T))) { + return Error::fail(Error::Code::CorruptData, std::string(label) + " count exceeds the allocation limit"); + } + std::vector result(static_cast(count)); + if (meshopt_decodeVertexBuffer(result.data(), result.size(), sizeof(T), encoded.data(), encoded.size()) != 0) { + return Error::fail(Error::Code::CorruptData, "could not decode " + std::string(label)); } + return result; } - for (const glm::uvec3 triangle : *triangles) { - if (glm::any(glm::greaterThanEqual(triangle, glm::uvec3(static_cast(vertex_indices->size()))))) { - return Error::fail(Error::Code::CorruptData, "DAG cluster contains an invalid local triangle index"); + + inline Expected> decode_triangles(const std::uint64_t count, const std::vector& encoded) + { + if (!count_fits(count, sizeof(glm::uvec3))) { + return Error::fail(Error::Code::CorruptData, "DAG triangle count exceeds the allocation limit"); + } + std::vector result(static_cast(count)); + if (meshopt_decodeIndexBuffer(result.data(), result.size() * 3, sizeof(std::uint32_t), encoded.data(), encoded.size()) != 0) { + return Error::fail(Error::Code::CorruptData, "could not decode DAG triangles"); } + return result; } - result.clusters.push_back({ - .id = encoded.id, - .vertex_indices = std::move(*vertex_indices), - .local_triangles = std::move(*triangles), - .uvs = std::move(*uvs), - .texture_id - = encoded.texture_id == (std::numeric_limits::max)() ? std::nullopt : std::optional { encoded.texture_id }, - .absolute_error = encoded.absolute_error, - }); - } - std::vector textures; - textures.reserve(clustering.textures.size()); - try { - for (const v1::Texture& texture : clustering.textures) { - cv::Mat image = mesh::io::read_texture_from_encoded_bytes(texture.jpeg); - if (image.empty()) { - return Error::fail(Error::Code::CorruptData, "could not decode DAG texture"); + inline Expected<::Clustering> decode_clustering(Clustering clustering) + { + if (auto valid = validate(clustering); !valid) { + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate decoded DAG clustering"); } - textures.push_back(std::move(image)); + auto positions = decode_vertex_buffer(clustering.vertex_count, clustering.encoded_positions, "DAG positions"); + if (!positions) { + return Error::propagate(std::move(positions), "decode DAG positions"); + } + + ::Clustering result; + result.positions = std::move(*positions); + result.clusters.reserve(clustering.clusters.size()); + for (const v1::Cluster& encoded : clustering.clusters) { + auto triangles = decode_triangles(encoded.triangle_count, encoded.encoded_triangles); + auto vertex_indices = decode_vertex_buffer(encoded.vertex_count, encoded.encoded_vertex_indices, "DAG vertex indices"); + auto uvs = decode_vertex_buffer(encoded.uv_count, encoded.encoded_uvs, "DAG UVs"); + if (!triangles) { + return Error::propagate(std::move(triangles), "decode DAG cluster triangles"); + } + if (!vertex_indices) { + return Error::propagate(std::move(vertex_indices), "decode DAG cluster vertex indices"); + } + if (!uvs) { + return Error::propagate(std::move(uvs), "decode DAG cluster UVs"); + } + for (const std::uint32_t vertex : *vertex_indices) { + if (vertex >= result.positions.size()) { + return Error::fail(Error::Code::CorruptData, "DAG cluster contains an invalid global vertex index"); + } + } + for (const glm::uvec3 triangle : *triangles) { + if (glm::any(glm::greaterThanEqual(triangle, glm::uvec3(static_cast(vertex_indices->size()))))) { + return Error::fail(Error::Code::CorruptData, "DAG cluster contains an invalid local triangle index"); + } + } + result.clusters.push_back({ + .id = encoded.id, + .vertex_indices = std::move(*vertex_indices), + .local_triangles = std::move(*triangles), + .uvs = std::move(*uvs), + .texture_id + = encoded.texture_id == (std::numeric_limits::max)() ? std::nullopt : std::optional { encoded.texture_id }, + .absolute_error = encoded.absolute_error, + }); + } + + std::vector textures; + textures.reserve(clustering.textures.size()); + try { + for (const v1::Texture& texture : clustering.textures) { + cv::Mat image = mesh::io::read_texture_from_encoded_bytes(texture.jpeg); + if (image.empty()) { + return Error::fail(Error::Code::CorruptData, "could not decode DAG texture"); + } + textures.push_back(std::move(image)); + } + } catch (const cv::Exception& error) { + return Error::fail(classify_texture_error(error, Error::Code::CorruptData), "could not decode DAG texture: " + error.msg); + } + result.textures = TextureSet(std::move(textures)); + return result; } - } catch (const cv::Exception& error) { - return Error::fail( - classify_texture_error(error, Error::Code::CorruptData), - "could not decode DAG texture: " + error.msg); - } - result.textures = TextureSet(std::move(textures)); - return result; -} -} // namespace format + } // namespace format -Expected validate_batch(const dag::ClusterBatch& batch, const Error::Code code) -{ - if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { - return Error::fail(code, "DAG metadata group-assignment count does not match clustering cluster count"); + Expected validate_batch(const dag::ClusterBatch& batch, const Error::Code code) + { + if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { + return Error::fail(code, "DAG metadata group-assignment count does not match clustering cluster count"); + } + return {}; } - return {}; -} -Expected read_metadata(const std::filesystem::path& path) -{ - auto payload = io::envelope::read_from_path(path); - if (!payload) { - return Error::propagate(std::move(payload), "read DAG metadata"); - } - auto metadata = format::decode_metadata(std::move(*payload)); - if (!metadata) { - return Error::propagate(std::move(metadata), "decode DAG metadata read from \"" + path.string() + "\""); + Expected read_metadata(const std::filesystem::path& path) + { + auto payload = io::envelope::read_from_path(path); + if (!payload) { + return Error::propagate(std::move(payload), "read DAG metadata"); + } + auto metadata = format::decode_metadata(std::move(*payload)); + if (!metadata) { + return Error::propagate(std::move(metadata), "decode DAG metadata read from \"" + path.string() + "\""); + } + return std::move(*metadata); } - return std::move(*metadata); -} } // namespace @@ -409,8 +404,7 @@ Expected ClusterBatch::read(const std::filesystem::path& node .clustering = std::move(*clustering), }; if (auto valid = validate_batch(batch, Error::Code::CorruptData); !valid) { - return Error::propagate(std::move(valid), - "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); + return Error::propagate(std::move(valid), "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); } return batch; } @@ -419,8 +413,7 @@ Expected ClusterBatch::write(const std::filesystem::path& node_path, const { const auto node_paths = paths(node_path); if (auto valid = validate_batch(batch, Error::Code::InvalidInput); !valid) { - return Error::propagate(std::move(valid), - "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); + return Error::propagate(std::move(valid), "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); } auto clustering = format::encode_clustering(batch.clustering); if (!clustering) { @@ -444,15 +437,9 @@ Expected ClusterBatch::write(const std::filesystem::path& node_path, const return {}; } -std::vector Metadata::paths(const std::filesystem::path& node_path) const -{ - return { add_extension(node_path, ".dagmeta") }; -} +std::vector Metadata::paths(const std::filesystem::path& node_path) const { return { add_extension(node_path, ".dagmeta") }; } -Expected Metadata::read(const std::filesystem::path& node_path) const -{ - return read_metadata(paths(node_path).front()); -} +Expected Metadata::read(const std::filesystem::path& node_path) const { return read_metadata(paths(node_path).front()); } Expected>> cluster_batch_from_extension(const std::string_view extension) { diff --git a/src/terrainlib/Error.cpp b/src/terrainlib/Error.cpp index 19d34121..e5acb8dd 100644 --- a/src/terrainlib/Error.cpp +++ b/src/terrainlib/Error.cpp @@ -29,10 +29,7 @@ std::string_view code_name(const Error::Code code) return "Unknown"; } -std::string describe_system_error(const std::error_code& cause) -{ - return fmt::format("{} ({}:{})", cause.message(), cause.category().name(), cause.value()); -} +std::string describe_system_error(const std::error_code& cause) { return fmt::format("{} ({}:{})", cause.message(), cause.category().name(), cause.value()); } } // namespace @@ -42,32 +39,20 @@ Error::Error(const Code code, Frame frame) m_frames.push_back(std::move(frame)); } -Error Error::make(const Code code, std::string message, const std::source_location location) -{ - return Error(code, Frame { std::move(message), location }); -} +Error Error::make(const Code code, std::string message, const std::source_location location) { return Error(code, Frame { std::move(message), location }); } -Error Error::make(const Code code, - const std::string_view operation, - const std::filesystem::path& path, - const std::source_location location) +Error Error::make(const Code code, const std::string_view operation, const std::filesystem::path& path, const std::source_location location) { return make(code, fmt::format("{} \"{}\"", operation, path.string()), location); } -Error Error::make(const Code code, - const std::string_view operation, - const std::error_code& cause, - const std::source_location location) +Error Error::make(const Code code, const std::string_view operation, const std::error_code& cause, const std::source_location location) { return make(code, fmt::format("{}: {}", operation, describe_system_error(cause)), location); } -Error Error::make(const Code code, - const std::string_view operation, - const std::filesystem::path& path, - const std::error_code& cause, - const std::source_location location) +Error Error::make( + const Code code, const std::string_view operation, const std::filesystem::path& path, const std::error_code& cause, const std::source_location location) { return make(code, fmt::format("{} \"{}\": {}", operation, path.string(), describe_system_error(cause)), location); } @@ -79,9 +64,7 @@ Error Error::make(const Code code, const std::error_code& cause, const std::source_location location) { - return make(code, - fmt::format("{} \"{}\" -> \"{}\": {}", operation, source.string(), destination.string(), describe_system_error(cause)), - location); + return make(code, fmt::format("{} \"{}\" -> \"{}\": {}", operation, source.string(), destination.string(), describe_system_error(cause)), location); } std::unexpected Error::fail(const Code code, std::string message, const std::source_location location) @@ -89,27 +72,18 @@ std::unexpected Error::fail(const Code code, std::string message, const s return std::unexpected { make(code, std::move(message), location) }; } -std::unexpected Error::fail(const Code code, - const std::string_view operation, - const std::filesystem::path& path, - const std::source_location location) +std::unexpected Error::fail(const Code code, const std::string_view operation, const std::filesystem::path& path, const std::source_location location) { return std::unexpected { make(code, operation, path, location) }; } -std::unexpected Error::fail(const Code code, - const std::string_view operation, - const std::error_code& cause, - const std::source_location location) +std::unexpected Error::fail(const Code code, const std::string_view operation, const std::error_code& cause, const std::source_location location) { return std::unexpected { make(code, operation, cause, location) }; } -std::unexpected Error::fail(const Code code, - const std::string_view operation, - const std::filesystem::path& path, - const std::error_code& cause, - const std::source_location location) +std::unexpected Error::fail( + const Code code, const std::string_view operation, const std::filesystem::path& path, const std::error_code& cause, const std::source_location location) { return std::unexpected { make(code, operation, path, cause, location) }; } diff --git a/src/terrainlib/Error.h b/src/terrainlib/Error.h index 4777d29b..a304f8da 100644 --- a/src/terrainlib/Error.h +++ b/src/terrainlib/Error.h @@ -35,17 +35,10 @@ class Error { std::source_location location; }; - static Error make(Code code, - std::string message, - std::source_location location = std::source_location::current()); - static Error make(Code code, - std::string_view operation, - const std::filesystem::path& path, - std::source_location location = std::source_location::current()); - static Error make(Code code, - std::string_view operation, - const std::error_code& cause, - std::source_location location = std::source_location::current()); + static Error make(Code code, std::string message, std::source_location location = std::source_location::current()); + static Error make( + Code code, std::string_view operation, const std::filesystem::path& path, std::source_location location = std::source_location::current()); + static Error make(Code code, std::string_view operation, const std::error_code& cause, std::source_location location = std::source_location::current()); static Error make(Code code, std::string_view operation, const std::filesystem::path& path, @@ -58,17 +51,11 @@ class Error { const std::error_code& cause, std::source_location location = std::source_location::current()); - static std::unexpected fail(Code code, - std::string message, - std::source_location location = std::source_location::current()); - static std::unexpected fail(Code code, - std::string_view operation, - const std::filesystem::path& path, - std::source_location location = std::source_location::current()); - static std::unexpected fail(Code code, - std::string_view operation, - const std::error_code& cause, - std::source_location location = std::source_location::current()); + static std::unexpected fail(Code code, std::string message, std::source_location location = std::source_location::current()); + static std::unexpected fail( + Code code, std::string_view operation, const std::filesystem::path& path, std::source_location location = std::source_location::current()); + static std::unexpected fail( + Code code, std::string_view operation, const std::error_code& cause, std::source_location location = std::source_location::current()); static std::unexpected fail(Code code, std::string_view operation, const std::filesystem::path& path, @@ -82,17 +69,12 @@ class Error { std::source_location location = std::source_location::current()); template - static std::unexpected propagate(std::expected&& result, - std::string message = "propagated", - std::source_location location = std::source_location::current()); + static std::unexpected propagate( + std::expected&& result, std::string message = "propagated", std::source_location location = std::source_location::current()); template - static std::unexpected propagate(std::expected&& result, - Code code, - std::string message, - std::source_location location = std::source_location::current()); - static std::unexpected propagate(Error&& error, - std::string message = "propagated", - std::source_location location = std::source_location::current()); + static std::unexpected propagate( + std::expected&& result, Code code, std::string message, std::source_location location = std::source_location::current()); + static std::unexpected propagate(Error&& error, std::string message = "propagated", std::source_location location = std::source_location::current()); Code code() const { return m_code; } std::string to_string() const; @@ -110,8 +92,7 @@ template using Expected = std::expected; template -std::unexpected Error::propagate( - std::expected&& result, std::string message, const std::source_location location) +std::unexpected Error::propagate(std::expected&& result, std::string message, const std::source_location location) { return std::unexpected { std::move(result).error().with_context(std::move(message), location), @@ -119,8 +100,7 @@ std::unexpected Error::propagate( } template -std::unexpected Error::propagate( - std::expected&& result, const Code code, std::string message, const std::source_location location) +std::unexpected Error::propagate(std::expected&& result, const Code code, std::string message, const std::source_location location) { return std::unexpected { std::move(result).error().reclassified(code, std::move(message), location), diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index 3343497d..ee2154fc 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -55,8 +55,7 @@ namespace { case ChecksumAlgorithm::Crc32c: break; default: - return Error::fail(Error::Code::Unsupported, - "unsupported checksum algorithm " + std::to_string(static_cast(checksum_algorithm))); + return Error::fail(Error::Code::Unsupported, "unsupported checksum algorithm " + std::to_string(static_cast(checksum_algorithm))); } switch (compression_algorithm) { @@ -65,8 +64,7 @@ namespace { case CompressionAlgorithm::ZstdDefaultCompressionWithChecksum: break; default: - return Error::fail(Error::Code::Unsupported, - "unsupported compression algorithm " + std::to_string(static_cast(compression_algorithm))); + return Error::fail(Error::Code::Unsupported, "unsupported compression algorithm " + std::to_string(static_cast(compression_algorithm))); } const bool no_compression = compression_algorithm == CompressionAlgorithm::None @@ -115,8 +113,7 @@ Expected compress_with_checksum( const std::size_t compressed_size = ZSTD_compress2(context.get(), compressed_data.data(), compressed_data.size(), uncompressed_data.data(), uncompressed_data.size()); if (ZSTD_isError(compressed_size)) { - return Error::fail(Error::Code::Internal, - std::string("Zstandard compression failed: ") + ZSTD_getErrorName(compressed_size)); + return Error::fail(Error::Code::Internal, std::string("Zstandard compression failed: ") + ZSTD_getErrorName(compressed_size)); } compressed_data.resize(compressed_size); @@ -181,8 +178,7 @@ Expected checked_decompress(const Bytes& compressed_data, if (!content_size_is_known && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { return Error::fail(Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit"); } - return Error::fail(Error::Code::CorruptData, - std::string("Zstandard decompression failed: ") + ZSTD_getErrorName(decompressed_size)); + return Error::fail(Error::Code::CorruptData, std::string("Zstandard decompression failed: ") + ZSTD_getErrorName(decompressed_size)); } if (content_size_is_known && decompressed_size != uncompressed_data.size()) { return Error::fail(Error::Code::CorruptData, "Zstandard decompressed size does not match its frame header"); diff --git a/src/terrainlib/io/conversion.h b/src/terrainlib/io/conversion.h index 578ba09f..620ba9f8 100644 --- a/src/terrainlib/io/conversion.h +++ b/src/terrainlib/io/conversion.h @@ -17,8 +17,7 @@ namespace io::conversion { namespace detail { - inline std::unexpected allocation_error( - const std::source_location location = std::source_location::current()) noexcept + inline std::unexpected allocation_error(const std::source_location location = std::source_location::current()) noexcept { return Error::fail(Error::Code::ResourceExhausted, "allocate raster conversion output", location); } @@ -142,13 +141,11 @@ Expected> to_raster(const cv::Mat& source) return Error::fail(Error::Code::InvalidInput, "cannot convert an empty OpenCV matrix to a raster"); } if (source.dims != 2) { - return Error::fail(Error::Code::Unsupported, - "unsupported OpenCV matrix dimension count " + std::to_string(source.dims)); + return Error::fail(Error::Code::Unsupported, "unsupported OpenCV matrix dimension count " + std::to_string(source.dims)); } if (source.type() != detail::cv_type) { return Error::fail(Error::Code::InvalidInput, - "OpenCV matrix type " + std::to_string(source.type()) + " does not match expected type " - + std::to_string(detail::cv_type)); + "OpenCV matrix type " + std::to_string(source.type()) + " does not match expected type " + std::to_string(detail::cv_type)); } if (source.cols < 0 || source.rows < 0 || static_cast(source.cols) > std::numeric_limits::max() || static_cast(source.rows) > std::numeric_limits::max()) { diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h index 8ce377b6..cf0e770e 100644 --- a/src/terrainlib/io/envelope.h +++ b/src/terrainlib/io/envelope.h @@ -125,8 +125,7 @@ Expected serialize(const typename Schema::latest_type& payload, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template -Expected deserialize( - std::span bytes, std::size_t max_decompressed_size = default_max_decompressed_size); +Expected deserialize(std::span bytes, std::size_t max_decompressed_size = default_max_decompressed_size); } // namespace io::envelope diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h index 37a88b5c..3c0de760 100644 --- a/src/terrainlib/mesh/codec/SfMesh.h +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -12,10 +12,7 @@ namespace mesh::codec { class SfMesh final : public store::Codec { public: - std::vector paths(const std::filesystem::path& node_path) const override - { - return { add_extension(node_path, ".sfmesh") }; - } + std::vector paths(const std::filesystem::path& node_path) const override { return { add_extension(node_path, ".sfmesh") }; } Expected read(const std::filesystem::path& node_path) const override { @@ -25,8 +22,7 @@ class SfMesh final : public store::Codec { return Error::propagate(std::move(payload), "read SF mesh"); } if (auto valid = mesh::sf::validate(*payload); !valid) { - return Error::propagate( - std::move(valid), Error::Code::CorruptData, "validate SF mesh read from \"" + path.string() + "\""); + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate SF mesh read from \"" + path.string() + "\""); } auto decoded = mesh::sf::decode_payload(std::move(*payload)); if (!decoded) { @@ -35,10 +31,7 @@ class SfMesh final : public store::Codec { return std::move(*decoded); } - Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override - { - return write(node_path, mesh, {}); - } + Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { return write(node_path, mesh, {}); } Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const { diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h index f724123b..c1288e73 100644 --- a/src/terrainlib/octree/storage/IndexFile.h +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -99,8 +99,7 @@ inline Expected> decode_index(const StoreIndex& encode for (const Id& child : *children) { auto child_status = result.get(child); if (!child_status) { - return Error::propagate( - std::move(child_status), Error::Code::CorruptData, "look up child while validating store index"); + return Error::propagate(std::move(child_status), Error::Code::CorruptData, "look up child while validating store index"); } has_child = has_child || child_status->has_value(); } @@ -118,8 +117,7 @@ inline Expected> decode_index(const StoreIndex& encode } auto parent_status = result.get(*parent); if (!parent_status) { - return Error::propagate( - std::move(parent_status), Error::Code::CorruptData, "look up parent while validating store index"); + return Error::propagate(std::move(parent_status), Error::Code::CorruptData, "look up parent while validating store index"); } if (!parent_status->has_value() || parent_status->value() == store::NodeStatus::Leaf) { return Error::fail(Error::Code::CorruptData, "index contains a node without a valid indexed parent"); @@ -136,8 +134,7 @@ inline Expected read_store_metadata(const std::filesystem::path& return metadata; } if (auto valid = validate(*metadata); !valid) { - return Error::propagate( - std::move(valid), Error::Code::CorruptData, "validate store metadata read from \"" + path.string() + "\""); + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate store metadata read from \"" + path.string() + "\""); } return std::move(*metadata); } diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h index 968bd6a6..f8cee12b 100644 --- a/src/terrainlib/octree/storage/open_runtime.h +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -32,8 +32,7 @@ Expected> open_folder(const std::filesyste return Error::fail(Error::Code::Io, "check existence of", base_path, error); } if (base_exists && !std::filesystem::is_directory(base_path, error)) { - return Error::fail( - Error::Code::Io, "check directory", base_path, error ? error : std::make_error_code(std::errc::not_a_directory)); + return Error::fail(Error::Code::Io, "check directory", base_path, error ? error : std::make_error_code(std::errc::not_a_directory)); } if (!base_exists) { std::filesystem::create_directories(base_path, error); @@ -55,14 +54,12 @@ Expected> open_folder(const std::filesyste } if (index_exists != metadata_exists) { return Error::fail(Error::Code::CorruptData, - "octree dataset metadata and index must either both exist or both be absent: " - + (index_exists ? metadata_path : index_path).string()); + "octree dataset metadata and index must either both exist or both be absent: " + (index_exists ? metadata_path : index_path).string()); } if (index_exists) { auto indexed = store::open_index(index_path, format, payload_class, resolve_codec); if (!indexed) { - return Error::propagate( - std::move(indexed), "open indexed octree storage at \"" + base_path.string() + "\""); + return Error::propagate(std::move(indexed), "open indexed octree storage at \"" + base_path.string() + "\""); } return store::Storage(std::move(*indexed)); } diff --git a/src/terrainlib/sf/validate_index.h b/src/terrainlib/sf/validate_index.h index b0d3ad8d..09cb4083 100644 --- a/src/terrainlib/sf/validate_index.h +++ b/src/terrainlib/sf/validate_index.h @@ -2,8 +2,8 @@ #include -#include "octree/StoreTraits.h" #include "Error.h" +#include "octree/StoreTraits.h" #include "store/Index.h" namespace sf { @@ -12,8 +12,7 @@ inline Expected validate_index(const store::Index& in { for (const auto& [key, status] : index) { if (status == store::NodeStatus::Inner) { - return Error::fail( - Error::Code::CorruptData, "Structura Fundamentalis topology contains Inner node " + key.to_string()); + return Error::fail(Error::Code::CorruptData, "Structura Fundamentalis topology contains Inner node " + key.to_string()); } } return {}; diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h index 7b8c01c3..56958e1a 100644 --- a/src/terrainlib/store/Codec.h +++ b/src/terrainlib/store/Codec.h @@ -17,10 +17,7 @@ class Codec { virtual std::vector paths(const std::filesystem::path& node_path) const = 0; - virtual Expected read(const std::filesystem::path&) const - { - return Error::fail(Error::Code::Unsupported, "codec does not support reading"); - } + virtual Expected read(const std::filesystem::path&) const { return Error::fail(Error::Code::Unsupported, "codec does not support reading"); } virtual Expected write(const std::filesystem::path&, const NodeData&) const { diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h index cedce8c1..22e90011 100644 --- a/src/terrainlib/store/RawStorage.h +++ b/src/terrainlib/store/RawStorage.h @@ -10,8 +10,8 @@ #include #include "store/Codec.h" -#include "store/path_layout.h" #include "store/Traits.h" +#include "store/path_layout.h" namespace store { @@ -155,8 +155,7 @@ class RawStorage { } std::filesystem::create_hard_link(source_paths[index], target_paths[index], error); if (error) { - return Error::fail( - Error::Code::Io, "create hard link", source_paths[index], target_paths[index], error); + return Error::fail(Error::Code::Io, "create hard link", source_paths[index], target_paths[index], error); } } return {}; diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h index c102a34e..2beb046e 100644 --- a/src/terrainlib/store/Storage.h +++ b/src/terrainlib/store/Storage.h @@ -86,8 +86,7 @@ class Storage { if (m_index.has_value()) { auto status = m_index->get(key); if (!status) { - return Error::propagate(std::move(status), - "look up node " + Traits::key_to_string(key) + " in storage index before loading"); + return Error::propagate(std::move(status), "look up node " + Traits::key_to_string(key) + " in storage index before loading"); } if (!status->has_value() || status->value() == NodeStatus::Virtual) { return Error::fail(Error::Code::NotFound, "node " + Traits::key_to_string(key) + " is not physically present in the index"); @@ -103,14 +102,11 @@ class Storage { } auto exists = has(key); if (!exists) { - return Error::propagate( - std::move(exists), "check whether node " + Traits::key_to_string(key) + " exists before saving"); + return Error::propagate(std::move(exists), "check whether node " + Traits::key_to_string(key) + " exists before saving"); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); - return Error::fail(Error::Code::AlreadyExists, - "save node to", - node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front()); + return Error::fail(Error::Code::AlreadyExists, "save node to", node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front()); } auto result = m_raw.save(key, data); @@ -120,8 +116,7 @@ class Storage { if (m_index.has_value()) { auto added = m_index->add(key); if (!added) { - return Error::propagate( - std::move(added), "add saved node " + Traits::key_to_string(key) + " to storage index"); + return Error::propagate(std::move(added), "add saved node " + Traits::key_to_string(key) + " to storage index"); } m_dirty = m_dirty || added.value(); } @@ -135,14 +130,11 @@ class Storage { } auto exists = has(key); if (!exists) { - return Error::propagate( - std::move(exists), "check whether target node " + Traits::key_to_string(key) + " exists before copying"); + return Error::propagate(std::move(exists), "check whether target node " + Traits::key_to_string(key) + " exists before copying"); } if (exists.value() && !m_settings.allow_overwrite) { const auto node_paths = m_raw.paths(key).value(); - return Error::fail(Error::Code::AlreadyExists, - "copy node to", - node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front()); + return Error::fail(Error::Code::AlreadyExists, "copy node to", node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front()); } const auto prepare_target = [&]() -> Expected { if (!exists.value() || !m_index.has_value()) { @@ -150,8 +142,7 @@ class Storage { } auto removed = m_index->remove(key); if (!removed) { - return Error::propagate( - std::move(removed), "remove overwritten node " + Traits::key_to_string(key) + " from storage index"); + return Error::propagate(std::move(removed), "remove overwritten node " + Traits::key_to_string(key) + " from storage index"); } m_dirty = m_dirty || removed.value(); return {}; @@ -164,8 +155,7 @@ class Storage { if (m_index.has_value()) { auto added = m_index->add(key); if (!added) { - return Error::propagate( - std::move(added), "add copied node " + Traits::key_to_string(key) + " to storage index"); + return Error::propagate(std::move(added), "add copied node " + Traits::key_to_string(key) + " to storage index"); } m_dirty = m_dirty || added.value(); } @@ -181,8 +171,7 @@ class Storage { if (m_index.has_value()) { auto index_removed = m_index->remove(key); if (!index_removed) { - return Error::propagate( - std::move(index_removed), "remove node " + Traits::key_to_string(key) + " from storage index"); + return Error::propagate(std::move(index_removed), "remove node " + Traits::key_to_string(key) + " from storage index"); } m_dirty = m_dirty || index_removed.value(); } @@ -199,8 +188,7 @@ class Storage { } auto status = m_index->get(key); if (!status) { - return Error::propagate( - std::move(status), "look up node " + Traits::key_to_string(key) + " in storage index"); + return Error::propagate(std::move(status), "look up node " + Traits::key_to_string(key) + " in storage index"); } return status->has_value() && status->value() != NodeStatus::Virtual; } diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h index b776d17e..00fc23ae 100644 --- a/src/terrainlib/store/Traits.h +++ b/src/terrainlib/store/Traits.h @@ -20,8 +20,7 @@ concept HierarchyTraits = requires(typename Traits::Key key) { }; template -std::unexpected invalid_key_error(const typename Traits::Key& key, - const std::source_location location = std::source_location::current()) +std::unexpected invalid_key_error(const typename Traits::Key& key, const std::source_location location = std::source_location::current()) { return Error::fail(Error::Code::InvalidInput, "invalid hierarchy key " + Traits::key_to_string(key), location); } diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h index d98eeef8..b491d06b 100644 --- a/src/terrainlib/store/traverse.h +++ b/src/terrainlib/store/traverse.h @@ -35,8 +35,7 @@ Expected traverse(const Index& index, } auto root_status = index.get(root); if (!root_status) { - return Error::propagate( - std::move(root_status), "read traversal root status for node " + Traits::key_to_string(root)); + return Error::propagate(std::move(root_status), "read traversal root status for node " + Traits::key_to_string(root)); } if (!root_status.value().has_value()) { return {}; @@ -47,8 +46,7 @@ Expected traverse(const Index& index, depth_first = [&](const Key& current) -> Expected { auto status = index.get(current); if (!status) { - return Error::propagate(std::move(status), - "read node status during depth-first traversal for " + Traits::key_to_string(current)); + return Error::propagate(std::move(status), "read node status during depth-first traversal for " + Traits::key_to_string(current)); } if (!status.value().has_value()) { return {}; @@ -79,8 +77,7 @@ Expected traverse(const Index& index, auto status = index.get(current); if (!status) { - return Error::propagate(std::move(status), - "read node status during breadth-first traversal for " + Traits::key_to_string(current)); + return Error::propagate(std::move(status), "read node status during breadth-first traversal for " + Traits::key_to_string(current)); } if (!status.value().has_value()) { continue; diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp index 8d80f397..4762b5ff 100644 --- a/unittests/dag_builder/storage_compatibility.cpp +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -3,6 +3,7 @@ #include +#include "../temporary_directory.h" #include "build.h" #include "codec.h" #include "dag_node.h" @@ -10,7 +11,6 @@ #include "mesh/storage.h" #include "storage.h" #include "store/ThreadSafeStorage.h" -#include "../temporary_directory.h" namespace { diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp index f75c370d..dbdc1a24 100644 --- a/unittests/sf_builder/storage_finalization.cpp +++ b/unittests/sf_builder/storage_finalization.cpp @@ -2,9 +2,9 @@ #include +#include "../temporary_directory.h" #include "mesh/storage.h" #include "sf/finalize_storage.h" -#include "../temporary_directory.h" namespace { diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp index bca78214..9f4f3a6a 100644 --- a/unittests/sf_merger/storage_boundaries.cpp +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -2,10 +2,10 @@ #include +#include "../temporary_directory.h" #include "cut.h" #include "merge.h" #include "mesh/storage.h" -#include "../temporary_directory.h" namespace { diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp index 8dbd4dfe..2dcc9b12 100644 --- a/unittests/terrainlib/store_codec.cpp +++ b/unittests/terrainlib/store_codec.cpp @@ -6,11 +6,11 @@ #include +#include "../temporary_directory.h" #include "io/bytes.h" #include "mesh/codec/Gltf.h" #include "mesh/codec/SfMesh.h" #include "store/Codec.h" -#include "../temporary_directory.h" namespace { @@ -26,10 +26,7 @@ class MultiFileCodec final : public store::Codec { class WriteOnlyCodec final : public store::Codec { public: - std::vector paths(const std::filesystem::path& node_path) const override - { - return { add_extension(node_path, ".out") }; - } + std::vector paths(const std::filesystem::path& node_path) const override { return { add_extension(node_path, ".out") }; } Expected write(const std::filesystem::path&, const int&) const override { ++writes; @@ -49,8 +46,8 @@ void check_mesh_codec_reentrancy(const Codec& codec, const std::filesystem::path { std::vector> writes; for (int index = 0; index < 4; ++index) { - writes.push_back(std::async(std::launch::async, - [&codec, base, index] { return codec.write(base / std::to_string(index) / "node", triangle_mesh(index)).has_value(); })); + writes.push_back(std::async( + std::launch::async, [&codec, base, index] { return codec.write(base / std::to_string(index) / "node", triangle_mesh(index)).has_value(); })); } for (auto& write : writes) { REQUIRE(write.get()); diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp index 7c75ca74..7579354e 100644 --- a/unittests/terrainlib/store_compatibility.cpp +++ b/unittests/terrainlib/store_compatibility.cpp @@ -7,6 +7,7 @@ #include +#include "../temporary_directory.h" #include "io/bytes.h" #include "mesh/SimpleMesh.h" #include "mesh/codec/SfMeshFormat.h" @@ -14,7 +15,6 @@ #include "mesh/storage.h" #include "octree/storage/IndexFile.h" #include "octree/store_layout/Mappings.h" -#include "../temporary_directory.h" namespace { diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp index dcfd1f14..a70c1638 100644 --- a/unittests/terrainlib/store_storage.cpp +++ b/unittests/terrainlib/store_storage.cpp @@ -8,13 +8,13 @@ #include #include +#include "../temporary_directory.h" #include "io/bytes.h" #include "octree/StoreTraits.h" #include "octree/store_layout/Mappings.h" #include "raster_store/StoreTraits.h" #include "store/IndexedStorage.h" #include "string_utils.h" -#include "../temporary_directory.h" namespace { @@ -84,17 +84,11 @@ Expected write_int(const int value, const std::filesystem::path& path) class IntCodec final : public store::Codec { public: - std::vector paths(const std::filesystem::path& node_path) const override - { - return { add_extension(node_path, ".data") }; - } + std::vector paths(const std::filesystem::path& node_path) const override { return { add_extension(node_path, ".data") }; } Expected read(const std::filesystem::path& node_path) const override { return read_int(paths(node_path).front()); } - Expected write(const std::filesystem::path& node_path, const int& value) const override - { - return write_int(value, paths(node_path).front()); - } + Expected write(const std::filesystem::path& node_path, const int& value) const override { return write_int(value, paths(node_path).front()); } }; template @@ -137,10 +131,7 @@ class ConfigurableCodec final : public store::Codec { { } - std::vector paths(const std::filesystem::path& node_path) const override - { - return { add_extension(node_path, extension) }; - } + std::vector paths(const std::filesystem::path& node_path) const override { return { add_extension(node_path, extension) }; } Expected read(const std::filesystem::path& node_path) const override {