diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..51fc0aa5 --- /dev/null +++ b/.clang-format @@ -0,0 +1,192 @@ +--- +Language: Cpp +# BasedOnStyle: WebKit +AccessModifierOffset: -4 +AlignAfterOpenBracket: DontAlign +AlignArrayOfStructures: None +AlignConsecutiveMacros: None +AlignConsecutiveAssignments: None +AlignConsecutiveBitFields: None +AlignConsecutiveDeclarations: None +AlignEscapedNewlines: Right +AlignOperands: DontAlign +AlignTrailingComments: false +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortEnumsOnASingleLine: true +AllowShortBlocksOnASingleLine: Empty +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: Yes +AttributeMacros: + - __capability +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: All +BreakBeforeConceptDeclarations: true +BreakBeforeBraces: WebKit +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeComma +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 160 +CommentPragmas: '^ IWYU pragma:' +QualifierAlignment: Leave +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +PackConstructorInitializers: BinPack +BasedOnStyle: '' +ConstructorInitializerAllOnOneLineOrOnePerLine: false +AllowAllConstructorInitializersOnNextLine: true +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseLabels: false +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock +IndentRequires: false +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +LambdaBodyIndentation: Signature +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: Inner +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 4 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PenaltyIndentedWhitespace: 0 +PointerAlignment: Left +PPIndentWidth: -1 +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + BeforeNonEmptyParentheses: false +SpaceAroundPointerQualifiers: Default +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: Never +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +BitFieldColonSpacing: Both +Standard: Latest +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Never +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE + - NS_SWIFT_NAME + - CF_SWIFT_NAME +... + diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..ebcaef91 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,7 @@ +# Agent instructions + +This is the alpine-terrain-builder project. + +Follow the project [code style](docs/code-style.md). + +Before committing, run `git ls-files --eol --cached --others --exclude-standard` and verify that each changed text file's working-tree line endings (`w/...`) agree with the line endings required by its Git attributes (`attr/...`). Fix unintended mixed or mismatched line endings before committing. diff --git a/README.md b/README.md index 011b8cd6..3cbf664e 100644 --- a/README.md +++ b/README.md @@ -1,94 +1,3 @@ # alpine-terrain-builder -## Overview - -This repository contains the following tools: - -1. [tile_downloader](src/tile_downloader) for downloading orthophoto tiles, which are used for texturing the generated meshes. -2. [terrainbuilder](src/terrainbuilder/) for building the highest quality meshes from the downloaded orthophoto tiles and a heightmap dataset. -3. [terrainmerger](src/terrainmerger/) for building a LOD hierarchy from the highest quality meshes by merging them upwards. -4. [terrainconvert](src/terrainconvert/) for converting .tile files into .glb files. - -### Core Library -[terrainlib](src/terrainlib/) contains classes common to all the tools above. - -### Scripts -The tools typically only handle one tile per command which makes it infeasible to run them manually for a whole region due to the sheer amount of tiles. Therefore, we've created the following python scripts, which automate calling the relevant tools for a specified regions: - -1. [batch-builder.py](batch-builder.py) calls [terrainbuilder](src/terrainbuilder/) for all tiles in the specified region. -2. [batch-merger.py](batch-merger.py) calls [terrainmerger](src/terrainmerger/) for all tiles in the specified region. -3. [batch-converter.py](batch-converter.py) calls [terrainconvert](src/terrainconvert/) for all .tile files in the specified directory and its subdirectories. - -## Example Workflow - -In this example, we'll show how to build the hierarchy for Vienna's city center (Zoom: 13, X: 4468, Y:2840). - -### 1. Downloading tiles -Tile coordinates always use the Google/Mapbox/XYZ convention internally: the origin is north-west, X grows east, and Y grows south. Google Maps, Mapbox, OpenStreetMap, and most XYZ services use the common URL order `{zoom}/{x}/{y}`. - -The `basemap` and `gataki` providers select their complete URL pattern and Y direction automatically. Both currently use downward Y with the URL order `{zoom}/{y}/{x}`. The `basemap` provider downloads the basemap.at orthophoto, and the Gataki mirror uses: - -`https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/{zoom}/{y}/{x}.jpeg` - -Example for the root tile: https://gataki.cg.tuwien.ac.at/raw/basemap/tiles/13/2840/4468.jpeg - -For another service, pass a quoted `--url` pattern containing `{zoom}`, `{x}`, and `{y}`. Placeholder placement selects URL coordinate order. Custom URLs default to downward Y; use `--url-y-direction up` for legacy TMS. Downloaded files always use the common Google/Mapbox layout `{zoom}/{x}/{y}.jpeg`, independently of the remote URL. - -The following command downloads the mirror's `{zoom}/{y}/{x}` URLs and writes the root tile to `./tiles/13/4468/2840.jpeg`: - -``` -./tile-downloader --provider gataki --zoom 13 --x 4468 --y 2840 --max-zoom-level 19 -``` - -For example, a custom Google/Mapbox URL can be selected with `--url 'https://example.test/{zoom}/{x}/{y}.jpeg'`. - -### 2. Download heightmap dataset -The meshes are built from a heightmap dataset, therefore we need to download one. For this example we'll only use a small part of the complete dataset for the whole of austria (available at https://gataki.cg.tuwien.ac.at/raw/Oe_2020/, 268 GB to 1.1 TB). The part we're gonna use contains Vienna's city center and it is available at https://gataki.cg.tuwien.ac.at/raw/vienna/innenstadt_gs_1m_mgi.tif (228 MB). - -![Vienna city center heightmap](readme_img/innenstadt_gs_1m_mgi.jpg) - -### 3. Building reference meshes -The following command will build reference meshes for the root tile `13/4468/2840` at the target zoom level `19` from the tile textures in `./tiles/` and the dataset `./innenstadt_gs_1m_mgi.tif` and output them in the folder `./meshes`. The SRS used is `EPSG:3857`, as the [demo renderer](https://github.com/polskus/renderer-alpinite) expects coordinates in the WebMercator (EPSG:3857) system. - -``` -python3 batch-builder.py --terrainbuilder ./terrainbuilder --tile 13 4468 2840 --target-zoom 19 --output-dir ./meshes -- --textures ./tiles/ --dataset ./innenstadt_gs_1m_mgi.tif --mesh-srs EPSG:3857 -``` - -### 4. Building tile hierarchy by merging and simplifying reference meshes -The following command will build the LOD hierarchy from the reference meshes generated in the previous step at `./meshes` and output simplified GLB/GLTF files at `./hierarchy/{zoom}/{Y}/{X}.simplified.glb`. - -> [!WARNING] -> This process is very time intensive and may end up taking hours depending on the region size! - -``` -python3 batch-merger.py --terrainmerger ./terrainmerger --root-tile 13 4468 2840 --max-zoom 19 --input-dir ./meshes --output-dir ./hierarchy -``` - -### 5. Converting missing meshes from .TILE to .GLB -Due to the separation of concerns in step `3. Building reference meshes` the tilebuilder does not output .GLB files, and in step `4. Building tile hierarchy by merging and simplifying reference meshes` the terrainmerger only outputs simplified GLB files for the next higher level. Therefore, there are no GLB files present for the highest zoom level `19`. Fortunately, there is the terrainconvert tool, which enables the conversion between .TILE and .GLB files without simplification. -The following command runs a script, which checks if there is a `*.simplified.glb` file for every `*.tile` present. - -> [!NOTE] -> Even though we name the converted GLB files using the suffix `.simplified.glb`, there is no simplification happening during the conversion. This naming is only used for consistency's sake, as the terrainmerger creates GLB files with this suffix and the [demo renderer](https://github.com/polskus/renderer-alpinite) expects a consistent naming scheme. You could also, in theory, rename the whole hierarchy to just use `*.glb`, make the relevant change in the [demo renderer](https://github.com/polskus/renderer-alpinite) and it would work too. - -``` -python3 batch-converter.py --terrainconverter ./terrainconvert --dir ./hierarchy/19 --suffix .simplified -``` - -### 6. Deployment -To make the tiles accessible to the [demo renderer](https://github.com/polskus/renderer-alpinite) just serve the final hierarchy via a http(s) server, which the computer running the [demo renderer](https://github.com/polskus/renderer-alpinite) can access. The default URL format used by the [demo renderer](https://github.com/polskus/renderer-alpinite) is `/{zoom}/{Y}/{X}.simplified.glb`. - -See [Modifying the tile server URL](https://github.com/polskus/renderer-alpinite/blob/main/README.md#modifying-the-tile-server-url) for information on how to change the server URL, file structure and suffix. - -## building -In order to build, you need to install: - -- fmt (https://fmt.dev/latest/index.html) -- GDAL 3 (https://gdal.org/) -- glm (https://github.com/g-truc/glm) -- catch2 (version 2.x) (https://github.com/catchorg/Catch2/tree/v2.x) -- FreeImage -- tbb (intel threading building blocks) - -sudo apt-get install libcgal-dev libopencv-dev libfmt-dev libglm-dev libgdal-dev catch2 libfreeimage-dev libtbb-dev libcurl4-openssl-dev -(libgmp-dev libmpfr-dev libsqlite3-dev) +See the project [code style](docs/code-style.md) before contributing. diff --git a/cmake/SetupGDAL.cmake b/cmake/SetupGDAL.cmake index fdc85178..e08315b9 100644 --- a/cmake/SetupGDAL.cmake +++ b/cmake/SetupGDAL.cmake @@ -39,7 +39,7 @@ function(alp_setup_gdal) -DBUILD_TESTING=OFF -DGEOS_BUILD_DEVELOPER=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON - ) + ) alp_setup_cmake_project(gdal URL https://github.com/OSGeo/gdal.git diff --git a/docs/code-style.md b/docs/code-style.md new file mode 100644 index 00000000..bed47211 --- /dev/null +++ b/docs/code-style.md @@ -0,0 +1,122 @@ +# Code style + +## Core code style + +- Class names use CamelCase. Method, function, and variable names use snake_case. +- Class data members have an `m_` prefix and are usually private. Struct data members have no prefix and are usually public. +- Use `void set_attribute(int value)` and `int attribute() const` for setters and getters; avoid a `get_` prefix. Follow the [Qt recommendations](https://wiki.qt.io/API_Design_Principles#Naming_Boolean_Getters,_Setters,_and_Properties) for boolean getters. +- Structs are usually small and simple, have few or no methods, and never use inheritance. +- Files use CamelCase when they contain a CamelCase class. Otherwise, files use snake_case and their contents belong to a matching snake_case namespace. +- Reflect the directory and file structure in namespaces. For example, `folder/structure.h` corresponds to `namespace folder::structure { ... }`. +- Indent with four spaces. Do not use tabs. +- Format new code, and only new code, with the repository's [`.clang-format`](../.clang-format) file. Do not reformat unrelated existing code. +- Follow the [Qt API design principles](https://wiki.qt.io/API_Design_Principles) and the [C++ Core Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines) where this guide does not specify a rule. + +## Avoid repeated type names + +Avoid repeating the same name across namespaces and types. For example, `io::envelope::Envelope` is undesirable. A supporting type such as `io::envelope::Version` reads well, but placing the main class in that namespace forces its longer, repetitive name. + +Prefer `io::Envelope`. Put supporting types either inside `io::Envelope` as public types or in an `io::envelope_details` namespace. + +## Pointers vs. references + +Avoid out parameters in new APIs when practical; prefer returning a small struct. When an out parameter is necessary, use a pointer rather than a non-const reference. The address-of operator at the call site makes it clear that the argument can be modified: + +```cpp +color.get_hsv(&hue, &saturation, &value); +``` + +## The art of naming + +Names are a central part of API design. Avoid abbreviations, even familiar ones, because consistent full words are easier to understand and remember. Choose precise names that preserve useful context without adding redundant words. If a concept is difficult to name or describe in one sentence, reconsider whether it belongs in the API. + +Use consistent names for related classes and operations. A function name should make clear whether the function has side effects. Give parameters descriptive names and use the same names in declarations, definitions, and documentation. Avoid repeating an enum type's name in scoped enumerators when the scope already provides the context. + +## Error handling + +Functions that can fail return `Expected`, which is an alias for `std::expected`. + +### Checking results + +Use the short boolean form in control flow: + +```cpp +if (!result) { + // Handle error. +} + +if (result) { + // Handle value. +} +``` + +### Creating errors + +Use `Error::fail` where an error originates instead of `std::unexpected`: + +```cpp +return Error::fail( + Error::Code::InvalidInput, + "invalid storage layout"); +``` + +Choose the most specific error code available. + +### Forwarding errors + +Generally, errors can be forwarded with `Error::propagate`: + +```cpp +auto result = operation(); // Expected +if (!result) { + // result can be either a std::expected or an Error. + return Error::propagate(std::move(result)); +} +// The current function returns Expected. +``` + +`propagate` adds the source file and line as additional context. In some cases, it can be useful to avoid that: + +```cpp +auto result = operation(); +if (!result) { + return result; +} +``` + +This is appropriate for transparent helpers, recursive calls, and simple delegation where another error frame would add no useful information. When the success types differ and direct return is impossible, use `propagate`. + +### Adding context + +At meaningful abstraction or operation boundaries, propagate with a concise context message: + +```cpp +auto result = write_payload(path); +if (!result) { + // result can be either a std::expected or an Error. + return Error::propagate(std::move(result), "write DAG metadata"); +} +``` + +Useful context identifies what the caller was trying to accomplish, such as: + +- The domain operation: `"gather relevant input leaves"`. +- The data role: `"read DAG clustering"`. +- The affected node: `"add saved node " + key + " to storage index"`. +- The higher-level phase: `"finalize merged output dataset"`. + +Avoid merely repeating details already supplied by the lower-level error. For example, if the lower layer records the exact path and I/O operation, the caller should add the file's semantic role rather than repeat the path unnecessarily. + +The error type can be changed (reclassified) as well: + +```cpp +auto result = write_payload(path); +if (!result) { + return Error::propagate( + std::move(result), + Error::Code::CorruptData, + "write DAG metadata"); +} +``` + +Reclassify an existing error only if the current abstraction genuinely changes its meaning. diff --git a/docs/raster-store/README.md b/docs/raster-store/README.md index 04429875..e78c6fd0 100644 --- a/docs/raster-store/README.md +++ b/docs/raster-store/README.md @@ -11,6 +11,7 @@ baseline, not a finalized binary-format specification. - [Architecture](architecture.md) - [Storage format](storage-format.md) - [Sampling and pyramid generation](sampling-and-generation.md) +- [Implemented refactor status](refactor-status.md) ## Plans diff --git a/docs/raster-store/architecture.md b/docs/raster-store/architecture.md index 204c7ac7..392c02e9 100644 --- a/docs/raster-store/architecture.md +++ b/docs/raster-store/architecture.md @@ -1,5 +1,64 @@ # Architecture +## Implementation status + +This document describes the intended raster-fundamentalis and tile-base +architecture. Its RF format, snapshot-publication, builder, merger, pyramid, +and server requirements are future work; they were not acceptance criteria +for the completed 2D/3D hierarchy-store refactor. The implemented boundary is +recorded in [refactor-status.md](refactor-status.md), and the original plan is +kept in [refactor-plan.md](refactor-plan.md). + +The refactor delivered the dimension-neutral mechanisms needed by that future +work: + +- `store::Index`, `store::traverse`, layouts, runtime codecs, storage, + typed errors, and cache interfaces; +- `octree::StoreTraits` plus legacy 3D layout/index/open adapters; +- `raster_store::StoreTraits` for in-memory topology keyed by + `radix::tile::Id`; and +- SF-only topology validation under `sf`. + +It deliberately did not define a persistent RF index, RF payload codec, RF +opening API, `rf_builder`, `rf_merger`, tile-base generator, or tile server. + +### Final public names + +The shared API uses `store::NodeStatus`, `store::NodeStatusOrMissing`, +`store::Index`, `store::traverse`, `store::RawStorage`, +`store::Storage`, and `store::IndexedStorage`. Mesh storage aliases live under +`mesh::storage`; DAG batch and metadata aliases live under `dag::storage`. +The `octree` namespace retains the 3D key, traits, path mappings, legacy index +DTO, and mesh-opening compatibility functions. + +A legacy 3D mesh dataset is opened through the 3D adapter: + +```cpp +#include "mesh/storage.h" + +auto opened = mesh::storage::open_folder_indexed(dataset_path); +if (!opened.has_value()) { + return std::unexpected(opened.error()); +} +mesh::storage::IndexedStorage storage = std::move(opened.value()); +``` + +The 2D traits adapter can exercise the shared topology without implying a +persistent RF format: + +```cpp +#include "raster_store/StoreTraits.h" +#include "store/Index.h" +#include "store/traverse.h" + +store::Index index; +const radix::tile::Id tile{2, {1, 3}}; +auto added = index.add(tile); +auto walked = store::traverse(index, [](const auto &id, store::NodeStatus status) { + // In-memory hierarchy processing only. +}); +``` + ## System boundary The design separates authoritative data management from delivery generation: diff --git a/docs/raster-store/before-refactor/README.md b/docs/raster-store/before-refactor/README.md deleted file mode 100644 index f9d3d9f0..00000000 --- a/docs/raster-store/before-refactor/README.md +++ /dev/null @@ -1,433 +0,0 @@ -# SF storage and merge architecture before the 2D/3D refactor - -This report describes the existing Structura Fundamentalis (SF) builder, -sparse octree index, storage stack, and mask-based dataset merger. Its purpose -is to identify the mechanisms that can be generalised from 3D octrees to a -shared 2D/3D hierarchy without carrying mesh- and ECEF-specific behaviour into -the common storage layer. - -The central conclusion is: - -> Generalise the sparse hierarchy, storage, traversal, merge decisions, and -> unchanged-subtree reuse. Keep GDAL ingestion, raster processing, mesh -> construction, spatial transforms, and mask geometry in dimension-specific -> adapters. - -The current storage templates are already payload-generic, but they are not -hierarchy-generic: `octree::Id` is embedded throughout indexing, traversal, -caches, disk paths, and serialization. - -## Current architecture - -```mermaid -flowchart TB - subgraph Applications - SFB[sf-builder] - SFM[sf-merger] - end - - subgraph Input - GDALR[GDAL raster datasource] - GDALV[GDAL vector mask] - OLD[Base SF dataset] - NEW[New SF dataset] - end - - subgraph Domain3D["3D and mesh policy"] - SPACE[octree::Space and ECEF bounds] - MESHBUILD[Height raster to SimpleMesh] - MESHMASK[Polygon to spherical extruded MeshMask] - MESHMERGE[Clip, combine, and texture atlas] - FALLBACK[Reconstruct child by clipping ancestor mesh] - end - - subgraph GenericCandidate["Mostly generalisable mechanisms"] - INDEX[IndexMap and NodeStatus] - WALK[Depth-first and breadth-first traversal] - DRIVER[Merger and merge result actions] - STORAGE[Storage and IndexedStorage] - RAW[RawStorage] - CODEC[Codec policy] - LAYOUT[Layout strategy] - LINK[Hard-link unchanged payload] - end - - GDALR --> SFB - SFB --> SPACE - SFB --> MESHBUILD - MESHBUILD --> STORAGE - - OLD --> SFM - NEW --> SFM - GDALV --> MESHMASK - MESHMASK --> SFM - SFM --> DRIVER - DRIVER --> FALLBACK - DRIVER --> MESHMERGE - DRIVER --> STORAGE - - STORAGE --> INDEX - STORAGE --> RAW - RAW --> CODEC - RAW --> LAYOUT - RAW --> LINK - WALK --> INDEX - DRIVER --> WALK -``` - -The significant code boundaries are: - -- hierarchy identity: [`octree::Id`](../../../src/terrainlib/octree/Id.h); -- sparse topology: [`IndexMap`](../../../src/terrainlib/octree/IndexMap.h); -- traversal: [`octree::traverse`](../../../src/terrainlib/octree/traverse.h); -- logical storage: - [`Storage_`](../../../src/terrainlib/octree/storage/Storage.h); -- raw file and hard-link storage: - [`RawStorage_`](../../../src/terrainlib/octree/storage/RawStorage.h); -- disk paths: [`disk::Layout`](../../../src/terrainlib/octree/disk/Layout.h); -- merge driver: [`Merger`](../../../src/sf_merger/merge.h). - -## Sparse index model - -The index has four logical states: - -| State | Physical payload | Indexed descendants | -|---|---:|---:| -| `Missing` | no | no | -| `Virtual` | no | yes | -| `Leaf` | yes | no | -| `Inner` | yes | yes | - -`Missing` is represented by absence from `IndexMap`. The other states are -defined by -[`NodeStatus`](../../../src/terrainlib/octree/NodeStatus.h). - -```mermaid -stateDiagram-v2 - [*] --> Missing - - Missing --> Leaf: add physical root or node - Missing --> Virtual: add deeper descendant - - Virtual --> Inner: add physical payload here - Leaf --> Inner: add physical descendant - - Inner --> Leaf: remove final descendant - Leaf --> Missing: remove payload and no descendants - Virtual --> Missing: remove final descendant - Inner --> Virtual: remove payload but keep descendants -``` - -This state machine is dimension-independent. What is currently 3D-specific is -how a node finds its parent and children: `octree::Id` uses three -Morton-interleaved coordinates and eight children. - -Traversal is also reusable in concept. It: - -1. looks up the root in the sparse index; -2. visits only entries that exist; -3. enumerates children through the concrete ID type; and -4. supports depth-first or breadth-first order plus a refinement predicate. - -The only reason -[`traverse()`](../../../src/terrainlib/octree/traverse.h) -is 3D is its dependency on `octree::Id` and `octree::Id::children()`. - -## Creating an SF dataset from a GDAL datasource - -The current SF builder creates physical mesh nodes at one requested octree -level. - -```mermaid -sequenceDiagram - participant CLI as sf-builder CLI - participant DS as Dataset and GDAL - participant Space as octree::Space - participant Build as terrainbuilder - participant Reader as RawDatasetReader - participant Raster as Raster and RasterMask - participant Mesh as mesh operations - participant Store as Storage - participant Index as IndexMap and terrain.index - - CLI->>DS: Open raster datasource - DS-->>CLI: SRS, 2D bounds, and height range - - CLI->>Space: Transform bounds to ECEF - Space-->>CLI: Smallest enclosing octree node - - loop Intersecting children until target level - CLI->>Space: Node bounds and intersection tests - end - - loop Each target-level node - CLI->>Build: build_patch using node bounds - Build->>Reader: Map SRS bounds to source pixels - Reader->>DS: RasterIO band 1 as float - Reader-->>Raster: Height raster - Raster->>Raster: Build NoData validity mask - Raster->>Mesh: Generate positions and triangle grid - Mesh->>Mesh: Clip to node volume - Mesh->>Mesh: Generate UVs and transform output SRS - Mesh-->>Store: SimpleMesh with optional texture - Store->>Store: MeshCodec writes node file - end - - Store->>Index: Scan output paths - Index->>Index: Add leaves and virtual ancestors - Index->>Store: Write terrain.index -``` - -### Current components - -| Responsibility | Current component | -|---|---| -| Own and open a GDAL datasource | [`Dataset`](../../../src/terrainlib/Dataset.h) | -| One-time GDAL registration | [`initialize_gdal_once()`](../../../src/terrainlib/init.cpp) | -| ECEF octree geometry | [`octree::Space`](../../../src/terrainlib/octree/Space.h) | -| Batch enumeration and orchestration | [`build_all_patches()`](../../../src/sf_builder/terrainbuilder.cpp) | -| Direct source-window reading | [`RawDatasetReader`](../../../src/sf_builder/raw_dataset_reader.h) | -| Height-to-mesh conversion | [`build_reference_mesh_patch()`](../../../src/sf_builder/mesh_builder.cpp) | -| Optional imagery | [`TileProvider`](../../../src/sf_builder/tile_provider.h) and [`texture_assembler.h`](../../../src/sf_builder/texture_assembler.h) | -| Mesh persistence | `Storage_` | -| Sparse topology | `IndexMap` | -| Index creation and serialization | `save_or_create_index()` and `terrain.index` | - -For a new output, `build_all_patches()` opens unindexed storage. Individual -saves therefore do not build the index incrementally. The final -`save_or_create_index()` recursively scans the output directory, parses node -paths, and calls `IndexMap::add()`. - -### More suitable 2D GDAL path - -The separate -[`DatasetReader`](../../../src/tile_builder/DatasetReader.h) -is closer to what a 2D SF or raster-store builder needs: - -- it accepts requested target-SRS bounds and output dimensions; -- it creates a GDAL warped VRT; -- it reprojects into the target grid; and -- it returns `radix::Raster`. - -[`Tiler`](../../../src/tile_builder/Tiler.h) and -[`ParallelTiler`](../../../src/tile_builder/ParallelTiler.h) -already enumerate `radix::tile::Id` values over dataset bounds. - -Their grid calculations are useful, but `ParallelTileGenerator` is a -delivery-file writer rather than an SF snapshot builder. - -## Mask-based SF dataset merging - -```mermaid -flowchart TD - START[Open base and new datasets as IndexedStorage] - MASK[Read vector mask through GDAL and OGR] - PREP[Polygon repair, sphere projection, triangulation, radial extrusion] - ROOT[Start at octree root] - STATUS[Read left and right NodeStatus] - POLICY[Masked visitor evaluates node] - RECURSE[Clip mask to node bounds and recurse into 8 children] - LEFT[Keep base subtree] - RIGHT[Keep new subtree] - MERGE[Clip base outside mask and new inside mask] - ATLAS[Combine meshes and rebuild texture atlas] - WRITE[Write new physical node] - COPY[Traverse unchanged source subtree] - LINK[Hard-link each physical payload] - INDEX[Scan and save new output index] - - START --> ROOT - MASK --> PREP --> POLICY - ROOT --> STATUS --> POLICY - - POLICY -->|virtual or refinement required| RECURSE - RECURSE --> STATUS - - POLICY -->|unchanged left| LEFT --> COPY - POLICY -->|unchanged right| RIGHT --> COPY - COPY --> LINK - - POLICY -->|boundary node| MERGE --> ATLAS --> WRITE - POLICY -->|no retained data| INDEX - - LINK --> INDEX - WRITE --> INDEX -``` - -### Merge components - -- [`Merger`](../../../src/sf_merger/merge.h) is the recursive dispatcher. -- [`NodeLoader`](../../../src/sf_merger/NodeLoader.h) reports status and loads - payloads. -- When an exact payload is missing, `NodeLoader` searches physical ancestors - and reconstructs the requested child by clipping the ancestor mesh. That - reconstruction is 3D-specific. -- [`NodeData`](../../../src/sf_merger/merge/NodeData.h) lazily exposes a node - payload to a visitor. -- [`Result`](../../../src/sf_merger/merge/Result.h) gives the driver four - actions: recurse, ignore, preserve one source unchanged, or write a merged - payload. -- [`Masked`](../../../src/sf_merger/merge/visitor/Masked.h) implements - mesh-and-mask policy. -- [`NodeWriter`](../../../src/sf_merger/NodeWriter.h) writes changed nodes or - copies unchanged subtrees. - -The vector-mask pipeline in -[`mask.h`](../../../src/sf_merger/mask.h) -is almost entirely 3D and Earth-specific after OGR polygon loading: - -```text -OGR polygons - -> referenced 2D polygon mask - -> ECEF and spherical projection - -> triangulated surface - -> extrusion over an Earth-radius interval - -> closed 3D MeshMask -``` - -For 2D raster merging, only the OGR polygon loading and CRS transformation -ideas remain relevant. Triangulation, spherical projection, extrusion, 3D -clipping, and texture atlases should not enter the generic core. - -## Hard-link behaviour - -Hard links are implemented at the lowest storage layer in -[`RawStorage_::copy_from()`](../../../src/terrainlib/octree/storage/RawStorage.h): - -```mermaid -flowchart TD - COPY[copy_from node] - EXISTS{Source payload exists?} - EXT{Source and target extensions match?} - DECODE[Decode source payload] - ENCODE[Encode into target format] - REMOVE[Remove existing target] - DIRS[Create parent directories] - HARDLINK[Create hard link] - DONE[Add physical node to target index] - - COPY --> EXISTS - EXISTS -->|no| ERROR1[FileNotFound] - EXISTS -->|yes| EXT - EXT -->|no| DECODE --> ENCODE --> DONE - EXT -->|yes| REMOVE --> DIRS --> HARDLINK --> DONE -``` - -Properties: - -- Hard linking is payload-agnostic and fully generalisable. -- It happens only when source and target filename extensions match. -- Different formats cause decode and re-encode through the codec. -- There is no copy fallback if hard-link creation fails, including across - filesystems. -- Snapshot immutability is not enforced by `Storage_`; it is a caller-level - convention. -- The existing target file is removed before the link is created. -- The new output receives a fresh index; `terrain.index` itself is not linked. - -## Generalisation assessment - -| Mechanism | Assessment | Required change | -|---|---|---| -| `NodeStatus` state model | Reuse essentially unchanged | Move outside `octree` naming | -| `IndexMap` state transitions | Generalise | Template over node key and tree traits | -| Sparse traversal | Generalise | Obtain children through tree traits | -| `Storage_` | Good starting point | Also template over node key and layout | -| `RawStorage_` hard-link behaviour | Generalise | Key-neutral paths and explicit fallback policy | -| Codec concept | Reuse | No dimensional dependency | -| Cache interface | Generalise | Key type is currently `octree::Id` | -| Layout strategy | Generalise | Key-neutral path API | -| Index file | Replace or version | Record topology kind, format version, and validated key | -| Merge result algebra | Generalise | `Merged` must hold generic payload, not `SimpleMesh` | -| Recursive merge driver | Generalise | Tree traits, payload, loader/writer, and policy | -| Unchanged-subtree reuse | Reuse | Enumerate every physical state, including `Inner` | -| GDAL datasource ownership | Reuse or adapt | Typed, multi-band reads and explicit NoData | -| Mask loading through OGR | Reuse or adapt | Produce dimension-specific mask representation | -| `octree::Space` | Keep 3D | Add a sibling 2D grid or space policy | -| Mesh clipping, atlas, and UV code | Keep 3D | Do not place in generic storage | -| ECEF and spherical mask conversion | Keep 3D | 2D uses polygon classification or rasterisation | -| `MeshCodec` | Keep 3D | Add a raster chunk codec or container | - -## Existing `Inner` limitation - -The index supports `Inner`, but the SF merger effectively does not: - -- `Merger::call_merge()` dispatches only `Missing`, `Leaf`, and `Virtual` - combinations. -- `Masked::visit()` declares `Inner` unreachable. -- `NodeWriter::copy_subtree_to_output()` skips `Virtual` and asserts every - other visited node is `Leaf`. - -This matters because a physical coarse node coexisting with physical -descendants is exactly the `Inner` case. A general 2D/3D implementation should -make "has a physical payload" and "has indexed descendants" independent -properties and handle all four states throughout traversal and merging. - -## Recommended target architecture - -```mermaid -flowchart TB - subgraph Core["Dimension-neutral hierarchical store"] - TRAITS["TreeTraits<Key>
root, parent, children, validation"] - SI["SparseIndex<Key>"] - TR["Traversal<Key>"] - ST["Storage<Key, Payload, Codec>"] - DL["DiskLayout<Key>"] - ME["MergeEngine<Key, Payload, Policy>"] - SNAP[Snapshot and subtree reuse] - end - - subgraph D2["2D adapter"] - TILE[radix::tile::Id] - GRID[Web Mercator grid] - RASTER[RasterChunk] - GDAL[GDAL warped-window reader] - MASK2[2D polygon or raster mask policy] - RC[Raster codec and container] - end - - subgraph D3["3D adapter"] - OCT[octree::Id] - ECEF[ECEF octree space] - MESH[SimpleMesh] - MASK3[Extruded mesh-mask policy] - MC[MeshCodec] - end - - TILE --> TRAITS - OCT --> TRAITS - - RASTER --> ST - MESH --> ST - RC --> ST - MC --> ST - - MASK2 --> ME - MASK3 --> ME - - SI --> TR - SI --> ST - DL --> ST - TR --> ME - ST --> SNAP - ME --> SNAP -``` - -The central abstractions should therefore be: - -1. `TreeTraits`: root, parent, children, maximum depth, and key - validation. -2. `SparseIndex`: the current `IndexMap` algorithm. -3. `Traversal`: DFS, BFS, and refinement independent of child - count. -4. `DiskLayout`: key-to-path and path-to-key conversion. -5. `Storage`: logical storage and index maintenance. -6. `MergeEngine`: paired sparse-tree walking. -7. Generic merge actions such as `Recurse`, `Ignore`, `KeepLeft`, `KeepRight`, - and `Write`. -8. A snapshot copier that hard-links unchanged physical nodes without knowing - their payload type. -9. Separate 2D and 3D spatial, mask, and fallback policies. - -The GDAL builder and mask merger should be clients of this core, not part of -it. diff --git a/docs/raster-store/golden-e2e.sh b/docs/raster-store/golden-e2e.sh index b919938c..df44f53e 100755 --- a/docs/raster-store/golden-e2e.sh +++ b/docs/raster-store/golden-e2e.sh @@ -6,13 +6,13 @@ set -Eeuo pipefail -readonly SOURCE_DIR="/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store" -readonly BUILD_DIR="${SOURCE_DIR}/build/golden-e2e" -readonly RUN_ROOT="/data/scratch/codex/alpine-terrain-builder-golden-e2e" -readonly INPUT_ROOT="${RUN_ROOT}/inputs" -readonly REFERENCE_ROOT="${RUN_ROOT}/reference" -readonly LOG_ROOT="${RUN_ROOT}/logs" -readonly STATE_ROOT="${RUN_ROOT}/state" +readonly SOURCE_DIR="${SOURCE_DIR:-/home/codex/Documents/alpine-terrain-builder/terrain-builder-raster-store}" +readonly BUILD_DIR="${BUILD_DIR:-${SOURCE_DIR}/build/golden-e2e}" +readonly RUN_ROOT="${RUN_ROOT:-/data/scratch/codex/alpine-terrain-builder-golden-e2e}" +readonly INPUT_ROOT="${INPUT_ROOT:-${RUN_ROOT}/inputs}" +readonly REFERENCE_ROOT="${REFERENCE_ROOT:-${RUN_ROOT}/reference}" +readonly LOG_ROOT="${LOG_ROOT:-${RUN_ROOT}/logs}" +readonly STATE_ROOT="${STATE_ROOT:-${RUN_ROOT}/state}" readonly RUN_ID="${RUN_ID:-$(date +%Y%m%dT%H%M%S)}" readonly RUN_LOG_ROOT="${LOG_ROOT}/runs/${RUN_ID}" readonly TIMINGS_FILE="${RUN_LOG_ROOT}/timings.tsv" @@ -25,7 +25,7 @@ readonly DAG_BUILDER="${BUILD_DIR}/src/dag_builder/dag-builder" # GT rasters. They are recreated from the raw datasets at the start of a run. readonly RAW_GS="/data/raw/raster_data/Oe_2020/OeRect_01m_gs_31287.img" readonly RAW_GT="/data/raw/raster_data/Oe_2020/OeRect_01m_gt_31287.img" -readonly ELEVATION_ROOT="${INPUT_ROOT}/grossglockner-elevation-4km" +readonly ELEVATION_ROOT="${ELEVATION_ROOT:-${INPUT_ROOT}/grossglockner-elevation-4km}" readonly GS_VRT="${ELEVATION_ROOT}/grossglockner-gs-4km.vrt" readonly GT_VRT="${ELEVATION_ROOT}/grossglockner-gt-4km.vrt" readonly BASEMAP_TILES="${INPUT_ROOT}/basemap" @@ -109,11 +109,13 @@ verify_snapshot() { local snapshot="$1" local extension="$2" + local companion_extension="${3:-}" local payload_count - require_file "${snapshot}/terrain.index" - if [[ ! -s "${snapshot}/terrain.index" ]]; then - printf 'Index is empty: %s\n' "${snapshot}/terrain.index" >&2 + require_file "${snapshot}/octree.storemeta" + require_file "${snapshot}/octree.storeindex" + if [[ ! -s "${snapshot}/octree.storemeta" ]] || [[ ! -s "${snapshot}/octree.storeindex" ]]; then + printf 'Octree metadata or index is empty: %s\n' "${snapshot}" >&2 return 1 fi @@ -122,6 +124,15 @@ verify_snapshot() printf 'No %s payloads found in %s\n' "${extension}" "${snapshot}" >&2 return 1 fi + if [[ -n "${companion_extension}" ]]; then + local companion_count + companion_count="$(find "${snapshot}" -type f -name "*${companion_extension}" | wc -l)" + if [[ "${companion_count}" != "${payload_count}" ]]; then + printf '%s payload count (%s) does not match %s count (%s) in %s\n' \ + "${extension}" "${payload_count}" "${companion_extension}" "${companion_count}" "${snapshot}" >&2 + return 1 + fi + fi printf 'Verified %s: %s payloads\n' "${snapshot}" "${payload_count}" } @@ -135,7 +146,7 @@ build_sf() if [[ -f "${marker}" ]]; then printf '[%s] SKIP %s: completion marker exists\n' "$(timestamp)" "${name}" - verify_snapshot "${output}" ".terrain" + verify_snapshot "${output}" ".sfmesh" return fi @@ -151,10 +162,10 @@ build_sf() batch \ --target-level "${SF_TARGET_LEVEL}" \ --output "${output}" \ - --format .terrain \ + --format .sfmesh \ --threads "${SF_THREADS}" - verify_snapshot "${output}" ".terrain" + verify_snapshot "${output}" ".sfmesh" touch "${marker}" } @@ -164,7 +175,7 @@ merge_sf() if [[ -f "${marker}" ]]; then printf '[%s] SKIP merge_sf: completion marker exists\n' "$(timestamp)" - verify_snapshot "${SF_MERGED}" ".terrain" + verify_snapshot "${SF_MERGED}" ".sfmesh" return fi @@ -183,7 +194,7 @@ merge_sf() --output "${SF_MERGED}" \ --verbosity info - verify_snapshot "${SF_MERGED}" ".terrain" + verify_snapshot "${SF_MERGED}" ".sfmesh" touch "${marker}" } @@ -194,12 +205,12 @@ build_dag() if [[ -f "${marker}" ]]; then printf '[%s] SKIP build_dag: completion marker exists\n' "$(timestamp)" - verify_snapshot "${DAG_MERGED}" ".bin" + verify_snapshot "${DAG_MERGED}" ".dag" ".dagmeta" return fi mkdir -p "${DAG_MERGED}" - if [[ -s "${DAG_MERGED}/terrain.index" ]]; then + if [[ -s "${DAG_MERGED}/octree.storeindex" ]]; then continuation=(--resume) fi @@ -210,7 +221,7 @@ build_dag() "${continuation[@]}" \ --verbosity info - verify_snapshot "${DAG_MERGED}" ".bin" + verify_snapshot "${DAG_MERGED}" ".dag" ".dagmeta" touch "${marker}" } @@ -222,7 +233,7 @@ write_manifest() ( cd "${snapshot}" - find . -type f ! -name terrain.index -print0 \ + find . -type f ! -name octree.storeindex ! -name octree.storemeta -print0 \ | sort -z \ | xargs -0 -r sha256sum ) > "${output}" @@ -258,7 +269,7 @@ record_hard_links() fi ((newly_written += 1)) - done < <(find "${SF_MERGED}" -type f -name '*.terrain' -print0) + done < <(find "${SF_MERGED}" -type f -name '*.sfmesh' -print0) printf 'merged_payloads=%s\nlinked_to_gataki=%s\nlinked_to_basemap=%s\nnewly_written=%s\n' \ "${merged_count}" "${linked_to_gataki}" "${linked_to_basemap}" "${newly_written}" \ diff --git a/docs/raster-store/refactor-plan.md b/docs/raster-store/refactor-plan.md deleted file mode 100644 index f783e5d0..00000000 --- a/docs/raster-store/refactor-plan.md +++ /dev/null @@ -1,1141 +0,0 @@ -# 2D/3D hierarchical store refactor plan - -Status: proposal for review. This document is an implementation plan, not a -record of completed work. - -## Purpose - -This plan describes how to extract the existing octree-specific index, -traversal, storage, and codec code into a shared 2D/3D store. The refactor -must preserve existing 3D datasets and prove, using test-only mappings and -codecs where necessary, that the shared mechanisms work with a 2D key. -Persistent raster-fundamentalis formats, adapters, and tools are later work. - -## Decisions already made - -- The shared implementation will live in `src/terrainlib/store` and use the - `store` namespace. -- The minimal 2D traits adapter used to exercise the shared hierarchy will - live in `src/terrainlib/raster_store` and use the `raster_store` namespace. -- Common Structura Fundamentalis validation and errors will live in - `src/terrainlib/sf` and use the `sf` namespace. -- Existing 3D Structura Fundamentalis datasets must remain readable and - writable without changing their on-disk contract. -- DAG datasets written by the current code when Phase 0 begins must remain - readable and writable throughout the refactor without changing their index - or payload serialization. Older DAG payload schemas are not supported. -- `Inner` is a valid shared topology state and is supported by DAG, - raster-fundamentalis, and tile-base datasets. It is not valid in Structura - Fundamentalis datasets. -- SF producer and processing boundaries validate indexed SF data and return a - typed `sf::InvalidTopology` error containing an offending key when `Inner` - is present. The diagnostic `sf_index_browser` is exempt. This is an SF data - invariant, not a generic store or octree-format rule. -- 3D compatibility includes both existing path layouts: - `flat` and `level_and_coordinate_directories`. -- A persistent 2D raster tile format is not defined by this refactor. - [architecture.md](architecture.md) and - [storage-format.md](storage-format.md) describe intended direction and - provisional requirements that will be finalized in later RF work. They must - not be retrofitted onto existing 3D datasets. -- A path layout strategy should contain a stable identifier and two - operations: key to extensionless `NodePath`, and `NodePath` to key. It - should not require an inheritance hierarchy, RTTI, global - self-registration, or heap allocation. -- A configured codec owns all filename endings and maps one `NodePath` to one - or more physical files. `Codec::paths()` must not need a payload. -- Codecs are stateful runtime objects behind a small interface. They may - support reading, writing, or both. Reading and writing return - `std::expected`; unsupported operations and other operational failures are - reported as error values. Reading and writing must be reentrant (callable - concurrently from different threads). -- The runtime glTF codec catches exceptions from the existing glTF mesh writer - and converts them to `CodecError`; changing the mesh I/O API is out of scope. -- `store::RawStorage` exclusively owns the configured - `std::unique_ptr>`. `store::Storage` owns the raw - storage and therefore owns the codec transitively. Storage consumers do not - receive a codec template parameter and application call sites do not manage - codec objects. -- This refactor does not add synchronization to storage, indexes, or caches - and does not change their concurrency guarantees. Existing caller-side - synchronization and concurrency behaviour are preserved; any concurrency - bug fix is separate work. -- Preserve `dag::ThreadSafeStorage` as the caller-side synchronization around - DAG output storage. DAG storage remains cacheless while shared-lock reads are - used. -- Legacy index metadata selects a codec through an explicit, caller-supplied - resolver supplied by the payload-domain opening function. The octree format - adapter does not contain a global codec registry or depend on mesh or DAG - payload types. Application-level storage consumers do not call the resolver - or handle the resulting codec object. -- Dimension-specific index persistence uses a small runtime - `store::IndexFormat` value containing ordinary function pointers. It - is not an inheritance hierarchy and does not use global registration. -- `copy_from()` hard-links every file when the input and output codecs return - the same path list for a common dummy `NodePath`. Otherwise it decodes with - the input codec and encodes with the output codec. -- Public store operations reject invalid hierarchy keys through - `std::expected`; invalid keys are not represented by assertions or generic - booleans. -- Preserve `StorageSettings::allow_overwrite`. It defaults to `false`; - rejected overwrites return `AlreadyExists` through `std::expected`, and - enabling it preserves the existing DAG-builder overwrite and debug-export - behaviour. -- The existing cache implementations are currently non-functional. Port their - public API where it remains useful, keep application call sites building, and - provide compile coverage only. Cache behaviour is not a compatibility - requirement of this refactor. -- The `.png` written beside changed meshes by `sf_merger::NodeWriter` is an - unmanaged debug artifact. It is not part of a logical node, is not returned - by `Codec::paths()`, and is not indexed or copied by storage. Its existing - application-local behaviour is preserved. -- 3D geometry, ECEF bounds, mesh codecs, mesh reconstruction, mask geometry, - and raster-specific processing remain outside the shared store. - -## Goals - -1. Use one sparse hierarchy implementation for `octree::Id` and - `radix::tile::Id`. -2. Make traversal, storage, cache, and the runtime codec boundary - dimension-neutral while keeping production format adapters 3D-only in this - refactor. -3. Replace the current layout-strategy class hierarchy with small path-mapping - values backed by function pairs. -4. Allow one logical node payload to consist of multiple files without making - layouts aware of those files. -5. Preserve all valid existing 3D index files and payload paths. -6. Prove the shared topology and traversal with `radix::tile::Id` without - defining a persistent 2D adapter or format. -7. Land the refactor in small, testable steps. Every phase should build and - pass tests before the next phase begins. - -## Non-goals - -- Changing `octree::Id`, `octree::Space`, `IdRect`, `OddLevelShifted`, or - other 3D spatial calculations. -- Defining or implementing GDAL ingestion, raster resampling, filtering, - source selection, or mask rasterisation. -- Defining or implementing a raster-fundamentalis index, payload format, - layout ID, codec, publication lifecycle, or persistent storage adapter. -- Implementing an `rf_builder`, `rf_merger`, tile-base generator, tile server, - snapshot-reuse operation, or other RF tool in this refactor. -- Defining the raster-fundamentalis merge policy or the final paired-hierarchy - walker/action algebra. That work is deferred until `rf_merger` requirements - are defined. -- Defining a shared subtree-copy abstraction, `Inner` subtree-copy behaviour, - or forced re-encoding policy for RF. Those decisions are deferred until - `rf_merger`. -- Adding merge semantics for `Inner` nodes in SF. Such nodes are invalid SF - input and must be rejected before merge dispatch. -- Changing the existing 3D hard-link policy by adding a silent file-copy - fallback. -- Refactoring unrelated octree, DAG, mesh, or tile-builder code. -- Changing the serialized schema of the current DAG `.bin` payloads. - -## Compatibility contract - -Before moving code, tests must lock down the following 3D behaviour: - -| Item | Required compatibility | -|---|---| -| Index filename | `terrain.index` | -| Index field order | layout ID, preferred extension, index map | -| Node-key encoding | existing `octree::Id` level/index serialization | -| Node-status encoding | `Leaf = 0`, `Inner = 1`, `Virtual = 2` | -| Valid SF statuses | `Leaf` and `Virtual`; reject `Inner` with `sf::InvalidTopology` | -| Valid DAG statuses | `Leaf`, `Inner`, and `Virtual` | -| Flat layout ID | `flat` | -| Flat path | `-` | -| Coordinate layout ID | `level_and_coordinate_directories` | -| Coordinate path | `///` | -| Default layout | existing level/coordinate layout | -| Layout detection | both existing layouts remain detectable | -| Mesh codec selection | legacy preferred extension selects terrain or configured glTF codec | -| DAG codec selection | legacy `.bin` preferred extension selects the ZPP Bits codec | -| DAG payload encoding | current `dag::ClusterBatch` ZPP Bits serialization: metadata, then clustering | -| Equal codec path lists | hard-link every file, or report an explicit error | -| Different codec path lists | decode with input codec and encode with output codec | -| Overwrite setting | `StorageSettings::allow_overwrite`, default `false`; enabled writes replace existing payloads | - -Compatibility means that each refactor phase can open the Phase 0 fixtures -written by the current code. DAG formats older than the Phase 0 baseline and -pre-refactor readers opening post-refactor output are not tested. Exact -byte-for-byte rewriting of an unordered index map is not required, but the -serialized schema and values must remain compatible during the migration. - -The 3D index disk type should remain a versioned 3D adapter. The shared store -must not add a topology field, new header, checksum, or compression layer to -`terrain.index`. - -## Proposed source boundary - -```text -src/terrainlib/ -├── store/ -│ ├── NodeStatus.h -│ ├── NodeStatusOrMissing.h -│ ├── Traits.h -│ ├── InvalidKey.h -│ ├── Index.h -│ ├── traverse.h -│ ├── NodePath.h -│ ├── PathMapping.h -│ ├── Layout.h -│ ├── IndexFormat.h -│ ├── Codec.h -│ ├── CodecError.h -│ ├── OpenError.h -│ ├── CopyError.h -│ ├── StorageSettings.h -│ ├── RawStorage.h -│ ├── Storage.h -│ ├── IndexedStorage.h -│ ├── cache/ -│ │ ├── Interface.h -│ │ ├── Dummy.h -│ │ └── Lru.h -│ └── codec/ -│ └── ZppBits.h -├── mesh/ -│ └── codec/ -│ ├── Terrain.h -│ └── Gltf.h -├── sf/ -│ ├── InvalidTopology.h -│ └── validate_index.h -├── octree/ -│ ├── Id.h -│ ├── StoreTraits.h -│ ├── store_layout/ -│ │ ├── Flat.h -│ │ ├── LevelAndCoordinateDirectories.h -│ │ └── Mappings.h -│ └── storage/ -│ ├── IndexFile.h -│ └── open.h -└── raster_store/ - └── StoreTraits.h -``` - -The exact file grouping may be collapsed if a file would only contain a few -lines. The important boundaries are: - -- `store` contains dimension- and payload-neutral mechanisms; -- `store::codec::ZppBits` is the reusable concrete codec for payload types - that provide ZPP Bits serialization; -- `mesh::codec` contains the separately configured terrain and glTF codecs; -- `sf` contains SF-specific topology validation and errors shared by - `sf_builder`, `sf_merger`, and `dag_builder`; -- `octree` contains the 3D format and key adapters; -- `raster_store` contains only the minimal 2D hierarchy traits adapter in this - refactor; and -- subdirectory names match their namespaces where a subnamespace is used. - -Shared topology and octree-format adapters accept `Inner`. The SF restriction -is enforced by `sf::validate_index()` at SF producer/consumer boundaries and -reported as `sf::InvalidTopology`. It must not be embedded in `store::Index`, -traversal, or the generic 3D disk adapter. `sf_index_browser` is a diagnostic -tool and intentionally does not apply SF validation, so it can display invalid -trees including `Inner`. - -`sf::validate_index()` returns -`std::expected`. SF application-level error types -must retain this error and `CopyError` when propagating failures; neither is -reduced to a log message, assertion, or generic boolean. - -For this refactor, `sf::validate_index()` checks only for `Inner`. It does not -validate other structural or disk/filesystem invariants. Possible future -extensions are recorded in [todo.md](todo.md). - -Temporary forwarding headers and aliases under `octree` are allowed during -migration. They must not contain a second implementation. - -DAG serialization remains owned by `dag_builder`. Consolidate the serializers -for `dag::Id`, `dag::Group`, `dag::NodeMetadata`, `dag::ClusterBatch`, -`radix::geometry::Aabb3d`, GLM vectors, `Clustering`, `Cluster`, and -`TextureSet` in `src/dag_builder/serialization.h`. The DAG storage adapter -includes that header explicitly so template instantiation does not depend on -caller include order. Preserve the current tuples exactly: - -- `ClusterBatch`: metadata, clustering; -- `NodeMetadata`: group assignment, groups; and -- `Group`: children, error, bounds. - -`Group::child_errors` is currently neither populated nor serialized. It remains -non-persistent in this refactor and the Phase 0 fixture locks down its omission. - -## Shared interfaces - -The names below are the intended shape, not signatures that must be copied -verbatim without testing. - -### Hierarchy traits - -The store should be parameterized by one traits type rather than assuming that -all key classes expose identical member functions: - -```cpp -template -concept HierarchyTraits = requires(typename Traits::Key key) { - typename Traits::Key; - typename Traits::Hasher; - { Traits::root() } -> std::same_as; - { Traits::parent(key) }; - { Traits::children(key) }; - { Traits::is_valid(key) } -> std::same_as; -}; -``` - -The concrete names should be: - -```cpp -store::Index -store::Index -``` - -`octree::StoreTraits` adapts the existing optional parent/children API without -changing `octree::Id`. - -`raster_store::StoreTraits` adapts `radix::tile::Id` and must: - -- treat zoom zero as the only root; -- never call `radix::tile::Id::parent()` at zoom zero, where it underflows; -- reject coordinates outside `[0, 2^zoom)`; -- accept zoom levels 0 through - `std::numeric_limits::digits`, inclusive; -- treat that maximum zoom as terminal because a child cannot be represented - by the `uint32_t` x/y coordinates; -- validate the maximum zoom without evaluating an overflowing - `uint32_t{1} << 32`; -- use `radix::tile::Id::Hasher`; and -- return children in the deterministic order produced by - `radix::tile::Id::children()`. - -This traits adapter defines only hierarchy operations. It does not define -persistent coordinates, a path layout, or an RF disk format. - -The shared code must obtain roots, parents, children, validation, and hashing -through the traits. It must not use dimension checks or specialize behaviour -on key types internally. - -Operations accepting a key validate it through `Traits::is_valid()`. Index -lookup and mutation, traversal with an explicit root, and storage operations -return an `std::expected` retaining an `InvalidKey` when validation fails. -Keys produced internally by `Traits::root()`, `Traits::parent()`, and -`Traits::children()` are trusted only after trait-specific tests establish that -they preserve validity. The child order affects traversal order and is locked -down by the 2D and 3D trait tests; it is not serialized as separate metadata. - -The API result shapes are: - -- index lookup returns `expected, InvalidKey>`; -- index mutation and predicates return `expected>`; -- traversal returns `expected>`; and -- storage `load`, `save`, `has`, `remove`, path lookup, and `copy_from` return - operation-specific `expected` types which retain `InvalidKey` and any - applicable codec, filesystem, missing-source, or overwrite error. - -During Phase 1, a forwarding `octree::IndexMap` compatibility wrapper preserves -the old optional/bool API for existing 3D callers with already-valid -`octree::Id` values. It delegates to `store::Index`, is not -a second implementation, and is removed after callers migrate. - -### Sparse index and traversal - -Move the existing four-state model to: - -```cpp -store::NodeStatus -store::NodeStatusOrMissing -store::Index -store::traverse(index, visitor, refine, root, order) -``` - -The index algorithm remains the current one: - -- adding a physical descendant creates virtual ancestors; -- adding a payload to a virtual node makes it `Inner`; -- removing a payload from an `Inner` node makes it `Virtual`; -- removing the final descendant collapses virtual ancestors; and -- a physical parent becomes `Leaf` after its last descendant is removed. - -Traversal must follow only indexed nodes and use `Traits::children`. Child -order is the order supplied by the traits and is therefore deterministic per -hierarchy, not universally fixed by `store`. - -### Node paths and path mappings - -`store::NodePath` is an extensionless logical location for one hierarchy -node. For example: - -```text -octree flat 12-123456 -octree coordinates 12/34/56/78 -``` - -It does not necessarily name a physical file. Replace -`octree::disk::layout::Strategy` and -`octree::disk::layout::StrategyRegister` with a value similar to: - -```cpp -template -struct store::PathMapping { - std::string_view id; - NodePath (*key_to_node_path)(const Key&); - std::optional (*node_path_to_key)(const NodePath&); -}; -``` - -`store::Layout` owns the base directory and one `PathMapping`. It -does not own a preferred extension. A configured codec expands the -extensionless `NodePath` into the physical file or files. - -The stable ID is format metadata, not a third strategy operation. The -dimension adapters provide ordinary lookup functions: - -```cpp -octree::store_layout::flat() -octree::store_layout::level_and_coordinate_directories() -octree::store_layout::from_id(id) -octree::store_layout::all() -``` - -This retains runtime selection from an index file while removing virtual -dispatch, RTTI type-to-ID lookup, static registration, and ownership through -`unique_ptr`. - -Path parsers validate the complete logical `NodePath`, not a file ending. -Codec or format-adapter code removes and validates physical file endings -before asking the layout to recover a key. Invalid disk input returns an error -or `nullopt`; it must not trigger an assertion. - -### Codec interface - -A codec is a configured runtime object for one logical payload type. It owns -all physical filename endings and may map one `NodePath` to several files: - -```cpp -template -class store::Codec { -public: - virtual ~Codec() = default; - - virtual std::vector - paths(const NodePath& node_path) const = 0; - - virtual std::expected - read(const NodePath&) const { - return std::unexpected( - CodecError::unsupported_operation("read")); - } - - virtual std::expected write( - const NodePath&, - const NodeData&) const { - return std::unexpected( - CodecError::unsupported_operation("write")); - } -}; -``` - -Concrete codecs contain their configuration and are constructed before -storage use. `CodecError` is a payload-neutral operational error that records -the failed operation, an error category, and a diagnostic message. Concrete -codecs convert their domain errors to it. Unsupported read or write operations -may use the base implementation and return its `UnsupportedOperation` error. -Codec writes preserve the current directory-creation behaviour: they create -the parent directories required by their output paths before writing. The -storage hard-link path continues to create its target parent directories before -linking. - -`Codec::paths()` has the following contract: - -- it needs no NodeData payload and performs no filesystem access; -- it returns every physical file belonging to the logical node; -- results depend only on codec configuration and the supplied `NodePath`; -- result order is stable and pairs corresponding input/output files; -- two codecs returning the same path list for the same `NodePath` must produce - mutually compatible files; and -- different artifact counts or filename endings produce different lists. - -Examples: - -```text -Terrain codec - 12/34/56/78 - -> 12/34/56/78.terrain - -glTF codec configured for binary output - 12/34/56/78 - -> 12/34/56/78.glb - -glTF codec configured for JSON output - 12/34/56/78 - -> 12/34/56/78.gltf - -Multi-file test codec - 12/34/56/78 - -> 12/34/56/78.data - -> 12/34/56/78.metadata -``` - -The mesh side has separate terrain and glTF codecs because they use different -format implementations. Binary `.glb` and JSON `.gltf` remain configurations -of one glTF codec because both use the same `cgltf` implementation. - -The shared module also provides: - -```cpp -template -struct store::codec::ZppBits : store::Codec { .. }; -``` - -It uses the existing `io::read_from_path()` and `io::write_to_path()` -functions, maps one node to `.bin`, and converts `io::Error` to -`CodecError`. It contains no DAG-specific serialization logic. DAG payload -serialization remains in `dag_builder/serialization.h`, and its field order -and meshoptimizer/JPEG encoding remain unchanged. - -### Storage and format adapters - -Generalize storage over traits and NodeData. It owns a configured codec through -the runtime interface: - -```cpp -store::RawStorage -store::Storage -store::IndexedStorage -store::cache::Interface -``` - -`RawStorage` owns the -`std::unique_ptr>`. `Storage` owns `RawStorage`, and -`IndexedStorage` owns or derives from `Storage`; no other layer shares codec -ownership. Moving storage transfers ownership. Caches, layouts, index formats, -resolvers, and application consumers never own the codec. - -A move disarms the source so its destructor cannot save moved-out index state. -Move assignment must finalize a displaced dirty destination through the -existing destructor-save policy rather than silently discard it. Tests cover -the DAG builder's move into and release from `dag::ThreadSafeStorage`. - -Domain-specific mesh codecs remain under `mesh::codec`. The reusable ZPP Bits -codec remains under `store::codec`. RF codecs are deferred. - -Index serialization is not a responsibility of `store::Index`. Opening and -saving a dataset receives the following small runtime values: - -```cpp -template -struct store::IndexMetadata { - store::Index index; - std::string layout_id; - std::string codec_selector; -}; - -template -struct store::IndexFormat { - std::string_view index_filename; - - std::expected, IndexFormatError> - (*read)(const std::filesystem::path& index_path); - - std::expected - (*write)( - const std::filesystem::path& index_path, - const IndexMetadata& metadata); - - std::optional> - (*mapping_from_id)(std::string_view id); - - PathMapping - (*default_mapping)(); -}; -``` - -The value provides: - -- the index filename; -- index read/write conversion; -- mapping lookup by stable ID; -- the default mapping. - -Legacy directory discovery is an octree opening helper which composes the -octree index format, the supplied payload-domain codec resolver, and the known -octree mappings. It is not a generic `IndexFormat` operation. Neither the -format value nor discovery may reintroduce a layout class hierarchy or global -registration. - -For 3D, the adapter reads and writes the current `octree` index DTO unchanged. -Its `codec_selector` is exactly the legacy `preferred_extension`, including -the leading dot. Storage retains the selected `IndexFormat`, index path, -layout ID, and codec selector as its index-persistence state, so explicit and -destructor-triggered index saves can reproduce the same metadata. - -When opening indexed storage or discovering a legacy unindexed directory, it -passes the legacy `preferred_extension` to a caller-supplied codec resolver. -The resolver is an ordinary callable and returns -`std::expected>, CodecError>`. It is not -a global registry. - -The payload domains provide ordinary resolver functions: - -```text -mesh::codec::from_extension - .terrain -> terrain codec - .glb -> glTF codec with binary container - .gltf -> glTF codec with JSON container - -dag::codec::from_extension - .bin -> store::codec::ZppBits - -dag::codec::metadata_from_extension - .bin -> read-only dag::codec::MetadataView -``` - -An unknown extension returns an explicit `UnsupportedCodec` error. Opening a -new empty store receives an explicit legacy codec selector plus an already -constructed codec at the payload-domain opening boundary; the generic storage -does not infer persistent metadata from `Codec::paths()`. Mesh and DAG -convenience functions select or resolve the codec and pass ownership into raw -storage, so application storage consumers do not handle codec objects. -Convenience functions in `src/dag_builder/storage.h` supply the DAG resolvers. -Preserve `DagMetaStorage` and `IndexedDagMetaStorage` as independently readable -views of the metadata prefix in each `ClusterBatch` `.bin` file. Their codec -returns `UnsupportedOperation` from writes instead of terminating or replacing -a full batch with metadata alone. `dag::codec::MetadataView` implements this -domain-specific read-only prefix view over the generic ZPP Bits codec. - -Opening functions return their requested storage type through -`std::expected<..., OpenError>`. `OpenError` is a typed sum which retains the -failing path and the underlying error where applicable: - -- index I/O or malformed index metadata (`IndexFormatError`); -- filesystem failure; -- unknown layout ID; -- unsupported codec selector or codec construction failure (`CodecError`); and -- invalid hierarchy key (`InvalidKey`). - -Loading and saving likewise return storage-level expected errors which retain -an invalid key, an underlying `CodecError`, and `AlreadyExists` for a rejected -save. `CopyError` retains invalid-key, missing-source, overwrite, filesystem, -and codec failures. The SF call chains changed in Phase 4 retain these errors -to their application boundary. DAG callers preserve their existing per-node -log-and-continue and command-line policies; making the DAG builder fail-fast is -not part of this refactor. - -Legacy unindexed-directory discovery remains in the 3D adapter: it recognizes -candidate endings by asking the supplied resolver, removes an accepted ending -to obtain a `NodePath`, and then invokes the selected layout parser. The -generic layout does not recover keys directly from codec-owned file paths. -Only a missing index triggers this discovery path. An unreadable or malformed -index, unknown layout, or unsupported codec selector returns `OpenError` and -must not silently fall back to directory discovery. - -Automatic dirty-index saving currently happens in the 3D storage destructor. -Preserve that behaviour for existing 3D entry points during the migration. - -### Copying one node and SF subtree reuse - -There are two separate responsibilities in the current implementation: - -1. `sf_merger::NodeWriter` traverses a source subtree and - `sf_merger::cut_leaf_node()` identifies an unchanged leaf. -2. `octree::Storage::copy_from()` delegates to - `octree::RawStorage::copy_from()`, where - `std::filesystem::create_hard_link()` performs the actual hard link and the - target index is updated on success. - -The filesystem hard-link implementation is therefore already in terrainlib. -Move the one-node storage operation into the shared store, but keep subtree -selection and traversal in `sf_merger`. There is no current DAG caller, and RF -subtree-copy requirements will be defined with `rf_merger`. - -The migrated call chains are: - -```text -sf_merger decides to keep a source subtree unchanged - -> NodeWriter traverses the source index - -> store::Storage::copy_from() - -> hard-link every codec path, or decode/encode - -sf_merger determines that a cut leaf is unchanged - -> cut_leaf_node() - -> store::Storage::copy_from() - -> hard-link every codec path, or decode/encode -``` - -`Storage::copy_from()` remains the operation for copying one logical node. -For one key, `copy_from()`: - -1. calls the input and output `Codec::paths()` with the same fixed dummy - `NodePath`; -2. when the lists are equal, calls both codecs again with their actual source - and target `NodePath` values and hard-links every source path to the - corresponding target path; -3. when the dummy lists differ, reads the payload with the input codec and - writes it with the output codec; and -4. updates the target index only after all links or the write complete. - -The dummy path must be fixed and collision-free, for example -`__codec_probe__/node`. Path lists are compared exactly, including count, -order, and filename endings. - -If overwrite is enabled for an indexed target, remove the target index entry -immediately before the first target file is modified. If linking several files -then fails partway through, return the error without a transactional rollback -guarantee; target links or old files may remain, but the logical node stays -unindexed. The copy operation stops immediately and propagates the failure -until the application aborts the overall operation. There is no journal, -rollback, cleanup guarantee, or silent copy fallback. An unsupported read or -write needed for re-encoding is returned through `CopyError`, retaining the -underlying `CodecError`. - -Hard-link rules: - -- never modify an existing linked payload in place; -- a matching codec path list hard-links every file; -- a different path list decodes with the input codec and encodes with the - output codec; -- hard-link failure is explicit; -- no silent file-copy fallback is introduced. - -#### SF-local subtree traversal - -`sf_merger::NodeWriter::copy_subtree_to_output()` remains in `sf_merger`. It -uses shared traversal and `Storage::copy_from()`, but it is not promoted to a -generic store API during this refactor. - -SF validation guarantees that its indexed inputs contain only `Leaf` and -`Virtual` nodes. The SF-local traversal skips `Virtual`, copies `Leaf`, and -does not define behaviour for `Inner`. An `Inner` node is rejected before -merge or cut processing with `sf::InvalidTopology` containing the offending -key. - -When `rf_merger` is designed, it can initially compose `store::traverse` and -`Storage::copy_from()`. At that point, the SF and RF implementations provide -enough evidence to decide whether a shared subtree copier is useful and how it -must handle RF `Inner` nodes. - -#### Error propagation - -The lower storage layer already represents ordinary copy failures, including -missing source files, directory creation failure, hard-link failure, decode -failure, and encode failure. The current `NodeWriter` consumes -`Storage::copy_from()` with `DEBUG_ASSERT_VAL`, as does the unchanged-leaf path -in `cut_leaf_node()`, while some overwrite paths terminate through -`LOG_ERROR_AND_EXIT()`. Their `void` call chains prevent the application from -reporting or handling these failures. - -Return failures from the SF-local subtree and cut functions through their -callers until the application boundary can report the affected key and path. -Codec, filesystem, unsupported-conversion, malformed-dataset, and overwrite -failures are propagated with `std::expected`; `CopyError` retains any -underlying `CodecError`. Assertions remain appropriate for internal -invariants, but operational failures are not assertion failures or -intentionally thrown exceptions. - -### Paired hierarchy walking is deferred - -This refactor does not extract `sf_merger::Merger` into a shared paired-tree -walker. SF only permits `Missing`, `Leaf`, and `Virtual`, and its current -recursion does not provide enough evidence to define the `Inner` behaviour -needed by raster-fundamentalis and tile-base merging. - -The previously proposed mutually exclusive actions `Recurse`, `Ignore`, -`KeepLeft`, `KeepRight`, and `Write` cannot express both an action for the -current physical payload and recursion into descendants. `Inner` merging may -require both. Whether the future interface uses a combined `WriteAndRecurse` -action or independent current-node and descendant decisions belongs to the -`rf_merger` design. - -For this refactor: - -- keep recursion, subtree traversal, and mesh policy in `sf_merger`; -- validate SF inputs and reject `Inner` through `std::expected`; -- move only the one-node `Storage::copy_from()` mechanism into `store`; and -- do not add a shared `store::merge` namespace. - -A future `rf_merger` task will define the paired-tree action algebra from the -2D requirements and may migrate `sf_merger` once both use cases are known. - -## Implementation phases - -Each phase should be one reviewable commit unless the tests and implementation -are clearer as two commits. Do not begin a later phase while the current phase -has failing tests. - -### Phase 0 — Capture current compatibility - -No production behaviour changes. - -1. Add golden SF fixtures created by the current code: - - one `terrain.index` using `flat`; - - one using `level_and_coordinate_directories`; - - across the fixtures, physical payload paths for a root, child, and deeper - descendant, without placing physical payloads at ancestor and descendant - keys in the same index; and - - index entries containing `Leaf` and `Virtual`, but no `Inner`. -2. Add one golden DAG dataset whose index selects `.bin`, whose payload - contains a valid serialized `dag::ClusterBatch`, and whose index contains - `Leaf`, `Virtual`, and `Inner`. Use the current metadata-then-clustering - format, include non-trivial group metadata, and prove that the same file can - be opened through the read-only `NodeMetadata` view. Lock down that - `Group::child_errors` is not serialized. -3. Test that all fixtures open, resolve the expected IDs and extensions, and - traverse the expected sparse nodes. -4. Add path round-trip tests for boundary IDs and both layouts. -5. Add storage tests for: - - matching-extension hard links; - - different-extension decode/re-encode; - - overwrite-enabled replacement; - - indexed and unindexed opens; and - - final index creation by directory scan. -6. Record the pre-refactor public aliases used by `sf_builder`, `sf_merger`, - `sf_index_browser`, `dag_builder`, and `dag_convert_debug`, including - `DagMetaStorage`, `IndexedDagMetaStorage`, and `dag::ThreadSafeStorage`. - -Exit criterion: the compatibility tests pass against the untouched -implementation and fail when any stable filename, layout ID, path encoding, -status value, index field order, or DAG payload serialization is deliberately -changed. - -### Phase 1 — Extract topology into `store` - -1. Move `NodeStatus` and `NodeStatusOrMissing` to `store`, preserving their - underlying values and serialization. -2. Introduce the hierarchy-traits concept and `octree::StoreTraits`. -3. Convert `IndexMap` into `store::Index`. -4. Convert traversal into `store::traverse`. -5. Add `raster_store::StoreTraits` for `radix::tile::Id`. -6. Run the same index-transition and DFS/BFS tests with both trait types, - including deterministic child order, invalid-key errors through - `std::expected`, maximum-depth children, and explicit traversal roots. -7. Provide the temporary forwarding `octree::IndexMap` wrapper and aliases so - downstream migration is separate; otherwise migrate downstream immediately. - -Exit criterion: 2D and 3D keys pass the same topology suite; existing 3D -callers still build through aliases; no filesystem code has changed. - -### Phase 2 — Introduce path mappings and runtime codecs - -This phase introduces the extensionless layout and runtime codec pieces -together. It does not cut production storage over to them yet. The existing -storage, layout strategies, and static codecs remain temporarily as the -working compatibility path until Phase 3 can replace the complete -layout-plus-codec path construction in one step. - -1. Add `store::NodePath`, `store::PathMapping`, and - `store::Layout`. -2. Add the stateful `store::Codec` interface with `paths()`, `read()`, - and `write()`, plus `CodecError`. -3. Add the runtime `store::codec::ZppBits`, preserving the existing `.bin` - path and serialized payload bytes. -4. Consolidate the DAG serialization functions in - `src/dag_builder/serialization.h` without changing their serialized field - order, meshoptimizer encoding, or JPEG texture encoding. -5. Add the terrain codec and one glTF codec configured for binary `.glb` or - JSON `.gltf`. Keep the current extension-dispatching `octree::MeshCodec` - only as temporary production compatibility glue until the Phase 3 cutover. - Test that glTF writer exceptions become `CodecError` values. -6. Port the two existing 3D layouts to ordinary function pairs without - changing stable IDs. The mappings return `level-index` and - `level/x/y/z` without file endings. -7. Add explicit `from_id()` and `all()` lookup functions in the 3D adapter. - Keep the singleton strategy registry only for the old production storage - path until Phase 3. -8. Compose each new 3D mapping with each applicable runtime codec in tests and - prove that they resolve all Phase 0 fixtures to identical physical payload - paths. -9. Add focused codec tests using single-file, multi-file, read/write, and - write-only test codecs. Test stable path ordering, unsupported operations, - directory creation, conversion of domain errors to `CodecError`, and - concurrent reads and writes. Exercise reentrancy of every production codec - used by the parallel DAG builder. - -Exit criterion: the extensionless mappings and runtime codecs together resolve -all Phase 0 fixtures to their existing physical payload paths; generic -`Layout` contains no extension; the new codec tests pass; existing production -storage and all callers still build unchanged through the temporary legacy -path. - -### Phase 3 — Generalize storage and index lifecycle - -1. Cut the production path construction over to the Phase 2 - `store::Layout` and runtime codecs. Move the legacy preferred extension - out of layout state and retain it as the 3D format adapter's codec selector. -2. Port legacy layout discovery so it recognizes codec endings through the - supplied resolver, strips the accepted ending, and then calls - `node_path_to_key()`. -3. Move copy error, raw storage, logical storage, and indexed storage into - `store`. Port the existing cache API where useful for source compatibility, - but do not require cache behaviour tests. -4. Make `RawStorage` exclusively own a configured - `std::unique_ptr>`. `Storage` owns it transitively through - raw storage; remove the codec template parameter from every storage type. -5. Replace every embedded `octree::Id` with `Traits::Key`. -6. Make every raw file operation obtain its complete file list through - `Codec::paths()`. `has()` requires every listed file, and `remove()` removes - every listed file. -7. Keep domain-specific mesh codecs outside the shared module under - `mesh::codec`. -8. Add the function-pointer-based `IndexFormat`, `IndexMetadata`, and - typed format/open errors. Split generic index maintenance from 3D index - serialization and legacy folder discovery. Test that only a missing index - starts discovery; all other index errors are returned. -9. Keep the current 3D `terrain.index` DTO and open functions as compatibility - adapters over the shared storage. Retain its exact preferred extension as - `codec_selector`, resolve it through the payload-domain mesh or DAG - resolver, and retain the format metadata required by automatic saving. -10. Add DAG storage convenience functions that supply the writable batch and - read-only metadata resolvers, and mesh storage convenience functions that - select the configured terrain or glTF codec. Preserve the DAG metadata - storage aliases. Migrate `dag_builder`, `dag_convert_debug`, and mesh - storage consumers without exposing codec objects at application call sites. -11. Migrate the existing octree storage aliases and all other application - callers, including adapting `dag::ThreadSafeStorage` to the new key and - expected-returning APIs while preserving its shared/exclusive locking. -12. Preserve the current 3D destructor-save behaviour until all callers have - explicit index finalization. Test move construction, move assignment over - dirty state, moved-from destruction, and the DAG move/release path. -13. Preserve `StorageSettings::allow_overwrite`, including the DAG builder's - overwrite mode and repeat debug export. Replace process termination on a - rejected overwrite with `AlreadyExists` in the storage-level expected - error. Test that a rejected save returns `AlreadyExists` and that enabling - overwrite replaces the existing payload. -14. Add resolver tests for `.terrain`, `.glb`, `.gltf`, writable - `ClusterBatch` `.bin`, and read-only `NodeMetadata` `.bin` dispatch, plus - explicit failure for an unknown preferred extension and metadata writes. -15. Instantiate the shared storage tests with `raster_store::StoreTraits` - using a test-only path mapping and codec. This proves the storage templates - contain no hidden `octree::Id` dependency without defining a stable RF - layout, codec, or disk format. -16. Delete the old strategy base class, strategy registry, concrete strategy - classes, static codec concept and codecs, and their temporary compatibility - glue once no call site uses them. - -Test that the Phase 0 DAG fixture opens through both new resolvers and that a -new deterministic `.bin` payload matches its golden bytes and remains readable -through the unchanged ZPP serialization functions. Test unknown layout IDs, -malformed index metadata, invalid hierarchy keys, and retained underlying -open/codec errors through `std::expected`. - -Exit criterion: all existing applications build and all Phase 0 fixtures pass -through the shared runtime codec and storage implementation. Phase 0 DAG -payload bytes and `.bin` paths remain unchanged. No extension-dispatching -mesh codec, layout inheritance, RTTI lookup, static registrar, owning strategy -pointer, or second storage implementation remains under `octree`. - -### Phase 4 — Harden node reuse and enforce SF topology - -1. Change `Storage::copy_from()` to compare input and output codec path lists - for the fixed dummy `NodePath`. -2. Hard-link all actual files when the lists match. A partially failed - multi-file operation returns an error without rolling back links already - created. -3. Decode with the input codec and encode with the output codec when lists - differ. -4. Test: - - one-file hard linking; - - multi-file hard linking; - - different path counts and endings; - - error propagation after a partially failed multi-file hard link; - - overwrite failure leaving an existing logical node unindexed even when - old or partial files remain; - - conversion between terrain and glTF; - - `copy_from()` overwrite rejection; - - `copy_from()` overwrite-enabled replacement; and - - missing-file, hard-link, decode, and encode error propagation. -5. Add `sf::validate_index()`, returning `sf::InvalidTopology` with the - offending key when it encounters `Inner`. Do not add other validation rules - in this refactor. -6. Apply the validator to SF-merger merge and cut inputs and the DAG builder's - SF input before processing. Apply it to SF-builder and SF-merger output - after the completed `terrain.index` has been written. If output validation - fails, retain the written index and payloads for diagnosis, propagate - `sf::InvalidTopology` to the command line, and report that the output is - invalid. Do not apply the validator when opening DAG datasets, through - generic octree/store adapters, or in the diagnostic `sf_index_browser`. - Extract an SF-builder finalization boundary into `sfbuilderlib` which writes - the index, validates it, and returns the typed result so output validation - and command-line propagation are directly testable. -7. Keep SF recursion, subtree traversal, and mesh policy in `sf_merger`. - Change its subtree and cut call chains to propagate validation and - `copy_from()` failures through `std::expected` to the application boundary. -8. Add integration tests proving: - - valid `Leaf`/`Virtual` SF merge behaviour is unchanged; - - an SF input containing `Inner` fails validation before merge dispatch; - - invalid SF output writes `terrain.index` for diagnosis and returns - `sf::InvalidTopology` to the application boundary; - - an unchanged SF subtree is hard-linked and a changed boundary node is - newly written; and - - an unchanged leaf in the SF cut path is hard-linked while a clipped leaf - is newly written. - -Exit criterion: one-node copying works through `Codec::paths()`, SF consumers -reject `Inner` with a typed error before processing, existing valid SF merge -and cut behaviour is preserved, invalid SF output remains inspectable with a -written `terrain.index`, and neither a shared subtree copier nor a paired-tree -walker has been introduced. - -### Phase 5 — Cleanup and documentation - -1. Remove temporary forwarding headers that no repository caller needs. -2. Remove obsolete files under `octree/disk` and the old generic - implementation under `octree/storage`. -3. Keep only 3D key, format, codec, and compatibility adapters under - `octree`. -4. Update includes, CMake source lists, and precompiled-header includes. -5. Update [architecture.md](architecture.md), - [storage-format.md](storage-format.md), [status-quo.md](status-quo.md), and - the before-refactor report with links to the implemented boundary. State - clearly that RF formats and tools remain future work and that architecture - requirements are not acceptance criteria for this refactor. Preserve the - before-refactor report as history; do not rewrite it as if it described the - new code. -6. Document the final public names, a legacy 3D opening example, and an - in-memory `radix::tile::Id` topology example. Do not document a persistent - RF format or opening API. - -Exit criterion: repository search finds no generic implementation tied to -`octree::Id`; all tests pass; the old layout strategy hierarchy is gone. - -### Phase 6 — Remove migration-only compatibility fixtures - -1. Delete the Phase 0 golden SF and DAG datasets and the tests whose only - purpose is opening or byte-comparing data written before the completed - refactor. -2. Keep the format round-trip, path, resolver, topology, storage, error, and - application integration tests which exercise the final implementation - without pre-refactor fixture files. - -Exit criterion: no pre-refactor dataset fixture remains, and the retained test -suite passes against data produced by the final implementation. - -## Test and verification plan - -Generic store tests should live in the existing `unittests_terrainlib` target. -Suggested files: - -```text -unittests/terrainlib/store_index.cpp -unittests/terrainlib/store_traverse.cpp -unittests/terrainlib/store_layout.cpp -unittests/terrainlib/store_codec.cpp -unittests/terrainlib/store_storage.cpp -unittests/terrainlib/store_compatibility.cpp # temporary through Phase 5 -unittests/terrainlib/sf_validate_index.cpp -``` - -The temporary DAG payload-compatibility fixture and resolver integration test -belong in `unittests_dagbuilder`, because `terrainlib` must not depend on -`dag::ClusterBatch` or its serializers. Phase 6 removes the fixture and its -byte-comparison test while retaining resolver tests built from current data. - -The validator's unit tests belong in `unittests_terrainlib`. Boundary tests -belong with their consumers: SF-builder output validation in -`unittests_sfbuilder`, merge and cut validation/error propagation in -`unittests_sfmerger`, and DAG-builder SF-input validation in -`unittests_dagbuilder`. `sf_index_browser` remains unvalidated by design. - -Cache migration has compile coverage only. Existing cache application call -sites must continue to build where the API remains meaningful, but no cache -behaviour or compatibility test is required. - -During implementation: - -1. Build in `$source_dir/build/$config_name`. -2. Run unit tests from that build directory. -3. Run the focused store tests after each edit. -4. Run the full `unittests_terrainlib` target at every phase boundary. -5. Run `unittests_dagbuilder` after the ZPP codec, DAG serialization header, or - DAG resolver changes. -6. Configure with `ALP_BUILD_SF_BUILDER`, `ALP_BUILD_SF_MERGER`, - `ALP_BUILD_SF_INDEX_BROWSER`, `ALP_BUILD_DAG_BUILDER`, and - `ALP_BUILD_DAG_CONVERT_DEBUG` enabled, then build `sf-builder`, `sf-merger`, - `sf-index-browser`, `dag-builder`, and `dag-convert-debug` after their - storage aliases move. -7. Run `unittests_sfbuilder`, `unittests_sfmerger`, and - `unittests_dagbuilder`, plus any existing merger integration fixture, after - the Phase 4 validation changes. -8. Inspect `git diff --check` and the final worktree before each commit. - -No formatting-only pass or unrelated refactor belongs in these commits. - -## Expected migration map - -| Current code | Target | -|---|---| -| `octree/NodeStatus.h` | `store/NodeStatus.h` plus temporary alias | -| `octree/NodeStatusOrMissing.h` | `store/NodeStatusOrMissing.h` plus temporary alias | -| `octree/IndexMap.*` | `store/Index.h` | -| `octree/traverse.h` | `store/traverse.h` | -| complete node paths embedded in layouts | extensionless `store/NodePath.h` plus codec endings | -| `octree/disk/Layout.h` | `store/Layout.h` | -| `octree/disk/layout/Strategy.h` | `store/PathMapping.h` | -| `StrategyRegister.h` | explicit dimension-adapter lookup functions | -| `strategy/Flat.h` | `octree/store_layout/Flat.h` | -| `strategy/LevelAndCoordinateDirectories.h` | `octree/store_layout/LevelAndCoordinateDirectories.h` | -| `octree/storage/cache/*` | `store/cache/*` where useful; compile compatibility only | -| `octree/storage/codec/Codec.h` | runtime `store/Codec.h` | -| `octree/storage/codec/DefaultCodec.h` | runtime `store/codec/ZppBits.h` | -| `octree/storage/codec/MeshCodec.h` | `mesh/codec/Terrain.h` and configured `mesh/codec/Gltf.h` | -| `octree/storage/codec/ReadOnlyCodec.h` metadata specialization | `dag::codec::MetadataView` | -| `octree/storage/RawStorage.h` | `store/RawStorage.h` | -| `octree/storage/Storage.h` | `store/Storage.h` | -| `octree/storage/IndexedStorage.h` | `store/IndexedStorage.h` | -| `octree::StorageSettings::allow_overwrite` | `store::StorageSettings::allow_overwrite`, preserving default and enabled behaviour | -| `octree/storage/helpers.*` | generic scan helpers plus 3D format adapter | -| `octree/disk/IndexFile.h` | versioned 3D format adapter under `octree` | -| DAG serializers in `dag_node.h`, `dag_id.h`, `metadata.h`, `encoded.h`, and `zpp_bits_glm.h` | `dag_builder/serialization.h` | -| `dag_builder/storage.h` aliases | Batch and read-only metadata storage aliases plus codec resolver convenience functions | -| `dag_builder/thread_safe_storage.h` | Adapted caller-side wrapper over shared storage | -| `sf_merger::NodeWriter` subtree loop | remains in `sf_merger`; return copy failures through `std::expected` | -| `sf_merger::NodeWriter` auxiliary `.png` write | remains an unmanaged, application-local debug artifact | -| `sf_merger::cut_leaf_node()` copy path | remains in `sf_merger`; return copy failures through `std::expected` | -| SF merger `Inner` `UNREACHABLE()` path | `sf::validate_index()` returning `sf::InvalidTopology` | - -## Risks and controls - -| Risk | Control | -|---|---| -| Existing indexes stop loading | Golden pre-refactor fixtures and unchanged 3D DTO | -| The refactor changes current DAG `.bin` bytes | Temporary golden DAG fixture, unchanged serializers, and explicit `.bin` resolvers | -| Octree format adapter gains DAG dependencies | Caller-supplied resolver owned by `dag_builder` | -| Unknown legacy extension silently selects the wrong codec | Return an explicit `UnsupportedCodec` error | -| Valid legacy paths are parsed differently | Characterization and round-trip tests before replacement | -| Template migration creates a large unreviewable diff | Compatibility aliases and phase-by-phase caller migration | -| `radix::tile::Id` root underflows | Traits intercept root parent lookup | -| Invalid `radix::tile::Id` values enter the shared index | Validate through `raster_store::StoreTraits` and test boundary zooms | -| Invalid `Inner` nodes reach SF merge dispatch | Validate every SF input first and return the offending key in a typed error | -| SF subtree copying is generalized before RF requirements exist | Keep it in `sf_merger`; reconsider extraction with `rf_merger` | -| A paired-walker API is fixed before RF semantics are known | Defer its action algebra until `rf_merger` requirements are defined | -| Multi-file hard linking fails partway through | Remove any existing index entry before modifying target files, stop immediately, leave the node unindexed, propagate the error, and abort the overall operation; old or partial files may remain | -| Incompatible codecs return the same path list | Treat path-list equality as a codec contract and test every concrete codec pairing | -| Output-only codec is selected for required input | Return a clear `UnsupportedOperation` error | -| Shared code accumulates mesh or provisional RF policy | Dependency tests/review against the source boundary | -| The refactor accidentally fixes the future RF disk format | Do not add a persistent 2D format adapter, layout, or codec | - -## Deferred raster-fundamentalis work - -This refactor is a prerequisite for, not an implementation of, the system -described in [architecture.md](architecture.md). That document and -[storage-format.md](storage-format.md) remain useful design input, but their RF -details are not acceptance criteria for this refactor and are not declared -final here. - -A later RF design phase must resolve and test at least: - -- the persistent index filename and schema, and how they use the existing - generic serialization envelope and versioning support; -- persistent tile keys, coordinate convention, layout IDs, and node paths; -- tile and source-attribution payload formats and codecs; -- snapshot construction, validation, publication, and crash expectations; -- hard-link preflight and unchanged-tile reuse; -- `rf_merger` policy, paired-tree walking, and `Inner` copy behaviour; and -- RF builders, converters, debugging outputs, and other tools. - -None of those decisions blocks completion of this refactor. -The removed ideas are retained, without implementation-plan status, in the -explicitly draft [rf_builder notes](rf_builder.md) and -[rf_merger notes](rf_merger.md). diff --git a/docs/raster-store/status-quo.md b/docs/raster-store/status-quo.md deleted file mode 100644 index 9d2b5701..00000000 --- a/docs/raster-store/status-quo.md +++ /dev/null @@ -1,274 +0,0 @@ -# Status quo and reuse assessment - -This document evaluates the current repository after commit `9cf9065` -(`Consolidate raster handling and use std::expected`). It distinguishes -reusable mechanisms from interfaces that encode assumptions unsuitable for -the raster store. - -## Summary - -The project already contains most low-level ingredients: - -- `radix::Raster` for contiguous typed raster memory; -- `radix::tile::Id` for quadtree addressing; -- Web Mercator grid calculations and GDAL reprojection in `tile_builder`; -- sparse topology, indexed traversal, codecs, layouts, and hard-link reuse in - the octree storage code; and -- the SF merger's snapshot-like reuse of unchanged subtrees. - -There is no existing component that should become the raster store unchanged. -The best path is to compose the Radix raster and tile primitives with a new -2D storage layer, while extracting or adapting selected octree-storage ideas. - -## Reuse matrix - -| Component | Assessment | Intended use | -|---|---|---| -| `radix::Raster` | Reuse directly | In-memory payloads and source maps | -| `radix::RasterMask` | Reuse directly | Temporary validity/selection masks | -| `radix::raster::transform` | Reuse directly | Typed pixel transformations | -| `radix::raster::generate_mipmap` | Do not use as general generator | Only a 2x2 component-wise box average | -| `radix::tile::Id` | Reuse after hardening or through an adapter | Persistent quadtree keys | -| `radix::quad_tree::Node` | Do not reuse for disk index | Dense in-memory ownership model | -| CTB `GlobalMercator` and grid bounds | Reuse initially | Tile bounds and Web Mercator resolution | -| `Dataset` and GDAL setup | Reuse/adapt | Input discovery and reprojection | -| `DatasetReader` | Adapt substantially | Windowed, typed, multi-band ingestion | -| `Tiler` / `ParallelTiler` | Reuse calculations, replace orchestration | Candidate chunk enumeration | -| `ParallelTileGenerator` | Do not use as store builder | Small-file writer with incompatible lifecycle | -| Octree `IndexMap` algorithm | Reuse design; generalize or port | Sparse physical/virtual topology | -| Octree `Storage_` / `RawStorage_` | Reuse design and selected code | Codec boundary, indexing, hard-link copy | -| Octree disk layouts | Do not use as-is | They encode `octree::Id` and 3D paths | -| SF merger visitors and geometry code | Do not reuse | Mesh- and ECEF-specific semantics | -| SF merger unchanged-subtree copy | Reuse design | Snapshot construction and hard links | -| `zpp_bits` serialization helpers | Reuse cautiously | Versioned metadata/index serialization | - -## Radix raster - -### What can be reused - -`extern/radix/src/radix/raster.h` now provides a shared, value-typed raster: - -- rectangular `glm::uvec2` dimensions; -- contiguous row-major `std::vector` storage; -- element and byte spans; -- typed pixel access; -- move construction from an existing vector; -- a contiguous byte-valued `RasterMask`; -- dimension-checked concatenation; and -- masked and unmasked transforms. - -These properties fit both store arrays: - -```cpp -radix::Raster payload; -radix::Raster source_map; -``` - -The class correctly remains independent of Web Mercator, tile IDs, source -metadata, compression, and file I/O. Those belong to higher layers. - -### What needs adaptation around it - -Large chunks and filtered generation will benefit from operations not -currently supplied by `Raster`: - -- non-owning raster views and subwindows; -- explicit row stride where external codecs require it; -- halo/window assembly across neighbouring chunks; -- checked construction that reports allocation/dimension errors without - relying on assertions; and -- streaming or block processing when a full set of input chunks would exceed - the memory budget. - -These should be introduced only when required. They are not reasons to embed -store concepts into `Raster`. - -### What cannot be reused for final filtering - -`radix::raster::generate_mipmap` requires a square, power-of-two raster and -reduces each 2x2 group by component-wise averaging. It does not provide: - -- a selectable reconstruction or low-pass filter; -- the phase difference between vertex pixels and area pixels; -- halo samples across tile boundaries; -- linear-light colour and premultiplied-alpha handling; -- NoData-aware normalization; -- categorical reduction; or -- source-contribution tracking. - -It is therefore a useful simple raster utility, not the raster-store pyramid -generator. - -## Radix tile addressing - -### What can be reused - -`radix::tile::Id` already contains the required 2D identity: - -- zoom level; -- `x/y` coordinates; -- parent and four-child relationships; -- TMS/Slippy conversion; and -- hashing and ordering support. - -### Required hardening - -Persistent storage needs stronger invariants than the current convenience -type supplies: - -- Calling `parent()` at zoom zero currently underflows. -- Construction does not reject coordinates outside `[0, 2^z)`. -- Shifting `1u << zoom_level` limits valid conversion at high zooms. -- The scheme participates in identity, so the same spatial tile in TMS and - Slippy form becomes two keys. -- There is no persistent serialization contract or format version. - -The store should choose one canonical scheme and normalize all IDs at its API -boundary. Whether the hardening belongs in Radix or in a checked store adapter -is open. - -`radix::quad_tree::Node` is not a replacement for the index. It owns a -fully allocated group of four children whenever refined, represents no -missing child within such a group, and has no persistence or physical/virtual -status. The store requires a sparse map keyed by tile ID. - -## Tile builder and GDAL path - -### Reusable foundations - -The current tile builder already demonstrates: - -- opening GDAL raster datasets; -- reading dataset bounds; -- selecting Web Mercator or geodetic CTB grids; -- transforming arbitrary source SRS data during reads; -- calculating tile bounds and resolutions; -- enumerating intersecting `radix::tile::Id` values; and -- parallel per-tile processing. - -`ctb::GlobalMercator`, `Tiler::tile_for`, and `ParallelTiler` are useful -references and may be reused initially for grid math. - -### Required changes - -`DatasetReader` currently reads one band into a float raster, uses cubic GDAL -warping, and constructs a warped VRT for a requested output rectangle. The -store needs typed and multi-band reads, explicit alpha/NoData handling, -controlled resampling, source metadata, and deterministic alignment with the -store chunk grid. - -`Tiler` models delivery-oriented dimensions through `Border::Yes/No` and a -south/east extra pixel. Store chunks have no rendering overlap, and generated -vertex pixels require an explicit global sampling/filtering model. The border -boolean should not define store geometry. - -`ParallelTileGenerator` writes individual image files for explicitly -enumerated tiles. It is not suitable as the snapshot builder because it lacks: - -- source-map generation; -- old-snapshot reuse; -- atomic chunk containers; -- index transactions and publication; -- resumability validation; and -- bounded enumeration/streaming for very large tile sets. - -Its parallel work pattern and progress reporting may still inform the new -builder. - -## Octree index and storage - -### Reusable design - -The octree index captures the required topology semantics: - -- `Leaf`: physical payload without indexed descendants; -- `Inner`: physical payload with indexed descendants; -- `Virtual`: no physical payload, but indexed descendants; and -- absence from the map: missing node and subtree. - -Adding a physical node creates virtual ancestors and promotes a physical -ancestor from leaf to inner. Removing nodes collapses unused virtual chains. -Traversal follows only present index entries. - -The storage stack also has useful separation between: - -- logical indexed storage; -- raw path-based storage; -- disk layout; -- payload codec; and -- optional cache. - -`RawStorage_::copy_from` demonstrates hard-link reuse when source and target -extensions match. The SF merger demonstrates copying unchanged physical -subtrees and writing a new output index. - -### Why it cannot be reused unchanged - -The entire stack uses concrete `octree::Id` types. `IndexMap`, cache APIs, -layouts, filesystem path parsing, traversal, storage, serialization, and -formatting all embed this type. The coordinate-directory layout is -`level/x/y/z`, and the default codec is a mesh codec. - -The existing index file also records no tree kind. Reinterpreting its -serialized `(level, index)` octree IDs as web tiles would be unsafe. - -A 2D implementation can either: - -1. generalize the hierarchy/storage stack over an ID and layout policy; or -2. create a raster-store-specific 2D implementation using the same algorithms. - -Generalization avoids duplicate infrastructure but has a larger blast radius -in mature octree code. A separate implementation is initially safer but risks -long-term duplication. This requires an explicit decision before coding. - -### Behaviours that should not be copied blindly - -- Hard-link failure currently has no copy fallback. -- Existing indexes are trusted rather than reconciled against filesystem - contents when opened. -- An unindexed output discovers files through a final recursive directory - scan. -- Some error paths use assertions or process termination. -- The binary index lacks an explicit magic/version/topology header suitable - for a new durable format. - -The raster store should retain the useful topology and codec boundaries while -specifying stronger snapshot, validation, and error-handling rules. - -## SF builder and merger - -The SF builder's ECEF octree placement, mesh construction, texture atlases, -mask clipping, and mesh visitors are not reusable for a Web Mercator raster -store. - -The reusable ideas are architectural: - -- transform source data into a canonical spatial system during build; -- preserve physical ancestors as fallback representations; -- use an index to avoid per-node filesystem probing; -- stop refinement when a coherent representation is sufficient; and -- hard-link unchanged chunks into a new output dataset. - -Those ideas should be reimplemented against Radix tile IDs and raster payloads -rather than adapted through mesh abstractions. - -## Recommended component boundary - -The current components suggest the following dependency direction: - -```text -Radix - Raster, RasterMask, tile::Id, geometry - ↓ -Terrain library - GDAL dataset access, Web Mercator grid math, filtering primitives - ↓ -Raster store - source catalog, chunk container, sparse index, snapshots - ↓ -Builders and generators - ingestion, source selection, texture pyramids, height/geometry pyramids -``` - -The raster store should consume `radix::Raster`; Radix should not depend on -the store's source catalog, file format, or snapshot lifecycle. diff --git a/docs/raster-store/storage-format.md b/docs/raster-store/storage-format.md index 5767e0f8..5bfaf28e 100644 --- a/docs/raster-store/storage-format.md +++ b/docs/raster-store/storage-format.md @@ -1,12 +1,24 @@ # Storage format +> **Status:** future raster-store design. The completed 2D/3D store refactor +> did not implement or finalize the RF/TB formats described below. These +> requirements were not acceptance criteria for that refactor. See +> [refactor-status.md](refactor-status.md) for the implemented shared boundary. + +The 3D octree stores now use the versioned envelope formats described in +[envelope-migration-plan.md](envelope-migration-plan.md). The shared store +still supplies only an in-memory `raster_store::StoreTraits` adapter for +`radix::tile::Id`; there is currently no persistent RF index, `.amort` codec, +snapshot publisher, or RF opening API. Names and formats below remain +proposals until their own implementation plans approve them. + This document specifies the logical format and required invariants for the following data stores: - raster-fundamentalis (our authoritative raster store) - tile-base (the tile-pyramid for our tile-server) - the source-attribution table (for correct copyright attribution and data selection) ## Source-attribution table -The source attribution table is used in raster-fundamentalis, tile-base, and in abbreviated format in the delivered tiles. +The source attribution table is used in raster-fundamentalis, tile-base, and in abbreviated format in the delivered tiles. - stores a vector of structs (called Table), each struct describing one source - the struct (called Entity) contains: - the spatial resolution in pixel-width at the equator (EPSG:3857) @@ -50,7 +62,7 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - should generate tiles of requested resolution and pixel type (vertex|area) on the fly - requests by url, e.g.: layer/vertex|area/resolution/z/x/y.ending - live in src/tile-server/* -- the delivery tile format is not yet defined +- the delivery tile format is not yet defined ## serialization / deserialization envelope and versioning - `zpp::bits` serialises C++ objects to byte streams and deserialises them again. @@ -62,7 +74,7 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - `uint32 class_version`, identifying the versioned payload type; - `ChecksumAlgorithm checksum_algorithm`, default `HandledByCompressionLib`, alternatively `Crc32c` or `None`; - `string checksum`, empty when no external checksum is used, otherwise the CRC-32C value; - - `CompressionAlgorithm compression_algorithm`, default `ZstdBestCompressionWithChecksum`, alternatively `None`; + - `CompressionAlgorithm compression_algorithm`, default `ZstdDefaultCompressionWithChecksum`, alternatively `ZstdBestCompressionWithChecksum` or `None`; - `uint64 uncompressed_size`, the exact size of the uncompressed second level; - `Bytes compressed_data`, containing the second level. - The second level is a compressed byte vector. It is deserialised directly into the selected versioned payload class. @@ -81,12 +93,12 @@ tile-base is a hierarchy build from raster-fundamentalis, containing all data an - if the compression algorithm is missing or unsupported. - deserialization fails - we fail by returning an unexpected in these cases -- Compression uses libzstd at its best compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. `ZstdBestCompressionWithChecksum` writes an embedded zstd frame checksum, which libzstd verifies while decompressing. +- Compression uses libzstd at its default compression level. Libzstd is imported through the project's CMake install facility from https://github.com/AlpineMapsOrgDependencies/zstd. Both zstd modes write an embedded frame checksum, which libzstd verifies while decompressing; `ZstdBestCompressionWithChecksum` remains available for explicit callers. - `compress_with_checksum` accepts a `vector` and returns the compressed bytes plus the external checksum string. `checked_decompress` accepts both and returns the decompressed bytes. Both use `std::expected` and dispatch with a switch. - `Crc32c` is an external CRC-32C (Castagnoli) checksum of the complete uncompressed serialised payload. It uses the reflected polynomial `82F63B78`, an initial value of `FFFFFFFF`, and a final XOR of `FFFFFFFF`. The checksum string contains exactly eight lowercase hexadecimal characters. The empty payload has checksum `00000000`; the ASCII test vector `123456789` has checksum `e3069283`. CRC-32C detects accidental corruption but is not cryptographic. - `checked_decompress` accepts a maximum decompressed size, defaulting to 1 GiB. It uses a size reported by the compression format when available, otherwise it uses the maximum as its allocation bound and shrinks the result to the produced size. - Envelope deserialisation passes `uncompressed_size` as that maximum and requires the produced size to match it exactly. A mismatch is a decompression failure. A declared size above the caller's limit or the hard 1 GiB limit is a size-limit failure. -- `None` compression may be paired with `None` or `Crc32c`. `ZstdBestCompressionWithChecksum` may be paired with `HandledByCompressionLib` or `Crc32c`; with `Crc32c`, both the embedded zstd checksum and the external CRC-32C are checked. `HandledByCompressionLib` is invalid with `None` compression, and `None` checksum is invalid with `ZstdBestCompressionWithChecksum`. +- `None` compression may be paired with `None` or `Crc32c`. Either zstd mode may be paired with `HandledByCompressionLib` or `Crc32c`; with `Crc32c`, both the embedded zstd checksum and the external CRC-32C are checked. `HandledByCompressionLib` is invalid with `None` compression, and `None` checksum is invalid with a zstd mode. - `None` and `HandledByCompressionLib` require an empty checksum string. `Crc32c` requires its canonical eight-character checksum. Deserialisation verifies it after bounded decompression and before parsing the payload with `zpp::bits`; envelope deserialisation also requires the resulting size to exactly match `uncompressed_size`. - Decompression never allocates or produces output larger than 1 GiB. diff --git a/src/dag_builder/CMakeLists.txt b/src/dag_builder/CMakeLists.txt index a3ddc1df..5acf10af 100644 --- a/src/dag_builder/CMakeLists.txt +++ b/src/dag_builder/CMakeLists.txt @@ -1,17 +1,58 @@ add_library(dagbuilderlib - atlas/rect/atlas.cpp - atlas/rect/HorizontalStripPlanner.cpp - atlas/rect/PackingPlanner.cpp - merge/clusterings.cpp + ContinuationMode.h + Partitioning.h + TextureSet.h + VertexInCluster.h + VertexInClustering.h + build.h build.cpp + centroids.h + cluster.h + clusterize.h + codec.h codec.cpp + compact.h + correspondence.h + dag_id.h + dag_node.h + fixed.h + fixed_point.h + meshopt.h + metadata.h + optimize.h + parallel.h + partition.h + simplify.h + slice.h + split.h + storage.h + texture_reuse.h + texture_sizing.h + texturing.h + utils.h + validate.h + vertex_lock.h - build.cpp + atlas/BakeSource.h + atlas/bake_cluster_texture.h + atlas/pull_reproject_texture.h + atlas/rect/HorizontalStripPlanner.h atlas/rect/HorizontalStripPlanner.cpp + atlas/rect/PackingPlanner.h atlas/rect/PackingPlanner.cpp + atlas/rect/Plan.h + atlas/rect/Planner.h + atlas/rect/atlas.h atlas/rect/atlas.cpp + atlas/rect/create.h + atlas/reprojection.h + + merge/clusterings.h merge/clusterings.cpp + merge/clusters.h + merge/weld.h ) + target_include_directories(dagbuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(dagbuilderlib PUBLIC terrainlib libassert::assert metis rectpack2D meshoptimizer) if(ALP_BUILD_DAG_BUILDER) add_executable(dag-builder - cli.cpp + cli.h cli.cpp main.cpp ) target_link_libraries(dag-builder PRIVATE dagbuilderlib CLI11::CLI11) diff --git a/src/dag_builder/build.cpp b/src/dag_builder/build.cpp index 567fe005..e398e51f 100644 --- a/src/dag_builder/build.cpp +++ b/src/dag_builder/build.cpp @@ -15,7 +15,6 @@ #include "cluster.h" #include "clusterize.h" #include "compact.h" -#include "encoded.h" #include "geometry/geometry.h" #include "numeric/int_math.h" #include "log.h" @@ -26,20 +25,20 @@ #include "octree/IdRect.h" #include "octree/OddLevelShifted.h" #include "octree/Space.h" -#include "octree/Storage.h" -#include "octree/storage/open.h" -#include "octree/traverse.h" +#include "mesh/storage.h" +#include "store/traverse.h" #include "ProgressIndicator.h" #include "partition.h" #include "simplify.h" #include "slice.h" #include "range_utils.h" #include "storage.h" -#include "thread_safe_storage.h" +#include "store/ThreadSafeStorage.h" #include "utils.h" #include "vertex_lock.h" #include "parallel.h" #include "ContinuationMode.h" +#include "sf/validate_index.h" namespace dag { @@ -47,13 +46,16 @@ namespace { // Load a mesh from storage and clusterize it. std::optional load_and_clusterize_mesh( - const octree::MeshStorage &storage, + const mesh::storage::Storage &storage, const octree::Id &id) { const auto result = storage.load(id); if (!result) { - if (result.error() != mesh::io::LoadMeshErrorKind::FileNotFound) { - LOG_ERROR("Failed to read node {}: {}", id, result.error()); + if (result.error().code() != Error::Code::NotFound) { + LOG_ERROR( + "Failed to read node {}: {}", + id, + result.error().to_string()); } return std::nullopt; } @@ -120,7 +122,7 @@ LodResult build_lod( // Load input meshes, clusterize, and filter them to the target region. std::vector load_input_clusters( const std::span input_ids, - const octree::MeshStorage &input_storage, + const mesh::storage::Storage &input_storage, const RegionFilter& filter) { std::vector result; @@ -147,8 +149,8 @@ std::vector load_input_clusters( // Dependencies shared across the whole dag building pipeline. struct BuildContext { - const octree::IndexedMeshStorage &input_storage; - ThreadSafeStorage output_storage; + const mesh::storage::IndexedStorage &input_storage; + store::ThreadSafeStorage output_storage; const BuildOptions &options; const octree::Space &space; const octree::OddLevelShifted &shifted_space; @@ -447,18 +449,18 @@ LevelWorkplan build_level_workplan( } // Pre-filter input nodes to only those that intersect the target root bounds. -std::vector> gather_relevant_input_leaves( - const octree::IndexMap &index, +Expected>> gather_relevant_input_leaves( + const store::Index &index, const octree::Space &space, const radix::geometry::Aabb3d &root_bounds) { const auto start = space.find_smallest_node_encompassing_bounds(root_bounds) .value_or(octree::Id::root()); std::vector> result(octree::Id::max_level() + 1); - octree::traverse( + auto traversal = store::traverse( index, - [&](const octree::Id &id, const octree::NodeStatus status) { - if (status == octree::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { + [&](const octree::Id &id, const store::NodeStatus status) { + if (status == store::NodeStatus::Leaf && radix::geometry::intersect(root_bounds, space.get_node_bounds(id))) { result[id.level()].push_back(id); } }, @@ -466,6 +468,9 @@ std::vector> gather_relevant_input_leaves( return radix::geometry::intersect(root_bounds, space.get_node_bounds(id)); }, start); + if (!traversal) { + return Error::propagate(std::move(traversal), "gather relevant input leaves"); + } return result; } @@ -500,7 +505,7 @@ std::unordered_set build_level( std::unordered_set already_built; if (ctx.options.continuation_mode != ContinuationMode::Overwrite) { for (const octree::Id &target : targets) { - if (ctx.output_storage.has(target)) { + if (DEBUG_ASSERT_VAL(ctx.output_storage.has(target)).value()) { already_built.insert(target); } } @@ -511,12 +516,19 @@ std::unordered_set build_level( } // Initialize debug storage if requested (contains .glb meshes) - std::optional debug_storage; + std::optional> debug_storage = std::nullopt; if (ctx.options.write_debug_meshes) { - debug_storage = octree::open_folder( + mesh::storage::OpenOptions options; + options.preferred_extension = ".glb"; + auto debug_result = mesh::storage::open_folder( ctx.output_storage.base_path().string() + "-debug", - false, - octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + std::move(options)); + if (!debug_result) { + LOG_ERROR_AND_EXIT( + "Failed to open debug mesh storage: {}", + debug_result.error().to_string()); + } + debug_storage.emplace(std::move(debug_result.value())); } tbb::concurrent_vector saved_ids; @@ -538,14 +550,21 @@ std::unordered_set build_level( const auto save_result = ctx.output_storage.save(target, *result); if (save_result) { if (debug_storage) { - const auto debug_save_result = debug_storage->save(target, clustering_to_mesh(result->clustering)); - if (!debug_save_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save debug mesh for node {}: {}", target, debug_save_result.error()); + const auto debug_mesh = clustering_to_mesh(result->clustering); + const auto debug_save_result = debug_storage->save(target, debug_mesh); + if (!debug_save_result) { + LOG_ERROR_AND_EXIT( + "Failed to save debug mesh for node {}: {}", + target, + debug_save_result.error().to_string()); } } saved_ids.push_back(target); } else { - LOG_ERROR("Failed to save node {}: {}", target, save_result.error()); + LOG_ERROR( + "Failed to save node {}: {}", + target, + save_result.error().to_string()); } } progress.task_finished(); @@ -564,22 +583,30 @@ std::unordered_set build_level( // Builds the DAG from input_storage into output_storage, restricted to levels within level_range. // Iterates octree levels from finest to coarsest, simplifying and re-clustering geometry at each // level from its children. -void build_levels( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, +Expected build_levels( + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range) { + auto validation = sf::validate_index(input_storage.index()); + if (!validation) { + return validation; + } const octree::OddLevelShifted shifted_space = octree::OddLevelShifted::earth(); const octree::Space space = octree::Space::earth(); const octree::Id root_node = options.root_node; const auto root_bounds = shifted_space.get_node_bounds_with_children(root_node); - auto input_by_level = gather_relevant_input_leaves(input_storage.index(), space, root_bounds); + auto input_by_level_result = gather_relevant_input_leaves(input_storage.index(), space, root_bounds); + if (!input_by_level_result) { + return Error::propagate(std::move(input_by_level_result), "prepare input leaves for DAG build"); + } + auto input_by_level = std::move(input_by_level_result.value()); // Find the maximum input level that has any nodes auto max_input_level_opt = find_max_input_level(input_by_level); if (!max_input_level_opt.has_value()) { LOG_WARN("No input nodes found for root {}", root_node); - return; + return {}; } const uint32_t max_input_level = max_input_level_opt.value(); @@ -587,19 +614,19 @@ void build_levels( const Range range = valid_range.intersect(level_range).to_range(max_input_level + 1); if (range.is_empty()) { LOG_WARN("Requested level range does not overlap with buildable levels {}-{}", root_node.level(), max_input_level); - return; + return {}; } // Seed prev_level_built with any already-built nodes one level finer than the first level. // TODO: use hierachical lookup here based on root_bounds and IdRect std::unordered_set prev_level_built; for (const auto &[id, status] : output_storage.index()) { - if (id.level() == range.end && status != octree::NodeStatus::Virtual) { + if (id.level() == range.end && status != store::NodeStatus::Virtual) { prev_level_built.insert(id); } } - BuildContext ctx{input_storage, ThreadSafeStorage(std::move(output_storage)), options, space, shifted_space, root_bounds}; + BuildContext ctx{input_storage, store::ThreadSafeStorage(std::move(output_storage)), options, space, shifted_space, root_bounds}; for (uint32_t level = range.end; level-- > range.start;) { prev_level_built = build_level( @@ -609,20 +636,24 @@ void build_levels( ctx); // Persist per level so a finished level can be read back before the whole run completes - if (const auto result = ctx.output_storage.save_or_create_index(); !result.has_value()) { - LOG_WARN("Could not save index after level {}: {}", level, result.error()); + if (const auto result = ctx.output_storage.save_or_create_index(); !result) { + LOG_WARN( + "Could not save index after level {}: {}", + level, + result.error().to_string()); } } output_storage = std::move(ctx.output_storage).release(); + return {}; } // Builds the complete DAG from input_storage into output_storage. -void build_full( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, +Expected build_full( + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options) { - build_levels(input_storage, output_storage, options, RangeFull{}); + return build_levels(input_storage, output_storage, options, RangeFull{}); } } // namespace dag diff --git a/src/dag_builder/build.h b/src/dag_builder/build.h index d39491d6..041fad1a 100644 --- a/src/dag_builder/build.h +++ b/src/dag_builder/build.h @@ -2,13 +2,15 @@ #include #include +#include #include "ContinuationMode.h" #include "Range.h" #include "build_config.h" #include "texturing.h" -#include "octree/storage/MeshStorage.h" +#include "mesh/storage.h" #include "storage.h" +#include "Error.h" namespace dag { @@ -30,14 +32,14 @@ struct BuildOptions { ContinuationMode continuation_mode = ContinuationMode::Error; }; -void build_full( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, +Expected build_full( + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options); -void build_levels( - const octree::IndexedMeshStorage &input_storage, - octree::IndexedDagStorage &output_storage, +Expected build_levels( + const mesh::storage::IndexedStorage &input_storage, + dag::storage::IndexedStorage &output_storage, const BuildOptions &options, const AnyRange &level_range); diff --git a/src/dag_builder/codec.cpp b/src/dag_builder/codec.cpp new file mode 100644 index 00000000..f5a5badf --- /dev/null +++ b/src/dag_builder/codec.cpp @@ -0,0 +1,460 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "codec.h" + +#include "Error.h" +#include "cluster.h" +#include "dag_node.h" +#include "io/envelope.h" +#include "io/envelope_file.h" +#include "mesh/io/texture.h" +#include "meshopt.h" + +namespace dag::codec { +namespace { + + Error::Code classify_texture_error(const cv::Exception& error, const Error::Code fallback) + { + return error.code == cv::Error::StsNoMem ? Error::Code::ResourceExhausted : fallback; + } + + namespace format { + + namespace v1 { + + struct OctreeId { + std::uint8_t level; + std::uint64_t index; + }; + + struct Id { + OctreeId source_batch; + std::uint32_t cluster_index; + }; + + struct Vec3d { + double x; + double y; + double z; + }; + + struct Bounds { + Vec3d min; + Vec3d max; + }; + + struct Group { + std::vector children; + double error; + Bounds bounds; + }; + + struct NodeMetadata { + std::vector group_assignment; + std::vector groups; + }; + + struct Texture { + std::vector jpeg; + }; + + struct Cluster { + std::uint64_t triangle_count; + std::uint64_t vertex_count; + std::uint64_t uv_count; + std::vector encoded_triangles; + std::vector encoded_vertex_indices; + std::vector encoded_uvs; + std::uint32_t id; + std::uint32_t texture_id; + double absolute_error; + }; + + struct Clustering { + std::uint64_t vertex_count; + std::vector encoded_positions; + std::vector clusters; + std::vector textures; + }; + + } // namespace v1 + + using MetadataSchema = io::envelope::PayloadSchema<"dag.NodeMetadata", io::envelope::Version<1, v1::NodeMetadata>>; + using ClusteringSchema = io::envelope::PayloadSchema<"dag.Clustering", io::envelope::Version<1, v1::Clustering>>; + using NodeMetadata = MetadataSchema::latest_type; + using Clustering = ClusteringSchema::latest_type; + + inline v1::OctreeId encode_id(const octree::Id& id) { return { .level = id.level(), .index = id.index_on_level() }; } + + inline Expected decode_id(const v1::OctreeId& id) + { + auto result = octree::Id::try_make(static_cast(id.level), static_cast(id.index)); + if (!result) { + return Error::fail(Error::Code::CorruptData, "DAG payload contains an invalid octree ID"); + } + return *result; + } + + inline NodeMetadata encode_metadata(const dag::NodeMetadata& metadata) + { + NodeMetadata result; + result.group_assignment = metadata.group_assignment; + result.groups.reserve(metadata.groups.size()); + for (const dag::Group& group : metadata.groups) { + v1::Group encoded_group = { + .children = {}, + .error = group.error, + .bounds = { + .min = {group.bounds.min.x, group.bounds.min.y, group.bounds.min.z}, + .max = {group.bounds.max.x, group.bounds.max.y, group.bounds.max.z}, + }, + }; + encoded_group.children.reserve(group.children.size()); + for (const dag::Id& child : group.children) { + encoded_group.children.push_back({ + .source_batch = encode_id(child.source_batch), + .cluster_index = child.cluster_index, + }); + } + result.groups.push_back(std::move(encoded_group)); + } + return result; + } + + inline Expected validate(const NodeMetadata& metadata) + { + for (const std::uint32_t group : metadata.group_assignment) { + if (group >= metadata.groups.size()) { + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains an invalid group assignment"); + } + } + for (const v1::Group& group : metadata.groups) { + if (!std::isfinite(group.error)) { + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains a non-finite error"); + } + for (const v1::Id& child : group.children) { + if (!decode_id(child.source_batch)) { + return Error::fail(Error::Code::InvalidInput, "DAG metadata contains an invalid child ID"); + } + } + } + return {}; + } + + inline Expected decode_metadata(NodeMetadata metadata) + { + if (auto valid = validate(metadata); !valid) { + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate decoded DAG metadata"); + } + dag::NodeMetadata result; + result.group_assignment = std::move(metadata.group_assignment); + result.groups.reserve(metadata.groups.size()); + for (v1::Group& group : metadata.groups) { + dag::Group decoded_group = { + .children = {}, + .error = group.error, + .bounds = { + {group.bounds.min.x, group.bounds.min.y, group.bounds.min.z}, + {group.bounds.max.x, group.bounds.max.y, group.bounds.max.z}, + }, + }; + decoded_group.children.reserve(group.children.size()); + for (const v1::Id& child : group.children) { + auto source_batch = decode_id(child.source_batch); + if (!source_batch) { + return Error::propagate(std::move(source_batch), "decode DAG metadata child ID"); + } + decoded_group.children.push_back({ *source_batch, child.cluster_index }); + } + result.groups.push_back(std::move(decoded_group)); + } + return result; + } + + inline Expected encode_clustering(const ::Clustering& clustering) + { + Clustering result { + .vertex_count = clustering.positions.size(), + .encoded_positions = meshopt::encode_vertex_buffer(clustering.positions), + .clusters = {}, + .textures = {}, + }; + result.clusters.reserve(clustering.clusters.size()); + for (const ::Cluster& cluster : clustering.clusters) { + if (cluster.texture_id == (std::numeric_limits::max)()) { + return Error::fail(Error::Code::InvalidInput, "DAG texture ID collides with the on-disk null sentinel"); + } + result.clusters.push_back({ + .triangle_count = cluster.local_triangles.size(), + .vertex_count = cluster.vertex_indices.size(), + .uv_count = cluster.uvs.size(), + .encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles), + .encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices), + .encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs), + .id = cluster.id, + .texture_id = cluster.texture_id.value_or((std::numeric_limits::max)()), + .absolute_error = cluster.absolute_error, + }); + } + result.textures.reserve(clustering.textures.size()); + try { + for (const cv::Mat& texture : clustering.textures) { + auto jpeg = mesh::io::write_texture_to_encoded_buffer(texture, ".jpeg"); + if (!texture.empty() && jpeg.empty()) { + return Error::fail(Error::Code::InvalidInput, "could not encode DAG texture"); + } + result.textures.push_back({ std::move(jpeg) }); + } + } catch (const cv::Exception& error) { + return Error::fail(classify_texture_error(error, Error::Code::InvalidInput), "could not encode DAG texture: " + error.msg); + } + return result; + } + + inline bool count_fits(const std::uint64_t count, const std::size_t element_size) + { + return count <= std::numeric_limits::max() / element_size && count * element_size <= io::envelope::default_max_decompressed_size; + } + + inline Expected validate(const Clustering& clustering) + { + if (!count_fits(clustering.vertex_count, sizeof(glm::dvec3))) { + return Error::fail(Error::Code::InvalidInput, "DAG position count exceeds the allocation limit"); + } + if (clustering.vertex_count != 0 && clustering.encoded_positions.empty()) { + return Error::fail(Error::Code::InvalidInput, "DAG position count and encoded data disagree"); + } + std::size_t decoded_size = static_cast(clustering.vertex_count) * sizeof(glm::dvec3); + for (const v1::Cluster& cluster : clustering.clusters) { + if (!count_fits(cluster.triangle_count, sizeof(glm::uvec3)) || !count_fits(cluster.vertex_count, sizeof(std::uint32_t)) + || !count_fits(cluster.uv_count, sizeof(glm::dvec2))) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster count exceeds the allocation limit"); + } + const std::size_t cluster_size = static_cast(cluster.triangle_count) * sizeof(glm::uvec3) + + static_cast(cluster.vertex_count) * sizeof(std::uint32_t) + static_cast(cluster.uv_count) * sizeof(glm::dvec2); + if (cluster_size > io::envelope::default_max_decompressed_size - decoded_size) { + return Error::fail(Error::Code::InvalidInput, "DAG decoded data exceeds the allocation limit"); + } + decoded_size += cluster_size; + if (cluster.uv_count != 0 && cluster.uv_count != cluster.vertex_count) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster UV count does not match its vertex count"); + } + if ((cluster.triangle_count != 0 && cluster.encoded_triangles.empty()) || (cluster.vertex_count != 0 && cluster.encoded_vertex_indices.empty()) + || (cluster.uv_count != 0 && cluster.encoded_uvs.empty())) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster count and encoded data disagree"); + } + if (cluster.texture_id != (std::numeric_limits::max)() && cluster.texture_id >= clustering.textures.size()) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster contains an invalid texture reference"); + } + if (!std::isfinite(cluster.absolute_error)) { + return Error::fail(Error::Code::InvalidInput, "DAG cluster contains a non-finite error"); + } + } + return {}; + } + + template + inline Expected> decode_vertex_buffer(const std::uint64_t count, const std::vector& encoded, const std::string_view label) + { + if (!count_fits(count, sizeof(T))) { + return Error::fail(Error::Code::CorruptData, std::string(label) + " count exceeds the allocation limit"); + } + std::vector result(static_cast(count)); + if (meshopt_decodeVertexBuffer(result.data(), result.size(), sizeof(T), encoded.data(), encoded.size()) != 0) { + return Error::fail(Error::Code::CorruptData, "could not decode " + std::string(label)); + } + return result; + } + + inline Expected> decode_triangles(const std::uint64_t count, const std::vector& encoded) + { + if (!count_fits(count, sizeof(glm::uvec3))) { + return Error::fail(Error::Code::CorruptData, "DAG triangle count exceeds the allocation limit"); + } + std::vector result(static_cast(count)); + if (meshopt_decodeIndexBuffer(result.data(), result.size() * 3, sizeof(std::uint32_t), encoded.data(), encoded.size()) != 0) { + return Error::fail(Error::Code::CorruptData, "could not decode DAG triangles"); + } + return result; + } + + inline Expected<::Clustering> decode_clustering(Clustering clustering) + { + if (auto valid = validate(clustering); !valid) { + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate decoded DAG clustering"); + } + auto positions = decode_vertex_buffer(clustering.vertex_count, clustering.encoded_positions, "DAG positions"); + if (!positions) { + return Error::propagate(std::move(positions), "decode DAG positions"); + } + + ::Clustering result; + result.positions = std::move(*positions); + result.clusters.reserve(clustering.clusters.size()); + for (const v1::Cluster& encoded : clustering.clusters) { + auto triangles = decode_triangles(encoded.triangle_count, encoded.encoded_triangles); + auto vertex_indices = decode_vertex_buffer(encoded.vertex_count, encoded.encoded_vertex_indices, "DAG vertex indices"); + auto uvs = decode_vertex_buffer(encoded.uv_count, encoded.encoded_uvs, "DAG UVs"); + if (!triangles) { + return Error::propagate(std::move(triangles), "decode DAG cluster triangles"); + } + if (!vertex_indices) { + return Error::propagate(std::move(vertex_indices), "decode DAG cluster vertex indices"); + } + if (!uvs) { + return Error::propagate(std::move(uvs), "decode DAG cluster UVs"); + } + for (const std::uint32_t vertex : *vertex_indices) { + if (vertex >= result.positions.size()) { + return Error::fail(Error::Code::CorruptData, "DAG cluster contains an invalid global vertex index"); + } + } + for (const glm::uvec3 triangle : *triangles) { + if (glm::any(glm::greaterThanEqual(triangle, glm::uvec3(static_cast(vertex_indices->size()))))) { + return Error::fail(Error::Code::CorruptData, "DAG cluster contains an invalid local triangle index"); + } + } + result.clusters.push_back({ + .id = encoded.id, + .vertex_indices = std::move(*vertex_indices), + .local_triangles = std::move(*triangles), + .uvs = std::move(*uvs), + .texture_id + = encoded.texture_id == (std::numeric_limits::max)() ? std::nullopt : std::optional { encoded.texture_id }, + .absolute_error = encoded.absolute_error, + }); + } + + std::vector textures; + textures.reserve(clustering.textures.size()); + try { + for (const v1::Texture& texture : clustering.textures) { + cv::Mat image = mesh::io::read_texture_from_encoded_bytes(texture.jpeg); + if (image.empty()) { + return Error::fail(Error::Code::CorruptData, "could not decode DAG texture"); + } + textures.push_back(std::move(image)); + } + } catch (const cv::Exception& error) { + return Error::fail(classify_texture_error(error, Error::Code::CorruptData), "could not decode DAG texture: " + error.msg); + } + result.textures = TextureSet(std::move(textures)); + return result; + } + + } // namespace format + + Expected validate_batch(const dag::ClusterBatch& batch, const Error::Code code) + { + if (batch.metadata.group_assignment.size() != batch.clustering.clusters.size()) { + return Error::fail(code, "DAG metadata group-assignment count does not match clustering cluster count"); + } + return {}; + } + + Expected read_metadata(const std::filesystem::path& path) + { + auto payload = io::envelope::read_from_path(path); + if (!payload) { + return Error::propagate(std::move(payload), "read DAG metadata"); + } + auto metadata = format::decode_metadata(std::move(*payload)); + if (!metadata) { + return Error::propagate(std::move(metadata), "decode DAG metadata read from \"" + path.string() + "\""); + } + return std::move(*metadata); + } + +} // namespace + +std::vector ClusterBatch::paths(const std::filesystem::path& node_path) const +{ + return { + add_extension(node_path, ".dag"), + add_extension(node_path, ".dagmeta"), + }; +} + +Expected ClusterBatch::read(const std::filesystem::path& node_path) const +{ + const auto node_paths = paths(node_path); + auto clustering_payload = io::envelope::read_from_path(node_paths[0]); + if (!clustering_payload) { + return Error::propagate(std::move(clustering_payload), "read DAG clustering"); + } + auto metadata = read_metadata(node_paths[1]); + if (!metadata) { + return Error::propagate(std::move(metadata)); + } + auto clustering = format::decode_clustering(std::move(*clustering_payload)); + if (!clustering) { + return Error::propagate(std::move(clustering), "decode DAG clustering read from \"" + node_paths[0].string() + "\""); + } + dag::ClusterBatch batch { + .metadata = std::move(*metadata), + .clustering = std::move(*clustering), + }; + if (auto valid = validate_batch(batch, Error::Code::CorruptData); !valid) { + return Error::propagate(std::move(valid), "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); + } + return batch; +} + +Expected ClusterBatch::write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const +{ + const auto node_paths = paths(node_path); + if (auto valid = validate_batch(batch, Error::Code::InvalidInput); !valid) { + return Error::propagate(std::move(valid), "validate DAG files \"" + node_paths[0].string() + "\" and \"" + node_paths[1].string() + "\""); + } + auto clustering = format::encode_clustering(batch.clustering); + if (!clustering) { + return Error::propagate(std::move(clustering), "encode DAG clustering for \"" + node_paths[0].string() + "\""); + } + const format::NodeMetadata metadata = format::encode_metadata(batch.metadata); + if (auto valid = format::validate(*clustering); !valid) { + return Error::propagate(std::move(valid), "validate DAG clustering for \"" + node_paths[0].string() + "\""); + } + if (auto valid = format::validate(metadata); !valid) { + return Error::propagate(std::move(valid), "validate DAG metadata for \"" + node_paths[1].string() + "\""); + } + auto data_written = io::envelope::write_to_path(*clustering, node_paths[0]); + if (!data_written) { + return Error::propagate(std::move(data_written), "write DAG clustering"); + } + auto metadata_written = io::envelope::write_to_path(metadata, node_paths[1]); + if (!metadata_written) { + return Error::propagate(std::move(metadata_written), "write DAG metadata"); + } + return {}; +} + +std::vector Metadata::paths(const std::filesystem::path& node_path) const { return { add_extension(node_path, ".dagmeta") }; } + +Expected Metadata::read(const std::filesystem::path& node_path) const { return read_metadata(paths(node_path).front()); } + +Expected>> cluster_batch_from_extension(const std::string_view extension) +{ + if (extension == ".dag") { + return std::make_unique(); + } + return Error::fail(Error::Code::Unsupported, "unsupported DAG codec selector: " + std::string(extension)); +} + +Expected>> metadata_from_extension(const std::string_view extension) +{ + if (extension == ".dag") { + return std::make_unique(); + } + return Error::fail(Error::Code::Unsupported, "unsupported DAG metadata codec selector: " + std::string(extension)); +} + +} // namespace dag::codec diff --git a/src/dag_builder/codec.h b/src/dag_builder/codec.h new file mode 100644 index 00000000..8510cac7 --- /dev/null +++ b/src/dag_builder/codec.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include + +#include "dag_node.h" +#include "store/Codec.h" + +namespace dag::codec { + +class ClusterBatch final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override; + Expected read(const std::filesystem::path& node_path) const override; + Expected write(const std::filesystem::path& node_path, const dag::ClusterBatch& batch) const override; +}; + +class Metadata final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override; + Expected read(const std::filesystem::path& node_path) const override; +}; + +Expected>> cluster_batch_from_extension(std::string_view extension); +Expected>> metadata_from_extension(std::string_view extension); + +} // namespace dag::codec diff --git a/src/dag_builder/dag_id.h b/src/dag_builder/dag_id.h index 232f66fd..3370d1bf 100644 --- a/src/dag_builder/dag_id.h +++ b/src/dag_builder/dag_id.h @@ -2,8 +2,6 @@ #include -#include - #include "octree/Id.h" #include "hash_utils.h" @@ -33,17 +31,3 @@ struct fmt::formatter { return fmt::format_to(ctx.out(), "{}:{}", id.source_batch, id.cluster_index); } }; - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -template -auto serialize(Archive &archive, dag::Id &id) { - return archive(id.source_batch, id.cluster_index); -} - -} diff --git a/src/dag_builder/dag_node.h b/src/dag_builder/dag_node.h index c276a27a..2b46e6ec 100644 --- a/src/dag_builder/dag_node.h +++ b/src/dag_builder/dag_node.h @@ -18,17 +18,3 @@ inline ClusterBatch make_leaf_batch(Clustering clustering) { } } - -namespace dag { - -template -auto serialize(Archive &archive, const dag::ClusterBatch &node) { - return archive(node.metadata, node.clustering); -} - -template -auto serialize(Archive &archive, dag::ClusterBatch &node) { - return archive(node.metadata, node.clustering); -} - -} // namespace dag diff --git a/src/dag_builder/encoded.h b/src/dag_builder/encoded.h deleted file mode 100644 index 2ccee0e9..00000000 --- a/src/dag_builder/encoded.h +++ /dev/null @@ -1,178 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "TextureSet.h" -#include "cluster.h" -#include "io/bytes.h" -#include "io/serialize.h" -#include "mesh/io/texture.h" -#include "meshopt.h" - -template -auto serialize(Archive &archive, const TextureSet &textures) { - size_t size = textures.size(); - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - for (const auto &texture : textures) { - const mesh::io::ImageAndExt item{texture, ".jpeg"}; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - } - - return result; - } -} -template -auto serialize(Archive &archive, TextureSet &textures) { - size_t size; - auto result = archive(size); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - std::vector images; - images.reserve(size); - for (size_t i = 0; i < size; i++) { - mesh::io::ImageAndExt item; - result = archive(item); - if (zpp::bits::failure(result)) { - return result; - } - - images.push_back(item.image); - } - textures = TextureSet(images); - - return result; - } -} - -template -auto serialize(Archive &archive, const Cluster &cluster) { - const size_t triangle_count = cluster.triangle_count(); - const size_t vertex_count = cluster.vertex_count(); - const size_t uv_count = cluster.uvs.size(); - - auto encoded_triangles = meshopt::encode_index_buffer(cluster.local_triangles); - auto encoded_vertex_indices = meshopt::encode_vertex_buffer(cluster.vertex_indices); - auto encoded_uvs = meshopt::encode_vertex_buffer(cluster.uvs); - - return archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); -} -template -auto serialize(Archive &archive, Cluster &cluster) { - size_t triangle_count; - size_t vertex_count; - size_t uv_count; - - std::vector encoded_triangles; - std::vector encoded_vertex_indices; - std::vector encoded_uvs; - - auto result = archive( - triangle_count, - vertex_count, - uv_count, - encoded_triangles, - encoded_vertex_indices, - encoded_uvs, - cluster.id, - cluster.texture_id, - cluster.absolute_error); - - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - meshopt::decode_index_buffer(cluster.local_triangles, triangle_count, encoded_triangles); - meshopt::decode_vertex_buffer(cluster.vertex_indices, vertex_count, encoded_vertex_indices); - meshopt::decode_vertex_buffer(cluster.uvs, uv_count, encoded_uvs); - - return result; - } -} - -template -auto serialize(Archive &archive, const Clustering &clustering) { - const size_t vertex_count = clustering.vertex_count(); - auto encoded_positions = meshopt::encode_vertex_buffer(clustering.positions); - - return archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); -} - -template -auto serialize(Archive &archive, Clustering &clustering) { - size_t vertex_count; - std::vector encoded_positions; - - auto result = archive( - vertex_count, - encoded_positions, - clustering.clusters, - clustering.textures); - - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - meshopt::decode_vertex_buffer(clustering.positions, vertex_count, encoded_positions); - - return result; - } -} - -inline std::expected -save_clustering(const Clustering &clustering, - const std::filesystem::path &path, - const bool make_dirs = true) { - return ::io::write_to_path(clustering, path, make_dirs); -} - -inline std::expected -load_clustering(const std::filesystem::path &path) { - return ::io::read_from_path(path); -} diff --git a/src/dag_builder/main.cpp b/src/dag_builder/main.cpp index acce9d48..177924b7 100644 --- a/src/dag_builder/main.cpp +++ b/src/dag_builder/main.cpp @@ -5,9 +5,7 @@ #include "cli.h" #include "build.h" #include "log.h" -#include "octree/storage/Storage.h" -#include "octree/storage/MeshStorage.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" #include "storage.h" #include "ContinuationMode.h" @@ -17,8 +15,24 @@ int main(int argc, char **argv) { Log::init(args.log_level); try { - const octree::IndexedMeshStorage input_storage = octree::open_folder_indexed(args.input_path); - octree::IndexedDagStorage output_storage = octree::open_folder_indexed(args.output_path); + auto input_result = mesh::storage::open_folder_indexed(args.input_path); + if (!input_result) { + LOG_ERROR( + "Failed to open input dataset {}: {}", + args.input_path, + input_result.error().to_string()); + return EXIT_FAILURE; + } + auto output_result = dag::storage::open_folder_indexed(args.output_path); + if (!output_result) { + LOG_ERROR( + "Failed to open output dataset {}: {}", + args.output_path, + output_result.error().to_string()); + return EXIT_FAILURE; + } + const mesh::storage::IndexedStorage input_storage = std::move(input_result.value()); + dag::storage::IndexedStorage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = args.continuation_mode == ContinuationMode::Overwrite; dag::BuildOptions options{ @@ -43,10 +57,21 @@ int main(int argc, char **argv) { options.relative_target_error = 0.001f; } - dag::build_levels(input_storage, output_storage, options, args.level_range); + const auto build_result = dag::build_levels( + input_storage, + output_storage, + options, + args.level_range); + if (!build_result) { + LOG_ERROR("Invalid Structura Fundamentalis input: {}", build_result.error().to_string()); + return EXIT_FAILURE; + } const auto index_result = output_storage.save_index(); - if (!index_result.has_value()) { - LOG_ERROR("Failed to save output index in {}: {}", args.output_path, index_result.error()); + if (!index_result) { + LOG_ERROR( + "Failed to save output index in {}: {}", + args.output_path, + index_result.error().to_string()); return EXIT_FAILURE; } diff --git a/src/dag_builder/metadata.h b/src/dag_builder/metadata.h index 5527fe3a..e5c5d35e 100644 --- a/src/dag_builder/metadata.h +++ b/src/dag_builder/metadata.h @@ -8,10 +8,6 @@ #include "cluster.h" #include "dag_id.h" #include "utils.h" -#include "zpp_bits_glm.h" -#include "octree/storage/defaults.h" -#include "octree/storage/codec/ZppBitsCodec.h" -#include "octree/storage/codec/ReadOnlyCodec.h" namespace dag { @@ -72,42 +68,3 @@ inline const radix::geometry::Aabb3d &get_group_bounds(const NodeMetadata &metad } } // namespace dag - -// Set default codec for metadata. -namespace octree { -template <> -struct DefaultCodec { - using type = ReadOnlyCodec>; -}; -} - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} -template -auto serialize(Archive &archive, radix::geometry::Aabb3d &bounds) { - return archive(bounds.min, bounds.max); -} - -template -auto serialize(Archive &archive, const dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} -template -auto serialize(Archive &archive, dag::Group &group) { - return archive(group.children, group.error, group.bounds); -} - -template -auto serialize(Archive &archive, const dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} -template -auto serialize(Archive &archive, dag::NodeMetadata &metadata) { - return archive(metadata.group_assignment, metadata.groups); -} - -} // namespace zpp::bits diff --git a/src/dag_builder/storage.h b/src/dag_builder/storage.h index 8c520529..76b89b5f 100644 --- a/src/dag_builder/storage.h +++ b/src/dag_builder/storage.h @@ -1,15 +1,66 @@ #pragma once -#include "dag_node.h" -#include "metadata.h" -#include "octree/storage/Storage.h" -#include "octree/storage/IndexedStorage.h" -namespace octree { +#include +#include -using DagStorage = Storage_; -using IndexedDagStorage = IndexedStorage_; +#include "codec.h" +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open_runtime.h" +#include "store/IndexedStorage.h" -using DagMetaStorage = Storage_; -using IndexedDagMetaStorage = IndexedStorage_; +namespace dag::storage { +inline constexpr std::string_view payload_class = "dag.ClusterBatch"; + +using OpenOptions = octree::storage::OpenOptions; +using Storage = store::Storage; +using IndexedStorage = store::IndexedStorage; +using MetadataStorage = store::Storage; +using IndexedMetadataStorage = + store::IndexedStorage; + +inline Expected open_index( + const std::filesystem::path &path) { + return store::open_index( + path, + octree::storage::index_format(), + payload_class, + dag::codec::cluster_batch_from_extension); } + +inline Expected open_folder( + const std::filesystem::path &path, + OpenOptions options = {}) { + return octree::storage::open_folder( + path, + std::string(payload_class), + ".dag", + dag::codec::cluster_batch_from_extension, + std::move(options)); +} + +inline Expected +open_folder_indexed(const std::filesystem::path &path, OpenOptions options = {}) { + return octree::storage::open_folder_indexed( + path, + std::string(payload_class), + ".dag", + dag::codec::cluster_batch_from_extension, + std::move(options)); +} + +inline Expected +open_metadata_indexed(const std::filesystem::path &path) { + const std::filesystem::path index_path = + path.filename() == octree::storage::index_file_name + ? path + : path / octree::storage::index_file_name; + return store::open_index( + index_path, + octree::storage::index_format(), + payload_class, + dag::codec::metadata_from_extension); +} + +} // namespace dag::storage diff --git a/src/dag_builder/thread_safe_storage.h b/src/dag_builder/thread_safe_storage.h deleted file mode 100644 index 9b5e6120..00000000 --- a/src/dag_builder/thread_safe_storage.h +++ /dev/null @@ -1,65 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" - -namespace dag { - -// A mutex-based thread-safe wrapper around a Storage_ instance. -// TODO: move this locking into Storage_ itself and remove this wrapper once that lands. -template -class ThreadSafeStorage { - static_assert(!std::is_const_v, "ThreadSafeStorage requires a non-const Storage"); - -public: - using value_type = typename Storage::value_type; - using load_error = typename Storage::load_error; - using save_error = typename Storage::save_error; - - explicit ThreadSafeStorage(Storage &&storage) - : _storage(std::move(storage)) {} - - ThreadSafeStorage(const ThreadSafeStorage &) = delete; - ThreadSafeStorage &operator=(const ThreadSafeStorage &) = delete; - ThreadSafeStorage(ThreadSafeStorage &&) = delete; - ThreadSafeStorage &operator=(ThreadSafeStorage &&) = delete; - - Storage release() && { - return std::move(this->_storage); - } - - std::expected load(const octree::Id &id) const { - std::shared_lock lock(this->_mutex); - return this->_storage.load(id); - } - - bool has(const octree::Id &id) const { - std::shared_lock lock(this->_mutex); - return this->_storage.has(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_storage.base_path(); - } - - std::expected save(const octree::Id &id, const value_type &value) const { - std::unique_lock lock(this->_mutex); - return this->_storage.save(id, value); - } - - auto save_or_create_index() const { - std::unique_lock lock(this->_mutex); - return this->_storage.save_or_create_index(); - } - -private: - mutable Storage _storage; - mutable std::shared_mutex _mutex; -}; - -} // namespace dag diff --git a/src/dag_builder/zpp_bits_glm.h b/src/dag_builder/zpp_bits_glm.h deleted file mode 100644 index 0dade30d..00000000 --- a/src/dag_builder/zpp_bits_glm.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include "glm_utils.h" - -namespace zpp::bits { - -template -auto serialize(Archive &archive, const glm::vec &v) { - return archive(as_span(v)); -} - -template -auto serialize(Archive &archive, glm::vec &v) { - return archive(as_span(v)); -} - -} diff --git a/src/dag_convert_debug/CMakeLists.txt b/src/dag_convert_debug/CMakeLists.txt index ade99815..d94dc228 100644 --- a/src/dag_convert_debug/CMakeLists.txt +++ b/src/dag_convert_debug/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable(dag-convert-debug - cli.cpp + cli.h cli.cpp main.cpp ) target_link_libraries(dag-convert-debug PRIVATE dagbuilderlib CLI11::CLI11) diff --git a/src/dag_convert_debug/main.cpp b/src/dag_convert_debug/main.cpp index 88f776fa..b0f8c8c5 100644 --- a/src/dag_convert_debug/main.cpp +++ b/src/dag_convert_debug/main.cpp @@ -2,13 +2,12 @@ #include #include "cli.h" +#include "codec.h" #include "dag_node.h" -#include "encoded.h" #include "log.h" #include "mesh/io.h" -#include "octree/storage/MeshStorage.h" -#include "octree/storage/codec/ZppBitsCodec.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" +#include "mesh/storage.h" #include "ProgressIndicator.h" #include "storage.h" #include "utils.h" @@ -16,27 +15,51 @@ namespace { void export_node(const cli::Args &args) { - const auto load_result = octree::ZppBitsCodec::load_from_path(args.input_path); - if (!load_result.has_value()) { - LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error()); + if (args.input_path.extension() != ".dag") { + LOG_ERROR("Expected a .dag input file, got {}", args.input_path); + return; + } + std::filesystem::path node_path = args.input_path; + node_path.replace_extension(); + const dag::codec::ClusterBatch codec; + const auto load_result = codec.read(node_path); + if (!load_result) { + LOG_ERROR("Failed to load node from {}: {}", args.input_path, load_result.error().to_string()); return; } const mesh::Simple mesh = clustering_to_mesh(load_result.value().clustering); const auto save_result = mesh::io::save_to_path(mesh, args.output_path); - if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh to {}: {}", args.output_path, save_result.error().description()); + if (!save_result) { + LOG_ERROR("Failed to save mesh to {}: {}", args.output_path, save_result.error().to_string()); } } void export_storage(const cli::Args &args) { - const octree::IndexedDagStorage input_storage = octree::open_folder_indexed(args.input_path); + auto input_result = dag::storage::open_folder_indexed(args.input_path); + if (!input_result) { + LOG_ERROR( + "Failed to open input storage {}: {}", + args.input_path, + input_result.error().to_string()); + return; + } + const dag::storage::IndexedStorage input_storage = std::move(input_result.value()); - octree::MeshStorage output_storage = octree::open_folder( + mesh::storage::OpenOptions options; + options.preferred_extension = ".glb"; + auto output_result = mesh::storage::open_folder( args.output_path, - false, - octree::OpenOptions{.preferred_extension_with_dot = ".glb"}); + std::move(options)); + if (!output_result) { + LOG_ERROR( + "Failed to open output storage {}: {}", + args.output_path, + output_result.error().to_string()); + return; + } + mesh::storage::Storage output_storage = std::move(output_result.value()); output_storage.settings().allow_overwrite = true; size_t exported_count = 0; @@ -45,14 +68,17 @@ void export_storage(const cli::Args &args) { auto progress_thread = progress.start_monitoring(); for (const auto &[id, status] : input_storage.index()) { - if (status == octree::NodeStatus::Virtual) { + if (status == store::NodeStatus::Virtual) { progress.task_finished(); continue; } const auto load_result = input_storage.load(id); - if (!load_result.has_value()) { - LOG_ERROR("Failed to load node {}: {}", id, load_result.error()); + if (!load_result) { + LOG_ERROR( + "Failed to load node {}: {}", + id, + load_result.error().to_string()); progress.task_finished(); continue; } @@ -60,8 +86,11 @@ void export_storage(const cli::Args &args) { const mesh::Simple mesh = clustering_to_mesh(load_result.value().clustering); const auto save_result = output_storage.save(id, mesh); - if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh for node {}: {}", id, save_result.error().description()); + if (!save_result) { + LOG_ERROR( + "Failed to save mesh for node {}: {}", + id, + save_result.error().to_string()); progress.task_finished(); continue; } @@ -73,8 +102,11 @@ void export_storage(const cli::Args &args) { progress_thread.join(); const auto index_result = output_storage.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR("Failed to save index for {}: {}", args.output_path, index_result.error()); + if (!index_result) { + LOG_ERROR( + "Failed to save index for {}: {}", + args.output_path, + index_result.error().to_string()); } LOG_INFO("Exported {} debug meshes to {}", exported_count, args.output_path); diff --git a/src/mesh_convert/CMakeLists.txt b/src/mesh_convert/CMakeLists.txt index 8779e0f7..01a98169 100644 --- a/src/mesh_convert/CMakeLists.txt +++ b/src/mesh_convert/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable(mesh-convert - cli.cpp + cli.h cli.cpp main.cpp ) target_link_libraries(mesh-convert PRIVATE terrainlib CLI11::CLI11) diff --git a/src/mesh_convert/main.cpp b/src/mesh_convert/main.cpp index a396ce50..a2f742ab 100644 --- a/src/mesh_convert/main.cpp +++ b/src/mesh_convert/main.cpp @@ -1,7 +1,6 @@ #include #include -#include #include "mesh/SimpleMesh.h" #include "mesh/io.h" @@ -11,9 +10,9 @@ void run(const cli::Args& args) { LOG_INFO("Loading input mesh..."); - const std::expected load_result = mesh::io::load_from_path(args.input_path); - if (!load_result.has_value()) { - LOG_ERROR("Failed to load mesh: {}", load_result.error().description()); + const auto load_result = mesh::io::load_from_path(args.input_path); + if (!load_result) { + LOG_ERROR("Failed to load mesh: {}", load_result.error().to_string()); return; } SimpleMesh mesh = load_result.value(); @@ -30,9 +29,9 @@ void run(const cli::Args& args) { } LOG_INFO("Writing output mesh..."); - const std::expected save_result = mesh::io::save_to_path(mesh, args.output_path); - if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh: {}", save_result.error().description()); + const auto save_result = mesh::io::save_to_path(mesh, args.output_path); + if (!save_result) { + LOG_ERROR("Failed to save mesh: {}", save_result.error().to_string()); return; } } diff --git a/src/sf_browser/CMakeLists.txt b/src/sf_browser/CMakeLists.txt index 8ed300c6..d3f379ce 100644 --- a/src/sf_browser/CMakeLists.txt +++ b/src/sf_browser/CMakeLists.txt @@ -2,16 +2,18 @@ add_subdirectory(res) add_executable(sf-browser main.cpp - core/Application.cpp - core/window/Window.cpp - core/shader/Shader.cpp - core/shader/ShaderProgram.cpp - core/shader/Uniform.h - core/shader/GLUniformAbstractions.h - core/Buffer.cpp - core/Camera.cpp + + core/Application.h core/Application.cpp + core/Buffer.h core/Buffer.cpp + core/Camera.h core/Camera.cpp core/geometry/UnitCube.h - core/rendering/OctreeRenderManager.cpp + core/rendering/OctreeRenderManager.h core/rendering/OctreeRenderManager.cpp + core/shader/GLUniformAbstractions.h + core/shader/Shader.h core/shader/Shader.cpp + core/shader/ShaderProgram.h core/shader/ShaderProgram.cpp + core/shader/Uniform.h + core/window/Window.h core/window/Window.cpp ) + target_include_directories(sf-browser PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(sf-browser PRIVATE terrainlib glfw glad_gl_core_46 sf-browser-resources imgui) diff --git a/src/sf_builder/CMakeLists.txt b/src/sf_builder/CMakeLists.txt index cfdabb70..4a32f9cc 100644 --- a/src/sf_builder/CMakeLists.txt +++ b/src/sf_builder/CMakeLists.txt @@ -1,7 +1,12 @@ add_library(sfbuilderlib - terrainbuilder.cpp - mesh_builder.cpp + border.h + mesh_builder.h mesh_builder.cpp + raw_dataset_reader.h + terrainbuilder.h terrainbuilder.cpp + texture_assembler.h + tile_provider.h ) + target_include_directories(sfbuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(sfbuilderlib PUBLIC terrainlib spdlog) diff --git a/src/sf_builder/main.cpp b/src/sf_builder/main.cpp index e7216346..888c2af4 100644 --- a/src/sf_builder/main.cpp +++ b/src/sf_builder/main.cpp @@ -25,7 +25,7 @@ OGRSpatialReference parse_srs(const std::string &user_input) { const auto result = srs::from_user_input(user_input); - if (!result.has_value()) { + if (!result) { LOG_ERROR(result.error()); exit(1); } @@ -231,7 +231,7 @@ int run(std::span args) { ->expected(2, 4); target->require_option(1); - single->add_option("--output", output_path, "Output path were the mesh is written to (.terrain, .gltf or .glb)") + single->add_option("--output", output_path, "Output path were the mesh is written to (.sfmesh, .gltf or .glb)") ->required(); single->add_option("--srs", target_srs_input, "EPSG code of the srs of the target bounds or id"); @@ -259,8 +259,8 @@ int run(std::span args) { ->required(); std::string output_format; batch->add_option("--format", output_format, "Output mesh format") - ->check(CLI::IsMember({".glb", ".gltf", ".terrain"})) - ->default_val(".terrain"); + ->check(CLI::IsMember({".glb", ".gltf", ".sfmesh"})) + ->default_val(".sfmesh"); uint32_t num_threads = 0; batch->add_option("--threads", num_threads, "Number of threads to use") ->check(CLI::PositiveNumber); @@ -317,7 +317,7 @@ int run(std::span args) { const auto maybe_s = actual_num_threads == 1 ? "" : "s"; LOG_INFO("Using {} thread{} for batch processing.", actual_num_threads, maybe_s); - terrainbuilder::build_all_patches( + const auto build_result = terrainbuilder::build_all_patches( dataset, target_level, texture_srs, @@ -326,6 +326,12 @@ int run(std::span args) { output_base_path, output_format, overwrite_existing); + if (!build_result) { + LOG_ERROR( + "Failed to finalize Structura Fundamentalis output: {}", + build_result.error().to_string()); + return 1; + } } return 0; diff --git a/src/sf_builder/terrainbuilder.cpp b/src/sf_builder/terrainbuilder.cpp index 083ab6ba..6195b213 100644 --- a/src/sf_builder/terrainbuilder.cpp +++ b/src/sf_builder/terrainbuilder.cpp @@ -41,9 +41,9 @@ #include "octree/Id.h" #include "octree/Space.h" -#include "octree/Storage.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" #include "octree/utils.h" +#include "store/ThreadSafeStorage.h" +#include "sf/finalize_storage.h" namespace terrainbuilder { @@ -75,7 +75,7 @@ std::optional build_patch( mesh_srs, target_bounds_srs, target_bounds, texture_srs, texture_bounds); - if (!mesh_result.has_value()) { + if (!mesh_result) { const BuildMeshError error = mesh_result.error(); if (error == BuildMeshError::OutOfBounds) { const radix::tile::SrsBounds dataset_bounds = dataset.bounds(); @@ -154,8 +154,9 @@ void build_and_save_patch( // metadata["texture_bounds"] = fmt::format( // "{{ \"min\": {{ \"x\": {}, \"y\": {} }}, \"max\": {{ \"x\": {}, \"y\": {} }} }}", // texture_bounds.min.x, texture_bounds.min.y, texture_bounds.max.x, texture_bounds.max.y); - if (!mesh::io::save_to_path(mesh, output_path, mesh::io::SaveOptions{.metadata = metadata}).has_value()) { - LOG_ERROR("Failed to save mesh to file {}", output_path); + const auto saved = mesh::io::save_to_path(mesh, output_path, mesh::io::SaveOptions{.metadata = metadata}); + if (!saved) { + LOG_ERROR("Failed to save mesh to file {}: {}", output_path, saved.error().to_string()); std::exit(2); } LOG_DEBUG("Writing mesh took {}s", format_secs_since(start)); @@ -172,7 +173,7 @@ T expect(const std::optional &opt, const std::string &msg) { } } -void build_all_patches( +Expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, @@ -189,14 +190,18 @@ void build_all_patches( LOG_ERROR_AND_EXIT("Output base path {} exists but is not a directory", output_base_path); } - octree::Storage storage = octree::open_folder( + mesh::storage::OpenOptions open_options; + open_options.preferred_extension = output_format; + auto storage_result = mesh::storage::open_folder( output_base_path, - false, - octree::OpenOptions { - .preferred_extension_with_dot = output_format - } - ); - storage.settings().allow_overwrite = overwrite_existing; + std::move(open_options)); + if (!storage_result) { + return Error::propagate( + std::move(storage_result), "open output terrain dataset \"" + output_base_path.string() + "\""); + } + mesh::storage::Storage raw_storage = std::move(storage_result.value()); + raw_storage.settings().allow_overwrite = overwrite_existing; + store::ThreadSafeStorage storage(std::move(raw_storage)); const auto dataset_srs = dataset.srs(); const auto dataset_bounds = dataset.bounds3d(true); @@ -281,7 +286,17 @@ void build_all_patches( tbb::task_group_context context; tbb::parallel_for(size_t(0), target_nodes.size(), [&](size_t i) { const auto &node = target_nodes[i]; - if (!overwrite_existing && storage.has(node)) { + const auto already_exists = storage.has(node); + if (!already_exists) { + LOG_ERROR( + "Failed to inspect node {}: {}", + node, + already_exists.error().to_string()); + progress.task_finished(); + context.cancel_group_execution(); + return; + } + if (!overwrite_existing && already_exists.value()) { progress.task_finished(); // TODO: correctly handle virtual nodes return; } @@ -304,8 +319,11 @@ void build_all_patches( const auto mesh = std::move(mesh_result.value()); mesh::validate(mesh); const auto save_result = storage.save(node, mesh); - if (!save_result.has_value()) { - LOG_ERROR("Failed to save mesh for node {}: {}", node, save_result.error()); + if (!save_result) { + LOG_ERROR( + "Failed to save mesh for node {}: {}", + node, + save_result.error().to_string()); progress.task_finished(); context.cancel_group_execution(); return; @@ -327,9 +345,12 @@ void build_all_patches( LOG_ERROR_AND_EXIT("Failed to build all terrain patches"); } - const auto index_result = storage.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", storage.base_path(), index_result.error()); + auto finalized_storage = std::move(storage).release(); + auto finalized = sf::finalize_storage(finalized_storage); + if (!finalized) { + return Error::propagate( + std::move(finalized), "finalize generated terrain storage \"" + output_base_path.string() + "\""); } + return {}; } } diff --git a/src/sf_builder/terrainbuilder.h b/src/sf_builder/terrainbuilder.h index b4c0f581..64ad66d0 100644 --- a/src/sf_builder/terrainbuilder.h +++ b/src/sf_builder/terrainbuilder.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -8,6 +9,8 @@ #include "octree/Id.h" #include "mesh/SimpleMesh.h" #include "tile_provider.h" +#include "mesh/storage.h" +#include "Error.h" namespace terrainbuilder { @@ -28,7 +31,7 @@ std::optional build_patch( const TileProvider *tile_provider, const OGRSpatialReference &mesh_srs); -void build_all_patches( +Expected build_all_patches( Dataset &dataset, const octree::Id::Level target_level, const OGRSpatialReference &texture_srs, diff --git a/src/sf_index_browser/CMakeLists.txt b/src/sf_index_browser/CMakeLists.txt index 22dbfda5..c580b6a5 100644 --- a/src/sf_index_browser/CMakeLists.txt +++ b/src/sf_index_browser/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable(sf-index-browser - cli.cpp + cli.h cli.cpp main.cpp ) target_link_libraries(sf-index-browser PRIVATE terrainlib CLI11::CLI11 ftxui::dom ftxui::component ftxui::screen) diff --git a/src/sf_index_browser/main.cpp b/src/sf_index_browser/main.cpp index 74c8f304..11f7db1a 100644 --- a/src/sf_index_browser/main.cpp +++ b/src/sf_index_browser/main.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -12,16 +13,15 @@ #include "log.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/Storage.h" -#include "octree/traverse.h" +#include "mesh/storage.h" +#include "mesh/storage.h" #include "cli.h" +#include "store/NodeStatusOrMissing.h" namespace { struct IndexNode { octree::Id id; - octree::NodeStatusOrMissing status; + store::NodeStatusOrMissing status; bool expanded = false; }; struct MeshNode { @@ -41,8 +41,9 @@ class TreeView { using iterator = std::list::iterator; using const_iterator = std::list::const_iterator; - explicit TreeView(const octree::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { - const octree::NodeStatusOrMissing status = this->_storage.index().get(root); + explicit TreeView(const mesh::storage::IndexedStorage &storage, const octree::Id root) : root(root), _storage(storage) { + const store::NodeStatusOrMissing status( + DEBUG_ASSERT_VAL(this->_storage.index().get(root)).value()); this->_view.emplace_back(IndexNode{root, status, false}); } @@ -78,14 +79,14 @@ class TreeView { } parent.expanded = true; - if (parent.status == octree::NodeStatusOrMissing::Leaf) { + if (parent.status == store::NodeStatusOrMissing::Leaf) { auto result = this->_storage.load(parent.id); DisplayEntry entry; - if (result.has_value()) { + if (result) { const auto &mesh = result.value(); entry = MeshNode{parent.id, mesh.vertex_count(), mesh.face_count()}; } else { - entry = ErrorNode{parent.id, result.error().description()}; + entry = ErrorNode{parent.id, result.error().to_string()}; } it++; it = this->_view.emplace(it, entry); @@ -97,11 +98,12 @@ class TreeView { } const auto children = parent.id.children().value(); for (const auto &child_id : children) { - auto status_opt = this->_storage.index().get(child_id); - if (!status_opt.has_value()) { + auto status_result = this->_storage.index().get(child_id); + DEBUG_ASSERT(status_result.has_value()); + if (!status_result->has_value()) { continue; } - const octree::NodeStatus status = status_opt.value(); + const store::NodeStatus status = status_result->value(); IndexNode child_node{child_id, status, false}; it++; it = this->_view.emplace(it, child_node); @@ -123,7 +125,7 @@ class TreeView { } parent.expanded = false; - if (parent.status == octree::NodeStatusOrMissing::Leaf) { + if (parent.status == store::NodeStatusOrMissing::Leaf) { it++; it = this->_view.erase(it); return; @@ -145,14 +147,16 @@ class TreeView { private: std::list _view; - const octree::IndexedStorage& _storage; + const mesh::storage::IndexedStorage& _storage; }; -const octree::Id find_deepest_root(const octree::IndexMap &index, const octree::Id &root = octree::Id::root()) { - switch (octree::NodeStatusOrMissing(index.get(root))) { - case octree::NodeStatusOrMissing::Missing: +const octree::Id find_deepest_root( + const store::Index &index, + const octree::Id &root = octree::Id::root()) { + switch (store::NodeStatusOrMissing(DEBUG_ASSERT_VAL(index.get(root)).value())) { + case store::NodeStatusOrMissing::Missing: return octree::Id::root(); - case octree::NodeStatusOrMissing::Leaf: + case store::NodeStatusOrMissing::Leaf: return root; default: break; @@ -163,7 +167,8 @@ const octree::Id find_deepest_root(const octree::IndexMap &index, const octree:: const auto children = current.children().value(); std::optional next; for (const octree::Id &child_id : children) { - if (!index.get(child_id).has_value()) { + const auto status = DEBUG_ASSERT_VAL(index.get(child_id)).value(); + if (!status.has_value()) { continue; } if (next.has_value()) { @@ -183,12 +188,17 @@ const octree::Id find_deepest_root(const octree::IndexMap &index, const octree:: } // namespace -octree::IndexedStorage open_path_indexed(const std::filesystem::path& path) { - if (std::filesystem::is_directory(path)) { - return octree::open_folder_indexed(path); - } else { - return octree::open_index(path).value(); +mesh::storage::IndexedStorage open_path_indexed(const std::filesystem::path& path) { + auto result = std::filesystem::is_directory(path) + ? mesh::storage::open_folder_indexed(path) + : mesh::storage::open_index(path); + if (!result) { + LOG_ERROR_AND_EXIT( + "Failed to open dataset {}: {}", + path, + result.error().to_string()); } + return std::move(result.value()); } std::string render_line_content(const DisplayEntry &entry) { @@ -219,8 +229,8 @@ ftxui::Element render_line(const DisplayEntry &entry, size_t base_level, bool is } int run(const cli::Args &args) { - const octree::IndexedStorage storage = open_path_indexed(args.dataset_path); - const octree::IndexMap &index = storage.index(); + const mesh::storage::IndexedStorage storage = open_path_indexed(args.dataset_path); + const store::Index &index = storage.index(); const octree::Id root_id = args.full_view ? octree::Id::root() : find_deepest_root(index); TreeView tree_view(storage, root_id); diff --git a/src/sf_merger/CMakeLists.txt b/src/sf_merger/CMakeLists.txt index c17cfc9a..2f647ab4 100644 --- a/src/sf_merger/CMakeLists.txt +++ b/src/sf_merger/CMakeLists.txt @@ -1,9 +1,30 @@ -add_library(sfmergerlib INTERFACE) +add_library(sfmergerlib INTERFACE + NodeLoader.h + NodeWriter.h + cut.h + earth.h + mask.h + merge.h + utils.h + + atlas/HorizontalStripLayoutPlanner.h + atlas/Layout.h + atlas/LayoutPlanner.h + atlas/Sheet.h + atlas/build.h + + merge/NodeData.h + merge/Result.h + merge/visitor/Masked.h + merge/visitor/Simple.h + merge/visitor/Visitor.h +) + target_include_directories(sfmergerlib INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(sfmergerlib INTERFACE terrainlib) add_executable(sf-merger + cli.h cli.cpp main.cpp - cli.cpp ) target_link_libraries(sf-merger PRIVATE sfmergerlib CLI11::CLI11) diff --git a/src/sf_merger/NodeLoader.h b/src/sf_merger/NodeLoader.h index c6ac4027..55e5a76a 100644 --- a/src/sf_merger/NodeLoader.h +++ b/src/sf_merger/NodeLoader.h @@ -9,28 +9,27 @@ #include "mesh/clip.h" #include "mesh/texture_trim.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" #include "octree/Space.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/cache/Dummy.h" -#include "octree/storage/cache/ICache.h" +#include "mesh/storage.h" +#include "store/NodeStatusOrMissing.h" +#include "store/cache/Interface.h" class NodeLoader { public: - NodeLoader(const octree::IndexedStorage &storage, octree::cache::ICache &cache, octree::Space space) - : _storage(storage), _cache(cache), _space(space) { - if (storage.cache().has_value() && dynamic_cast *>(&cache) != nullptr) { - LOG_WARN("Backing storage for NodeLoader instance is cached, but another cache was provided leading to double caching."); - } - } + NodeLoader( + const mesh::storage::IndexedStorage &storage, + store::cache::Interface &cache, + octree::Space space) + : _storage(storage), _cache(cache), _space(space) {} - octree::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { - return this->_storage.index().get(id); + store::NodeStatusOrMissing get_status(const octree::Id &id) const noexcept { + return store::NodeStatusOrMissing( + DEBUG_ASSERT_VAL(this->_storage.index().get(id)).value()); } // Try to retrieve or reconstruct node mesh by ID // TODO: return const ref instead? - std::optional load_node(const octree::Id &id) const noexcept { + std::optional load_node(const octree::Id &id) const { // Try cache if (auto cached = this->_cache.get(id); cached.has_value()) { return cached.value(); @@ -38,7 +37,7 @@ class NodeLoader { // Try storage auto mesh_opt = this->_storage.load(id); - if (mesh_opt.has_value()) { + if (mesh_opt) { auto mesh = mesh_opt.value(); this->_cache.put(id, mesh); return mesh; @@ -55,7 +54,7 @@ class NodeLoader { // Try storage auto parent_mesh_opt = this->_storage.load(parent_id); - if (parent_mesh_opt.has_value()) { + if (parent_mesh_opt) { auto parent_mesh = parent_mesh_opt.value(); const auto bounds = this->_space.get_node_bounds(id); SimpleMesh clipped = mesh::clip_on_bounds(parent_mesh, bounds); @@ -69,12 +68,12 @@ class NodeLoader { return std::nullopt; } - const octree::IndexedStorage &storage() const { + const mesh::storage::IndexedStorage &storage() const { return this->_storage; } private: - const octree::IndexedStorage &_storage; - octree::cache::ICache &_cache; + const mesh::storage::IndexedStorage &_storage; + store::cache::Interface &_cache; const octree::Space _space; }; diff --git a/src/sf_merger/NodeWriter.h b/src/sf_merger/NodeWriter.h index f160a8ad..3bca914f 100644 --- a/src/sf_merger/NodeWriter.h +++ b/src/sf_merger/NodeWriter.h @@ -1,49 +1,90 @@ #pragma once +#include +#include + #include +#include +#include "Error.h" #include "NodeLoader.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/storage/Storage.h" -#include "octree/traverse.h" +#include "mesh/storage.h" +#include "store/traverse.h" // TODO: make thread safe class NodeWriter { public: - NodeWriter(octree::Storage &storage) : _storage(storage) {} + NodeWriter(mesh::storage::Storage &storage) : _storage(storage) {} - bool has_node(const octree::Id &id) { + Expected has_node( + const octree::Id &id) { return this->_storage.has(id); } - void write_node(const octree::Id &id, const SimpleMesh &mesh) { + Expected write_node( + const octree::Id &id, + const SimpleMesh &mesh) { mesh::validate(mesh); - DEBUG_ASSERT_VAL(this->_storage.save(id, mesh)); - auto p = this->_storage.path_for(id); + auto save_result = this->_storage.save(id, mesh); + if (!save_result) { + return save_result; + } + auto path_result = this->_storage.path_for(id); + if (!path_result) { + return Error::propagate( + std::move(path_result), "resolve texture output path for node " + id.to_string()); + } + auto p = path_result.value(); // change extension to .png p.replace_extension(".png"); - cv::imwrite(p, mesh.texture.value_or(cv::Mat())); + if (mesh.texture.has_value()) { + try { + if (!cv::imwrite(p, mesh.texture.value())) { + return Error::fail(Error::Code::Io, "write node texture to", p); + } + } catch (const cv::Exception& error) { + return Error::fail( + Error::Code::Io, "write node texture to \"" + p.string() + "\": " + error.what()); + } + } + return {}; } - void copy_subtree_to_output( + Expected copy_subtree_to_output( const octree::Id &id, const NodeLoader &loader) { - octree::traverse( + std::optional error = std::nullopt; + auto traversal = store::traverse( loader.storage().index(), - [&](const octree::Id &child_id, const octree::NodeStatus &status) { - if (status == octree::NodeStatus::Virtual) { + [&](const octree::Id &child_id, const store::NodeStatus &status) { + if (error.has_value()) { return; } - DEBUG_ASSERT(status == octree::NodeStatus::Leaf); + if (status == store::NodeStatus::Virtual) { + return; + } + DEBUG_ASSERT(status == store::NodeStatus::Leaf); - DEBUG_ASSERT_VAL(this->_storage.copy_from(child_id, loader.storage())); + auto result = this->_storage.copy_from(child_id, loader.storage()); + if (!result) { + error = std::move(result).error(); + } }, - octree::always_refine, + [&](const octree::Id &) { return !error.has_value(); }, id); + if (!traversal) { + return Error::propagate( + std::move(traversal), "traverse source subtree rooted at " + id.to_string() + " while copying"); + } + if (error.has_value()) { + return Error::propagate( + std::move(error.value()), "copy subtree rooted at " + id.to_string() + " to output"); + } + return {}; } private: - octree::Storage &_storage; + mesh::storage::Storage &_storage; }; diff --git a/src/sf_merger/cut.h b/src/sf_merger/cut.h index 286f473b..b06437f0 100644 --- a/src/sf_merger/cut.h +++ b/src/sf_merger/cut.h @@ -1,63 +1,78 @@ #pragma once +#include + #include #include "mesh/SimpleMesh.h" #include "log.h" #include "mask.h" #include "octree/Id.h" -#include "octree/NodeStatus.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/open.h" +#include "mesh/storage.h" #include "octree/Space.h" -#include "octree/traverse.h" #include "utils.h" #include "containers/Cow.h" +#include "Error.h" +#include "sf/finalize_storage.h" +#include "sf/validate_index.h" struct Context { - const octree::IndexedStorage& input; - octree::Storage& output; + const mesh::storage::IndexedStorage& input; + mesh::storage::Storage& output; const octree::Space space; const bool keep_inside; }; -void cut_node( +Expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask); -inline void cut_leaf_node( +inline Expected cut_leaf_node( Context& ctx, const octree::Id& id, const MeshMask& mask ) { - DEBUG_ASSERT(ctx.input.index().is(octree::NodeStatus::Leaf, id)); + DEBUG_ASSERT(DEBUG_ASSERT_VAL( + ctx.input.index().is(store::NodeStatus::Leaf, id)).value()); - const SimpleMesh mesh = DEBUG_ASSERT_VAL(ctx.input.load(id)).value(); + auto mesh_result = ctx.input.load(id); + if (!mesh_result) { + return Error::propagate(std::move(mesh_result), "read cut input node " + id.to_string()); + } + const SimpleMesh mesh = std::move(*mesh_result); LOG_TRACE("Cutting mesh at {} using mask with {} vertices and {} triangles", id, mask.vertex_count(), mask.face_count()); const Cow clipped = clip_on_mask(mesh, mask, ctx.keep_inside); if (clipped.is_ref()) { LOG_TRACE("Mesh was fully inside the mask"); - DEBUG_ASSERT_VAL(ctx.output.copy_from(id, ctx.input)); + auto copy_result = ctx.output.copy_from(id, ctx.input); + if (!copy_result) { + return copy_result; + } } else { const SimpleMesh &clipped_mesh = clipped; if (!clipped_mesh.is_empty()) { LOG_TRACE("Mesh was clipped from {} vertices and {} triangles to {} vertices and {} triangles", mesh.vertex_count(), mesh.face_count(), clipped_mesh.vertex_count(), clipped_mesh.face_count()); - DEBUG_ASSERT_VAL(ctx.output.save(id, clipped_mesh)); + auto save_result = ctx.output.save(id, clipped_mesh); + if (!save_result) { + return save_result; + } } else { LOG_TRACE("Mesh was fully outside the mask"); } } + return {}; } -inline void cut_virtual_node( +inline Expected cut_virtual_node( Context& ctx, const octree::Id& id, const MeshMask& mask ) { - DEBUG_ASSERT(ctx.input.index().is(octree::NodeStatus::Virtual, id)); + DEBUG_ASSERT(DEBUG_ASSERT_VAL( + ctx.input.index().is(store::NodeStatus::Virtual, id)).value()); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); @@ -66,18 +81,22 @@ inline void cut_virtual_node( const auto child_bounds = geometry::pad_bounds_relative(ctx.space.get_node_bounds(child_id), 0.125); LOG_TRACE("Clipping mask for {}", child_id); const MeshMask child_mask = clip_mask_on_bounds(mask, child_bounds); - cut_node(ctx, child_id, child_mask); + auto child_result = cut_node(ctx, child_id, child_mask); + if (!child_result) { + return child_result; + } } + return {}; } -inline void cut_node( +inline Expected cut_node( Context& ctx, const octree::Id& id, const MeshMask& mask ) { if (ctx.keep_inside && mask.is_empty()) { LOG_TRACE("Mesh was fully outside the mask"); - return; + return {}; } /* @@ -88,50 +107,68 @@ inline void cut_node( mesh::io::save_to_path(mask.mesh, new_path); */ - const auto status_opt = ctx.input.index().get(id); - if (!status_opt.has_value()) { - return; + const auto status_result = ctx.input.index().get(id); + DEBUG_ASSERT(status_result.has_value()); + if (!status_result->has_value()) { + return {}; } - const octree::NodeStatus status = status_opt.value(); + const store::NodeStatus status = status_result->value(); switch (status) { - case octree::NodeStatus::Virtual: - cut_virtual_node(ctx, id, mask); - break; - case octree::NodeStatus::Leaf: - cut_leaf_node(ctx, id, mask); - break; + case store::NodeStatus::Virtual: + return cut_virtual_node(ctx, id, mask); + case store::NodeStatus::Leaf: + return cut_leaf_node(ctx, id, mask); default: UNREACHABLE(); - break; + return {}; } + return {}; } -inline void cut_dataset( - const octree::IndexedStorage &input, +inline Expected cut_dataset( + const mesh::storage::IndexedStorage &input, const MeshMask& mask, - octree::Storage &output, + mesh::storage::Storage &output, const bool keep_inside) { + auto validation = sf::validate_index(input.index()); + if (!validation) { + return Error::propagate( + std::move(validation), "validate cut input dataset \"" + input.base_path().string() + "\""); + } Context ctx(input, output, octree::Space::earth(), keep_inside); - cut_node(ctx, octree::Id::root(), mask); - const auto index_result = output.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output.base_path(), index_result.error()); + auto cut_result = cut_node(ctx, octree::Id::root(), mask); + if (!cut_result) { + return Error::propagate(std::move(cut_result), "cut dataset \"" + input.base_path().string() + "\""); + } + auto finalized = sf::finalize_storage(output); + if (!finalized) { + return Error::propagate( + std::move(finalized), "finalize cut output dataset \"" + output.base_path().string() + "\""); } + return {}; } -inline void cut_dataset( - const octree::IndexedStorage &input_dataset, +inline Expected cut_dataset( + const mesh::storage::IndexedStorage &input_dataset, const MeshMask& mask, const std::filesystem::path &output_path, const bool keep_inside) { LOG_TRACE("Creating output dataset at {}", output_path); std::filesystem::create_directories(output_path); - octree::IndexedStorage output_dataset = octree::open_folder_indexed(output_path, octree::OpenOptions(octree::disk::layout::strategy::make_default(), ".glb")); + mesh::storage::OpenOptions options; + options.default_mapping = octree::store_layout::level_and_coordinate_directories(); + options.preferred_extension = ".glb"; + auto output_result = mesh::storage::open_folder_indexed(output_path, std::move(options)); + if (!output_result) { + return Error::propagate( + std::move(output_result), "open cut output dataset \"" + output_path.string() + "\""); + } + mesh::storage::IndexedStorage output_dataset = std::move(output_result.value()); if (!output_dataset.index().empty()) { - LOG_ERROR_AND_EXIT("Output dataset has to be empty."); + return Error::fail(Error::Code::AlreadyExists, "output dataset already contains nodes: " + output_path.string()); } - cut_dataset(input_dataset, mask, output_dataset, keep_inside); + return cut_dataset(input_dataset, mask, output_dataset, keep_inside); } diff --git a/src/sf_merger/main.cpp b/src/sf_merger/main.cpp index 37e3f3dc..3cc7b7bc 100644 --- a/src/sf_merger/main.cpp +++ b/src/sf_merger/main.cpp @@ -7,8 +7,8 @@ #include "cut.h" #include "log.h" #include "mask.h" +#include "mesh/storage.h" #include "merge.h" -#include "octree/Storage.h" #include "optional_utils.h" #include "earth.h" @@ -17,7 +17,7 @@ std::optional load_mask_from_path(const std::filesystem::path& path) { LOG_INFO("Loading mask file from {}", path); const glm::dvec2 radius_range = mask::pad_radius_range(earth::radius_range(), 2); auto result = mask::load_from_path(path, radius_range); - if (result.has_value()) { + if (result) { const auto mask = result.value(); LOG_DEBUG("Loaded mask successfully ({} vertices, {} triangles)", mask.vertex_count(), mask.face_count()); @@ -34,25 +34,70 @@ std::optional load_mask_from_path(const std::filesystem::path& path) { void run(const cli::MergeArgs& args) { LOG_TRACE("Loading base dataset from {}", args.base_path); - octree::IndexedStorage base_dataset = octree::open_folder_indexed(args.base_path); + auto base_result = mesh::storage::open_folder_indexed(args.base_path); + if (!base_result) { + LOG_ERROR_AND_EXIT( + "Failed to open base dataset {}: {}", + args.base_path, + base_result.error().to_string()); + } + mesh::storage::IndexedStorage base_dataset = std::move(base_result.value()); LOG_TRACE("Loading new dataset from {}", args.new_path); - octree::IndexedStorage new_dataset = octree::open_folder_indexed(args.new_path); + auto new_result = mesh::storage::open_folder_indexed(args.new_path); + if (!new_result) { + LOG_ERROR_AND_EXIT( + "Failed to open new dataset {}: {}", + args.new_path, + new_result.error().to_string()); + } + mesh::storage::IndexedStorage new_dataset = std::move(new_result.value()); LOG_TRACE("Creating output dataset at {}", args.output_path); std::filesystem::create_directories(args.output_path); - octree::Storage output_dataset = octree::open_folder(args.output_path, false, octree::OpenOptions{.preferred_extension_with_dot = std::string(base_dataset.layout().extension_with_dot())}); + mesh::storage::OpenOptions options; + options.default_mapping = base_dataset.layout().mapping(); + options.preferred_extension = std::string(base_dataset.codec_selector().value_or(".sfmesh")); + auto output_result = mesh::storage::open_folder(args.output_path, std::move(options)); + if (!output_result) { + LOG_ERROR_AND_EXIT( + "Failed to open output dataset {}: {}", + args.output_path, + output_result.error().to_string()); + } + mesh::storage::Storage output_dataset = std::move(output_result.value()); std::optional mask = flatten(map(args.mask_path, load_mask_from_path)); - return merge_datasets(base_dataset, new_dataset, output_dataset, mask); + const auto merge_result = merge_datasets( + base_dataset, + new_dataset, + output_dataset, + mask); + if (!merge_result) { + LOG_ERROR_AND_EXIT("Failed to merge datasets: {}", merge_result.error().to_string()); + } } void run(const cli::CutArgs& args) { LOG_TRACE("Loading input dataset from {}", args.input_path); - const octree::IndexedStorage input_dataset = octree::open_folder_indexed(args.input_path); + auto input_result = mesh::storage::open_folder_indexed(args.input_path); + if (!input_result) { + LOG_ERROR_AND_EXIT( + "Failed to open input dataset {}: {}", + args.input_path, + input_result.error().to_string()); + } + const mesh::storage::IndexedStorage input_dataset = std::move(input_result.value()); const MeshMask mask = DEBUG_ASSERT_VAL(load_mask_from_path(args.mask_path)).value(); - cut_dataset(input_dataset, mask, args.output_path, args.keep_inside); + const auto cut_result = cut_dataset( + input_dataset, + mask, + args.output_path, + args.keep_inside); + if (!cut_result) { + LOG_ERROR_AND_EXIT("Failed to cut dataset: {}", cut_result.error().to_string()); + } } void run(const cli::Args &args) { diff --git a/src/sf_merger/mask.h b/src/sf_merger/mask.h index 8ceb8900..304cbce3 100644 --- a/src/sf_merger/mask.h +++ b/src/sf_merger/mask.h @@ -165,7 +165,8 @@ uint64_t point_count(const OGRGeometry &geometry) { const OGRwkbGeometryType geometry_type = wkbFlatten(geometry.getGeometryType()); if (geometry_type == wkbPolygon) { const OGRPolygon *polygon = geometry.toPolygon(); - uint64_t count = polygon->getExteriorRing()->getNumPoints(); + const OGRLinearRing *exterior_ring = polygon->getExteriorRing(); + uint64_t count = exterior_ring ? exterior_ring->getNumPoints() : 0; for (int i = 0; i < polygon->getNumInteriorRings(); ++i) { count += polygon->getInteriorRing(i)->getNumPoints(); } @@ -354,7 +355,10 @@ inline std::expected load_referenced_from_data MultipolygonWithHoles2 polygons; for (auto &&feature_layer_pair : dataset->GetFeatures()) { OGRGeometry *geometry = feature_layer_pair.feature->GetGeometryRef(); - DEBUG_ASSERT(geometry); + if (!geometry) { + LOG_ERROR("Mask feature has no geometry"); + return std::unexpected(LoadErrorKind::InvalidGeometry); + } const uint64_t original_point_count = point_count(*geometry); std::unique_ptr simplified = simplify_geometry(*geometry, *tolerance); @@ -565,7 +569,7 @@ inline std::expected load_referenced_from_path inline std::expected load_from_path(const std::filesystem::path &path, const glm::dvec2& radius_range) { auto ref_mask_res = load_referenced_from_path(path); - if (!ref_mask_res.has_value()) { + if (!ref_mask_res) { return std::unexpected(ref_mask_res.error()); } ReferencedPolygonMask ref_polygon_mask = std::move(ref_mask_res.value()); diff --git a/src/sf_merger/merge.h b/src/sf_merger/merge.h index 180dbb59..8f902a02 100644 --- a/src/sf_merger/merge.h +++ b/src/sf_merger/merge.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -12,10 +13,14 @@ #include "merge/visitor/Simple.h" #include "merge/visitor/Visitor.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/storage/cache/Dummy.h" - -inline std::string get_dataset_name(const octree::Storage &storage) { +#include "mesh/storage.h" +#include "store/NodeStatusOrMissing.h" +#include "store/cache/Dummy.h" +#include "Error.h" +#include "sf/finalize_storage.h" +#include "sf/validate_index.h" + +inline std::string get_dataset_name(const mesh::storage::Storage &storage) { return storage.layout().base_path().filename().string(); } @@ -29,7 +34,7 @@ using smallest_uint_t = template class Merger { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; using Context = Visitor::Context; using Result = merge::Result; @@ -40,62 +45,87 @@ class Merger { NodeWriter output) : _visitor(visitor), _left(left), _right(right), _output(output) { } - void merge_root() { + Expected merge_root() { const octree::Id id = octree::Id::root(); const Context ctx = this->_visitor.make_root_context(); - merge_node(id, ctx); + return merge_node(id, ctx); } - void merge_node(const octree::Id &id, const Context& ctx) { + Expected merge_node(const octree::Id &id, const Context& ctx) { const Status left_status = this->_left.get_status(id); const Status right_status = this->_right.get_status(id); return this->merge_node(id, left_status, right_status, ctx); } - void merge_node( + Expected merge_node( const octree::Id &id, const Status left_status, const Status right_status, const Context& ctx ) { LOG_DEBUG("[{}] Start merging (left = {}, right = {})", id, left_status, right_status); - if (this->_output.has_node(id)) { + auto has_result = this->_output.has_node(id); + if (!has_result) { + return Error::propagate( + std::move(has_result), "check merged output for node " + id.to_string()); + } + if (has_result.value()) { LOG_DEBUG("[{}] Already merged, skipping...", id); - return; // Already merged previously + return {}; // Already merged previously } const auto merge_result = this->call_merge(id, left_status, right_status, ctx); - std::visit([&](const auto &result) { + return std::visit([&](const auto &result) -> Expected { using Result = std::decay_t; if constexpr (std::is_same_v>) { LOG_DEBUG("[{}] needs recursion", id); DEBUG_ASSERT(id.has_children()); const auto children = id.children().value(); for (const auto &child_id : children) { - this->merge_node(child_id, result.context); + auto child_result = this->merge_node(child_id, result.context); + if (!child_result) { + return child_result; + } } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] remains unchanged (same as {})", id, (result.source == merge::Source::Left ? "left" : "right")); // If the node should remain unchanged, but its not present on disk we cannot copy it. if (result.source == merge::Source::Left && left_status == Status::Missing) { auto mesh_opt = this->_left.load_node(id); if (mesh_opt.has_value()) { - this->_output.write_node(id, mesh_opt.value()); + auto write_result = this->_output.write_node(id, *mesh_opt); + if (!write_result) { + return write_result; + } } } else if (result.source == merge::Source::Right && right_status == Status::Missing) { auto mesh_opt = this->_right.load_node(id); if (mesh_opt.has_value()) { - this->_output.write_node(id, mesh_opt.value()); + auto write_result = this->_output.write_node(id, *mesh_opt); + if (!write_result) { + return write_result; + } } } else { - this->_output.copy_subtree_to_output(id, result.source == merge::Source::Left ? this->_left : this->_right); + auto copy_result = this->_output.copy_subtree_to_output( + id, + result.source == merge::Source::Left ? this->_left : this->_right); + if (!copy_result) { + return copy_result; + } } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was merged", id); - this->_output.write_node(id, result.mesh); + auto write_result = this->_output.write_node(id, result.mesh); + if (!write_result) { + return write_result; + } + return {}; } else if constexpr (std::is_same_v) { LOG_DEBUG("[{}] was ignored", id); - // do nothing + return {}; } }, merge_result); } @@ -151,34 +181,50 @@ class Merger { NodeWriter _output; }; -inline void merge_datasets( - const octree::IndexedStorage &left_dataset, - const octree::IndexedStorage &right_dataset, - octree::Storage &output_dataset, +inline Expected merge_datasets( + const mesh::storage::IndexedStorage &left_dataset, + const mesh::storage::IndexedStorage &right_dataset, + mesh::storage::Storage &output_dataset, const std::optional mask = std::nullopt) { + for (const mesh::storage::IndexedStorage *input : {&left_dataset, &right_dataset}) { + auto validation = sf::validate_index(input->index()); + if (!validation) { + return Error::propagate( + std::move(validation), "validate merge input dataset \"" + get_dataset_name(*input) + "\""); + } + } + LOG_TRACE("Merging {} and {} into {}", get_dataset_name(left_dataset), get_dataset_name(right_dataset), get_dataset_name(output_dataset)); octree::Space space = octree::Space::earth(); - octree::cache::Dummy left_cache; + store::cache::Dummy left_cache; NodeLoader left(left_dataset, left_cache, space); - octree::cache::Dummy right_cache; + store::cache::Dummy right_cache; NodeLoader right(right_dataset, right_cache, space); NodeWriter output(output_dataset); if (mask.has_value()) { merge::visitor::Masked visitor {mask.value(), space}; Merger merger(visitor, left, right, output); - merger.merge_root(); + auto result = merger.merge_root(); + if (!result) { + return Error::propagate(std::move(result), "merge masked datasets"); + } } else { merge::visitor::Simple visitor; Merger merger(visitor, left, right, output); - merger.merge_root(); + auto result = merger.merge_root(); + if (!result) { + return Error::propagate(std::move(result), "merge datasets"); + } } - const auto index_result = output_dataset.save_or_create_index(); - if (!index_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to save output index in {}: {}", output_dataset.base_path(), index_result.error()); + auto finalized = sf::finalize_storage(output_dataset); + if (!finalized) { + return Error::propagate( + std::move(finalized), "finalize merged output dataset \"" + get_dataset_name(output_dataset) + "\""); } + return {}; } diff --git a/src/sf_merger/merge/NodeData.h b/src/sf_merger/merge/NodeData.h index c3c9de13..2bdd72f1 100644 --- a/src/sf_merger/merge/NodeData.h +++ b/src/sf_merger/merge/NodeData.h @@ -6,18 +6,17 @@ #include "log.h" #include "mesh/SimpleMesh.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/Storage.h" #include "optional_utils.h" +#include "store/NodeStatusOrMissing.h" namespace merge { -template +template class NodeData { public: constexpr explicit NodeData(octree::Id id, const NodeLoader &loader) : _id(id), _loader(loader) {} - static constexpr octree::NodeStatusOrMissing status() { + static constexpr store::NodeStatusOrMissing status() { return Status; } @@ -26,8 +25,8 @@ class NodeData { } // Status::Leaf or Status::Inner guarantuess a node is present - template - std::enable_if_t + template + std::enable_if_t mesh() const { auto mesh = this->load_mesh(); if (mesh.has_value()) { @@ -38,8 +37,8 @@ class NodeData { } // Status::Missing and Status::Virtual do not exist on disk, but may be the child of a leaf or inner node - template - std::enable_if_t>> + template + std::enable_if_t>> mesh() const { return this->load_mesh(); } diff --git a/src/sf_merger/merge/visitor/Masked.h b/src/sf_merger/merge/visitor/Masked.h index 28fc8508..0a78b37b 100644 --- a/src/sf_merger/merge/visitor/Masked.h +++ b/src/sf_merger/merge/visitor/Masked.h @@ -11,7 +11,7 @@ #include "mesh/merging/helpers.h" #include "mesh/merging/mapping.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" #include "polygon/Polygon.h" #include "polygon/triangulate.h" #include "spatial_lookup/Hashmap.h" @@ -26,7 +26,7 @@ namespace merge::visitor { class Masked { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; struct Context { MeshMask mask; bool has_left_parent; diff --git a/src/sf_merger/merge/visitor/Simple.h b/src/sf_merger/merge/visitor/Simple.h index 531cd686..9305ce7b 100644 --- a/src/sf_merger/merge/visitor/Simple.h +++ b/src/sf_merger/merge/visitor/Simple.h @@ -5,13 +5,13 @@ #include "merge/visitor/Visitor.h" #include "mesh/merge.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" namespace merge::visitor { class Simple { public: - using Status = octree::NodeStatusOrMissing; + using Status = store::NodeStatusOrMissing; struct Context{}; using Result = merge::Result; diff --git a/src/sf_merger/merge/visitor/Visitor.h b/src/sf_merger/merge/visitor/Visitor.h index 35206fe4..6607e222 100644 --- a/src/sf_merger/merge/visitor/Visitor.h +++ b/src/sf_merger/merge/visitor/Visitor.h @@ -3,7 +3,7 @@ #include "merge/NodeData.h" #include "merge/Result.h" #include "octree/Id.h" -#include "octree/NodeStatusOrMissing.h" +#include "store/NodeStatusOrMissing.h" namespace merge { template @@ -15,10 +15,10 @@ concept Visitor = requires(T t, const octree::Id &id) { { t.make_root_context() } -> std::same_as; { - t.template visit( + t.template visit( id, - std::declval &>(), - std::declval &>(), + std::declval &>(), + std::declval &>(), std::declval()) } -> std::same_as>; }; diff --git a/src/terrainlib/CMakeLists.txt b/src/terrainlib/CMakeLists.txt index a51a7821..11c40f01 100644 --- a/src/terrainlib/CMakeLists.txt +++ b/src/terrainlib/CMakeLists.txt @@ -1,4 +1,47 @@ add_library(terrainlib + Error.h Error.cpp + Dataset.h Dataset.cpp + ImageKey.h + ProgressIndicator.h ProgressIndicator.cpp + Range.h + SphereProjector.h + VecHash.h + VecRange.h + array_utils.h + build_config.h + enumerate.h + fmt_impls.h + glm_utils.h + hash_utils.h + init.h init.cpp + log.h log.cpp + macros.h + opencv_utils.h + optional_utils.h + pch.h + range_utils.h + srs.h + string_utils.h + tile_path.h + type_utils.h + variant_utils.h + vector_utils.h + + containers/Buffer.h + containers/Cow.h + containers/FixedVector.h + containers/HybridIndexPairMap.h + containers/HybridVector.h + containers/OffsetTable.h + containers/OffsetVector.h + containers/SegmentedBuffer.h + containers/Size.h + containers/StampSet.h + containers/ThreadLocalCache.h + containers/TinyVector.h + containers/UnionFind.h + containers/Vector2D.h + ctb/CTBException.hpp ctb/GlobalGeodetic.hpp ctb/GlobalGeodetic.cpp ctb/GlobalMercator.hpp ctb/GlobalMercator.cpp @@ -6,55 +49,156 @@ add_library(terrainlib ctb/TileCoordinate.hpp ctb/types.hpp - io/bytes.cpp - io/compression.cpp - io/utils.cpp + geometry/CoordFrame.h + geometry/PlaneFrame.h + geometry/Rect.h + geometry/geometry.h - mesh/connectivity/adjacency.cpp - mesh/connectivity/boundary.cpp - mesh/connectivity/manifold.cpp - mesh/connectivity/topology.cpp - mesh/connectivity/vertex_index_range.cpp + io/bytes.h io/bytes.cpp + io/compression.h io/compression.cpp + io/conversion.h + io/envelope.h + io/envelope_file.h + io/utils.h io/utils.cpp + mesh/EncodedMesh.h + mesh/PointWithMeta.h + mesh/SimpleMesh.h + mesh/TriangleContainer.h + mesh/TriangleSoup.h + mesh/VertexMap.h + mesh/View.h + mesh/WindingOrder.h + mesh/bfs.h mesh/bfs.cpp + mesh/boolean.h mesh/boolean.cpp + mesh/bounds.h + mesh/cgal.h + mesh/cleanup.h mesh/cleanup.cpp + mesh/clip.h mesh/clip.cpp + mesh/codec/Gltf.h + mesh/codec/SfMesh.h + mesh/codec/SfMeshFormat.h + mesh/codec/from_extension.h + mesh/combine.h + mesh/connectivity/adjacency.h mesh/connectivity/adjacency.cpp + mesh/connectivity/boundary.h mesh/connectivity/boundary.cpp + mesh/connectivity/connected_components.h + mesh/connectivity/edges.h + mesh/connectivity/manifold.h mesh/connectivity/manifold.cpp + mesh/connectivity/topology.h mesh/connectivity/topology.cpp + mesh/connectivity/triangle_compare.h + mesh/connectivity/vertex_index_range.h mesh/connectivity/vertex_index_range.cpp + mesh/convert.h mesh/convert.cpp + mesh/encode.h + mesh/geometry.h mesh/geometry.cpp + mesh/holes.h mesh/holes.cpp + mesh/igl/convert.h + mesh/igl/cut.h + mesh/igl/cut_to_disk.h mesh/igl/igl.cpp + mesh/igl/manifold.h + mesh/igl/orient.h + mesh/io.h mesh/io.cpp + mesh/io/gltf.h mesh/io/gltf.cpp + mesh/io/options.h + mesh/io/texture.h mesh/io/texture.cpp + mesh/io/utils.h mesh/io/utils.cpp + mesh/merge.h + mesh/merging/EpsilonVertexDeduplicate.h + mesh/merging/ExactVertexDeduplicate.h + mesh/merging/SphereProjectionVertexDeduplicate.h + mesh/merging/VertexDeduplicate.h + mesh/merging/VertexId.h + mesh/merging/VertexMapping.h + mesh/merging/helpers.h + mesh/merging/mapping.h mesh/merging/mapping.cpp + mesh/normalize.h + mesh/reindex.h + mesh/split.h + mesh/storage.h + mesh/texture_trim.h mesh/texture_trim.cpp + mesh/utils.h + mesh/validate.h mesh/validate.cpp + mesh/validate_cgal.h + + numeric/StrongDouble.h + numeric/glm_math.h + numeric/int_math.h + numeric/number_types.h + numeric/number_utils.h + numeric/quantize.h + numeric/wide_integer.h + + octree/Id.h + octree/IdRect.h + octree/OddLevelShifted.h octree/OddLevelShifted.cpp + octree/Space.h octree/Space.cpp + octree/StoreTraits.h + octree/storage/IndexFile.h + octree/storage/open_runtime.h + octree/store_layout/Flat.h + octree/store_layout/LevelAndCoordinateDirectories.h + octree/store_layout/Mappings.h + octree/utils.h + + polygon/Polygon.h + polygon/clip.h polygon/clip.cpp + polygon/triangulate.h polygon/triangulate.cpp + polygon/utils.h polygon/utils.cpp + + range/AnyRange.h + range/Bound.h + range/Range.h + range/RangeBounds.h + range/RangeFrom.h + range/RangeFull.h + range/RangeInclusive.h + range/RangeTo.h + range/RangeToInclusive.h + + raster_store/StoreTraits.h + sf/finalize_storage.h + sf/validate_index.h + + spatial_lookup/CellBased.h + spatial_lookup/CellBasedStorage.h + spatial_lookup/Grid.h + spatial_lookup/GridStorage.h + spatial_lookup/Hashmap.h + spatial_lookup/HashmapStorage.h + spatial_lookup/NDLoopHelper.h + spatial_lookup/QuantizedHash.h + spatial_lookup/SpatialLookup.h + + store/Codec.h + store/Index.h + store/IndexFormat.h + store/IndexedStorage.h + store/NodeStatus.h + store/NodeStatusOrMissing.h + store/path_layout.h + store/RawStorage.h + store/Storage.h + store/ThreadSafeStorage.h + store/Traits.h + store/cache/Dummy.h + store/cache/Interface.h + store/cache/Lru.h + store/open.h + store/traverse.h + + texture/Mask.h + texture/dilate_colors.h texture/dilate_colors.cpp + + uv/atlas.h uv/atlas.cpp + uv/fit.h + uv/unwrap.h uv/unwrap.cpp +) +set_source_files_properties( + mesh/igl/igl.cpp mesh/io/gltf.cpp - mesh/io/terrain.cpp - mesh/io/texture.cpp - mesh/io/utils.cpp - - mesh/merging/mapping.cpp - - mesh/bfs.cpp - mesh/boolean.cpp - mesh/cleanup.cpp - mesh/clip.cpp - mesh/convert.cpp - mesh/geometry.cpp - mesh/holes.cpp - mesh/io.cpp - mesh/texture_trim.cpp - mesh/validate.cpp - - octree/storage/helpers.cpp - octree/IndexMap.cpp - octree/Space.cpp - octree/OddLevelShifted.cpp - - polygon/triangulate.cpp - polygon/utils.cpp - polygon/clip.cpp - - texture/dilate_colors.cpp - - uv/unwrap.cpp - uv/atlas.cpp - - Dataset.cpp - init.cpp - log.cpp - ProgressIndicator.cpp - tile_path.h + PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON ) target_compile_features(terrainlib PUBLIC cxx_std_23) diff --git a/src/terrainlib/Error.cpp b/src/terrainlib/Error.cpp new file mode 100644 index 00000000..e5acb8dd --- /dev/null +++ b/src/terrainlib/Error.cpp @@ -0,0 +1,139 @@ +#include "Error.h" + +#include + +#include + +namespace { + +std::string_view code_name(const Error::Code code) +{ + switch (code) { + case Error::Code::InvalidInput: + return "InvalidInput"; + case Error::Code::NotFound: + return "NotFound"; + case Error::Code::AlreadyExists: + return "AlreadyExists"; + case Error::Code::Unsupported: + return "Unsupported"; + case Error::Code::CorruptData: + return "CorruptData"; + case Error::Code::Io: + return "Io"; + case Error::Code::ResourceExhausted: + return "ResourceExhausted"; + case Error::Code::Internal: + return "Internal"; + } + return "Unknown"; +} + +std::string describe_system_error(const std::error_code& cause) { return fmt::format("{} ({}:{})", cause.message(), cause.category().name(), cause.value()); } + +} // namespace + +Error::Error(const Code code, Frame frame) + : m_code(code) +{ + m_frames.push_back(std::move(frame)); +} + +Error Error::make(const Code code, std::string message, const std::source_location location) { return Error(code, Frame { std::move(message), location }); } + +Error Error::make(const Code code, const std::string_view operation, const std::filesystem::path& path, const std::source_location location) +{ + return make(code, fmt::format("{} \"{}\"", operation, path.string()), location); +} + +Error Error::make(const Code code, const std::string_view operation, const std::error_code& cause, const std::source_location location) +{ + return make(code, fmt::format("{}: {}", operation, describe_system_error(cause)), location); +} + +Error Error::make( + const Code code, const std::string_view operation, const std::filesystem::path& path, const std::error_code& cause, const std::source_location location) +{ + return make(code, fmt::format("{} \"{}\": {}", operation, path.string(), describe_system_error(cause)), location); +} + +Error Error::make(const Code code, + const std::string_view operation, + const std::filesystem::path& source, + const std::filesystem::path& destination, + const std::error_code& cause, + const std::source_location location) +{ + return make(code, fmt::format("{} \"{}\" -> \"{}\": {}", operation, source.string(), destination.string(), describe_system_error(cause)), location); +} + +std::unexpected Error::fail(const Code code, std::string message, const std::source_location location) +{ + return std::unexpected { make(code, std::move(message), location) }; +} + +std::unexpected Error::fail(const Code code, const std::string_view operation, const std::filesystem::path& path, const std::source_location location) +{ + return std::unexpected { make(code, operation, path, location) }; +} + +std::unexpected Error::fail(const Code code, const std::string_view operation, const std::error_code& cause, const std::source_location location) +{ + return std::unexpected { make(code, operation, cause, location) }; +} + +std::unexpected Error::fail( + const Code code, const std::string_view operation, const std::filesystem::path& path, const std::error_code& cause, const std::source_location location) +{ + return std::unexpected { make(code, operation, path, cause, location) }; +} + +std::unexpected Error::fail(const Code code, + const std::string_view operation, + const std::filesystem::path& source, + const std::filesystem::path& destination, + const std::error_code& cause, + const std::source_location location) +{ + return std::unexpected { make(code, operation, source, destination, cause, location) }; +} + +std::unexpected Error::propagate(Error&& error, std::string message, const std::source_location location) +{ + return std::unexpected { std::move(error).with_context(std::move(message), location) }; +} + +Error Error::with_context(std::string message, const std::source_location location) && +{ + Error result = std::move(*this); + result.m_frames.push_back(Frame { + std::move(message), + location, + }); + return result; +} + +Error Error::reclassified(const Code code, std::string message, const std::source_location location) && +{ + Error result = std::move(*this); + result.m_frames.push_back(Frame { + fmt::format("{} (reclassified {} -> {})", message, code_name(result.m_code), code_name(code)), + location, + }); + result.m_code = code; + return result; +} + +std::string Error::to_string() const +{ + std::string result = fmt::format("[{}]", code_name(m_code)); + for (auto frame = m_frames.rbegin(); frame != m_frames.rend(); ++frame) { + result += fmt::format("\n{}{}\n at {}:{} ({})", + frame == m_frames.rbegin() ? "" : "caused by: ", + frame->message, + frame->location.file_name(), + frame->location.line(), + frame->location.function_name()); + } + return result; +} diff --git a/src/terrainlib/Error.h b/src/terrainlib/Error.h new file mode 100644 index 00000000..a304f8da --- /dev/null +++ b/src/terrainlib/Error.h @@ -0,0 +1,108 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +class Error { +public: + enum class Code { + /// A caller supplied an invalid value, such as a malformed hierarchy key or incompatible dimensions. + InvalidInput, + /// A requested file, node, or key does not exist. + NotFound, + /// Creating or publishing an object failed because the destination already exists. + AlreadyExists, + /// A requested codec, layout, format version, algorithm, or data type is not supported. + Unsupported, + /// Persisted or external data violates its format or invariants, such as a checksum mismatch or invalid topology. + CorruptData, + /// A filesystem or device operation failed, excluding failures classified more specifically above. + Io, + /// An explicit size, storage, or other recoverable resource limit was exceeded. + ResourceExhausted, + /// An internal invariant or an otherwise valid operation failed unexpectedly. + Internal, + }; + + struct Frame { + std::string message; + std::source_location location; + }; + + static Error make(Code code, std::string message, std::source_location location = std::source_location::current()); + static Error make( + Code code, std::string_view operation, const std::filesystem::path& path, std::source_location location = std::source_location::current()); + static Error make(Code code, std::string_view operation, const std::error_code& cause, std::source_location location = std::source_location::current()); + static Error make(Code code, + std::string_view operation, + const std::filesystem::path& path, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + static Error make(Code code, + std::string_view operation, + const std::filesystem::path& source, + const std::filesystem::path& destination, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + + static std::unexpected fail(Code code, std::string message, std::source_location location = std::source_location::current()); + static std::unexpected fail( + Code code, std::string_view operation, const std::filesystem::path& path, std::source_location location = std::source_location::current()); + static std::unexpected fail( + Code code, std::string_view operation, const std::error_code& cause, std::source_location location = std::source_location::current()); + static std::unexpected fail(Code code, + std::string_view operation, + const std::filesystem::path& path, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + static std::unexpected fail(Code code, + std::string_view operation, + const std::filesystem::path& source, + const std::filesystem::path& destination, + const std::error_code& cause, + std::source_location location = std::source_location::current()); + + template + static std::unexpected propagate( + std::expected&& result, std::string message = "propagated", std::source_location location = std::source_location::current()); + template + static std::unexpected propagate( + std::expected&& result, Code code, std::string message, std::source_location location = std::source_location::current()); + static std::unexpected propagate(Error&& error, std::string message = "propagated", std::source_location location = std::source_location::current()); + + Code code() const { return m_code; } + std::string to_string() const; + +private: + Error(Code code, Frame frame); + [[nodiscard]] Error with_context(std::string message, std::source_location location) &&; + [[nodiscard]] Error reclassified(Code code, std::string message, std::source_location location) &&; + + Code m_code; + std::vector m_frames; +}; + +template +using Expected = std::expected; + +template +std::unexpected Error::propagate(std::expected&& result, std::string message, const std::source_location location) +{ + return std::unexpected { + std::move(result).error().with_context(std::move(message), location), + }; +} + +template +std::unexpected Error::propagate(std::expected&& result, const Code code, std::string message, const std::source_location location) +{ + return std::unexpected { + std::move(result).error().reclassified(code, std::move(message), location), + }; +} diff --git a/src/terrainlib/ProgressIndicator.cpp b/src/terrainlib/ProgressIndicator.cpp index a9f49272..cd9a4dcf 100644 --- a/src/terrainlib/ProgressIndicator.cpp +++ b/src/terrainlib/ProgressIndicator.cpp @@ -35,11 +35,13 @@ ProgressIndicator::ProgressIndicator(size_t n_steps) } void ProgressIndicator::task_finished() { - ++m_step; - if (m_step > m_n_steps) { + const size_t step = ++m_step; + if (step > m_n_steps) { throw std::runtime_error("Too many steps reported."); } - m_monitor_condition.notify_all(); + if (step == m_n_steps) { + m_monitor_condition.notify_one(); + } } std::jthread ProgressIndicator::start_monitoring() const { diff --git a/src/terrainlib/io/Error.h b/src/terrainlib/io/Error.h deleted file mode 100644 index 831c9842..00000000 --- a/src/terrainlib/io/Error.h +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once - -#include -#include - -#include "log.h" - -namespace io { - -class Error { -public: - enum Value : uint8_t { - OpenFile, - WriteBytes, - DetermineSize, - ReadBytes, - Serialize, - Deserialize, - OutOfMemory - }; - - Error() = default; - constexpr Error(Value value) : _value(value) {} - - constexpr operator Value() const { - return this->_value; - } - explicit operator bool() const = delete; - constexpr bool operator==(Error other) const { - return this->_value == other._value; - } - constexpr bool operator!=(Error other) const { - return !(*this == other); - } - - std::string to_string() const; - -private: - Value _value; - -public: - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; -}; - -} // namespace io - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const io::Error &error, FormatContext &ctx) const { - const char *name = "Unknown"; - - switch (error) { - case io::Error::Value::OpenFile: - name = "OpenFile"; - break; - case io::Error::Value::WriteBytes: - name = "WriteBytes"; - break; - case io::Error::Value::DetermineSize: - name = "DetermineSize"; - break; - case io::Error::Value::ReadBytes: - name = "ReadBytes"; - break; - case io::Error::Value::Serialize: - name = "Serialize"; - break; - case io::Error::Value::Deserialize: - name = "Deserialize"; - break; - case io::Error::Value::OutOfMemory: - name = "OutOfMemory"; - break; - default: - UNREACHABLE(); - } - - return fmt::format_to(ctx.out(), "{}", name); - } -}; - -#include -#include -inline std::string io::Error::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const io::Error &status) { - fmt::print(os, "{}", status); - return os; -} diff --git a/src/terrainlib/io/bytes.cpp b/src/terrainlib/io/bytes.cpp index 1412ce33..7c81c557 100644 --- a/src/terrainlib/io/bytes.cpp +++ b/src/terrainlib/io/bytes.cpp @@ -1,4 +1,5 @@ #include +#include #include "io/bytes.h" #include "io/utils.h" @@ -6,41 +7,49 @@ namespace io { -std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs) { +Expected write_bytes_to_path(const std::span bytes, const std::filesystem::path& path, bool make_dirs) +{ LOG_TRACE("Writing bytes to path {}", path); if (make_dirs) { - utils::create_parent_directories(path); + const std::error_code error = utils::create_parent_directories(path); + if (error) { + return Error::fail(Error::Code::Io, "create parent directories for", path, error); + } } - + std::ofstream file(path, std::ios::binary); if (!file.is_open()) { - LOG_DEBUG("Failed to open file for writing {}", path); - return std::unexpected(Error::OpenFile); + return Error::fail(Error::Code::Io, "open file for writing", path); } file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); if (!file.good()) { - LOG_ERROR("Failed to write bytes to file {}", path); - return std::unexpected(Error::WriteBytes); + return Error::fail(Error::Code::Io, "write bytes to", path); } return {}; } -std::expected, Error> read_bytes_from_path(const std::filesystem::path& path) { +Expected> read_bytes_from_path(const std::filesystem::path& path) +{ LOG_TRACE("Reading bytes from path {}", path); std::ifstream file(path, std::ios::binary | std::ios::ate); if (!file.is_open()) { - LOG_DEBUG("Failed to open file for reading {}", path); - return std::unexpected(Error::OpenFile); + std::error_code existence_error; + const bool exists = std::filesystem::exists(path, existence_error); + if (!existence_error && !exists) { + return Error::fail(Error::Code::NotFound, "open file for reading", path); + } + return existence_error + ? Error::fail(Error::Code::Io, "check existence of", path, existence_error) + : Error::fail(Error::Code::Io, "open file for reading", path); } const std::streamsize size = file.tellg(); if (size < 0) { - LOG_ERROR("Failed to determine size for file {}", path); - return std::unexpected(Error::DetermineSize); + return Error::fail(Error::Code::Io, "determine size of", path); } std::vector buffer(static_cast(size)); @@ -48,8 +57,7 @@ std::expected, Error> read_bytes_from_path(const std::files file.read(reinterpret_cast(buffer.data()), size); if (!file.good()) { - LOG_ERROR("Failed to read bytes from file {}", path); - return std::unexpected(Error::ReadBytes); + return Error::fail(Error::Code::Io, "read bytes from", path); } return buffer; diff --git a/src/terrainlib/io/bytes.h b/src/terrainlib/io/bytes.h index be3852dc..4d1af334 100644 --- a/src/terrainlib/io/bytes.h +++ b/src/terrainlib/io/bytes.h @@ -6,11 +6,11 @@ #include -#include "io/Error.h" +#include "Error.h" namespace io { -std::expected write_bytes_to_path(const std::span bytes, const std::filesystem::path &path, bool make_dirs = true); -std::expected, Error> read_bytes_from_path(const std::filesystem::path &path); +Expected write_bytes_to_path(const std::span bytes, const std::filesystem::path& path, bool make_dirs = true); +Expected> read_bytes_from_path(const std::filesystem::path& path); } diff --git a/src/terrainlib/io/compression.cpp b/src/terrainlib/io/compression.cpp index ba18e638..ee2154fc 100644 --- a/src/terrainlib/io/compression.cpp +++ b/src/terrainlib/io/compression.cpp @@ -13,207 +13,182 @@ namespace io::envelope { namespace { -using CompressionContext = std::unique_ptr; - -std::string crc32c_checksum(const Bytes &data) -{ - static constexpr auto table = [] { - constexpr std::uint32_t polynomial = 0x82f63b78u; - - std::array values{}; - for (std::size_t index = 0; index < values.size(); ++index) { - auto crc = static_cast(index); - for (int bit = 0; bit < 8; ++bit) { - crc = (crc >> 1u) ^ ((crc & 1u) != 0u ? polynomial : 0u); + using CompressionContext = std::unique_ptr; + + std::string crc32c_checksum(const Bytes& data) + { + static constexpr auto table = [] { + constexpr std::uint32_t polynomial = 0x82f63b78u; + + std::array values {}; + for (std::size_t index = 0; index < values.size(); ++index) { + auto crc = static_cast(index); + for (int bit = 0; bit < 8; ++bit) { + crc = (crc >> 1u) ^ ((crc & 1u) != 0u ? polynomial : 0u); + } + values[index] = crc; } - values[index] = crc; - } - return values; - }(); + return values; + }(); - std::uint32_t crc = 0xffffffffu; - for (const auto value : data) { - const auto index = (crc ^ std::to_integer(value)) & 0xffu; - crc = table[index] ^ (crc >> 8u); - } - crc = ~crc; + std::uint32_t crc = 0xffffffffu; + for (const auto value : data) { + const auto index = (crc ^ std::to_integer(value)) & 0xffu; + crc = table[index] ^ (crc >> 8u); + } + crc = ~crc; - constexpr char hex_digits[] = "0123456789abcdef"; - std::string checksum(8, '0'); - for (std::size_t index = 0; index < checksum.size(); ++index) { - const auto shift = static_cast((checksum.size() - index - 1) * 4); - checksum[index] = hex_digits[(crc >> shift) & 0x0fu]; - } - return checksum; -} + constexpr char hex_digits[] = "0123456789abcdef"; + std::string checksum(8, '0'); + for (std::size_t index = 0; index < checksum.size(); ++index) { + const auto shift = static_cast((checksum.size() - index - 1) * 4); + checksum[index] = hex_digits[(crc >> shift) & 0x0fu]; + } + return checksum; + } + + Expected validate_algorithms(const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) + { + switch (checksum_algorithm) { + case ChecksumAlgorithm::None: + case ChecksumAlgorithm::HandledByCompressionLib: + case ChecksumAlgorithm::Crc32c: + break; + default: + return Error::fail(Error::Code::Unsupported, "unsupported checksum algorithm " + std::to_string(static_cast(checksum_algorithm))); + } -std::expected validate_algorithms( - const CompressionAlgorithm compression_algorithm, - const ChecksumAlgorithm checksum_algorithm) -{ - switch (checksum_algorithm) { - case ChecksumAlgorithm::None: - case ChecksumAlgorithm::HandledByCompressionLib: - case ChecksumAlgorithm::Crc32c: - break; - default: - return std::unexpected(Error{ErrorCode::UnsupportedChecksumAlgorithm}); - } + switch (compression_algorithm) { + case CompressionAlgorithm::None: + case CompressionAlgorithm::ZstdBestCompressionWithChecksum: + case CompressionAlgorithm::ZstdDefaultCompressionWithChecksum: + break; + default: + return Error::fail(Error::Code::Unsupported, "unsupported compression algorithm " + std::to_string(static_cast(compression_algorithm))); + } - switch (compression_algorithm) { - case CompressionAlgorithm::None: - case CompressionAlgorithm::ZstdBestCompressionWithChecksum: - break; - default: - return std::unexpected(Error{ErrorCode::UnsupportedCompressionAlgorithm}); - } + const bool no_compression = compression_algorithm == CompressionAlgorithm::None + && (checksum_algorithm == ChecksumAlgorithm::None || checksum_algorithm == ChecksumAlgorithm::Crc32c); + const bool zstd_with_checksum = (compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum + || compression_algorithm == CompressionAlgorithm::ZstdDefaultCompressionWithChecksum) + && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib || checksum_algorithm == ChecksumAlgorithm::Crc32c); + if (!no_compression && !zstd_with_checksum) { + return Error::fail(Error::Code::InvalidInput, "invalid compression and checksum algorithm combination"); + } - const bool no_compression = compression_algorithm == CompressionAlgorithm::None - && (checksum_algorithm == ChecksumAlgorithm::None - || checksum_algorithm == ChecksumAlgorithm::Crc32c); - const bool zstd_with_checksum = - compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum - && (checksum_algorithm == ChecksumAlgorithm::HandledByCompressionLib - || checksum_algorithm == ChecksumAlgorithm::Crc32c); - if (!no_compression && !zstd_with_checksum) { - return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + return {}; } - return {}; -} - } // namespace -std::expected compress_with_checksum( - const Bytes &uncompressed_data, - const CompressionAlgorithm compression_algorithm, - const ChecksumAlgorithm checksum_algorithm) +Expected compress_with_checksum( + const Bytes& uncompressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) { - if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { - return std::unexpected(validation.error()); + if (auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return Error::propagate(std::move(validation), "validate compression algorithms"); } if (uncompressed_data.size() > default_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + return Error::fail(Error::Code::ResourceExhausted, "uncompressed data exceeds the compression size limit"); } - const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c - ? crc32c_checksum(uncompressed_data) - : std::string{}; + const std::string checksum = checksum_algorithm == ChecksumAlgorithm::Crc32c ? crc32c_checksum(uncompressed_data) : std::string {}; if (compression_algorithm == CompressionAlgorithm::None) { - return CompressedData{uncompressed_data, checksum}; + return CompressedData { uncompressed_data, checksum }; } - CompressionContext context{ZSTD_createCCtx(), &ZSTD_freeCCtx}; + CompressionContext context { ZSTD_createCCtx(), &ZSTD_freeCCtx }; if (!context) { - return std::unexpected(Error{ErrorCode::CompressionFailed}); + return Error::fail(Error::Code::Internal, "failed to create the Zstandard compression context"); } - if (ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_compressionLevel, ZSTD_maxCLevel())) + const int compression_level = compression_algorithm == CompressionAlgorithm::ZstdBestCompressionWithChecksum ? ZSTD_maxCLevel() : 0; + if (ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_compressionLevel, compression_level)) || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { - return std::unexpected(Error{ErrorCode::CompressionFailed}); + return Error::fail(Error::Code::Internal, "failed to configure the Zstandard compression context"); } const std::size_t capacity = ZSTD_compressBound(uncompressed_data.size()); Bytes compressed_data(capacity); - const std::size_t compressed_size = ZSTD_compress2( - context.get(), - compressed_data.data(), - compressed_data.size(), - uncompressed_data.data(), - uncompressed_data.size()); + const std::size_t compressed_size + = ZSTD_compress2(context.get(), compressed_data.data(), compressed_data.size(), uncompressed_data.data(), uncompressed_data.size()); if (ZSTD_isError(compressed_size)) { - return std::unexpected(Error{ErrorCode::CompressionFailed}); + return Error::fail(Error::Code::Internal, std::string("Zstandard compression failed: ") + ZSTD_getErrorName(compressed_size)); } compressed_data.resize(compressed_size); - return CompressedData{std::move(compressed_data), checksum}; + return CompressedData { std::move(compressed_data), checksum }; } -std::expected checked_decompress( - const Bytes &compressed_data, +Expected checked_decompress(const Bytes& compressed_data, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm, const std::string_view checksum, const std::size_t max_decompressed_size) { - if (const auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { - return std::unexpected(validation.error()); + if (auto validation = validate_algorithms(compression_algorithm, checksum_algorithm); !validation) { + return Error::propagate(std::move(validation), "validate decompression algorithms"); } if (checksum_algorithm != ChecksumAlgorithm::Crc32c && !checksum.empty()) { - return std::unexpected(Error{ErrorCode::InvalidAlgorithmCombination}); + return Error::fail(Error::Code::InvalidInput, "a checksum was supplied for an algorithm that does not accept one"); } - if (checksum_algorithm == ChecksumAlgorithm::Crc32c - && (checksum.size() != 8 - || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { - return (character >= '0' && character <= '9') - || (character >= 'a' && character <= 'f'); - }))) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + if (checksum_algorithm == ChecksumAlgorithm::Crc32c && (checksum.size() != 8 || !std::all_of(checksum.begin(), checksum.end(), [](const char character) { + return (character >= '0' && character <= '9') || (character >= 'a' && character <= 'f'); + }))) { + return Error::fail(Error::Code::CorruptData, "invalid CRC32C checksum encoding"); } - const std::size_t effective_max_decompressed_size = - std::min(max_decompressed_size, default_max_decompressed_size); + const std::size_t effective_max_decompressed_size = std::min(max_decompressed_size, default_max_decompressed_size); Bytes uncompressed_data; if (compression_algorithm == CompressionAlgorithm::None) { if (compressed_data.size() > effective_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + return Error::fail(Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit"); } uncompressed_data = compressed_data; } else { - ZSTD_frameHeader frame_header{}; - const std::size_t frame_header_result = ZSTD_getFrameHeader( - &frame_header, compressed_data.data(), compressed_data.size()); + ZSTD_frameHeader frame_header {}; + const std::size_t frame_header_result = ZSTD_getFrameHeader(&frame_header, compressed_data.data(), compressed_data.size()); if (ZSTD_isError(frame_header_result) || frame_header_result != 0) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + return Error::fail(Error::Code::CorruptData, "invalid or incomplete Zstandard frame header"); } if (frame_header.checksumFlag == 0) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + return Error::fail(Error::Code::CorruptData, "Zstandard frame does not contain the required checksum"); } - const unsigned long long frame_content_size = - ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); + const unsigned long long frame_content_size = ZSTD_getFrameContentSize(compressed_data.data(), compressed_data.size()); if (frame_content_size == ZSTD_CONTENTSIZE_ERROR - || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN - && frame_content_size > std::numeric_limits::max())) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + || (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN && frame_content_size > std::numeric_limits::max())) { + return Error::fail(Error::Code::CorruptData, "invalid Zstandard frame content size"); } const bool content_size_is_known = frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN; if (content_size_is_known && frame_content_size > effective_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + return Error::fail(Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit"); } - const std::size_t allocation_size = content_size_is_known - ? static_cast(frame_content_size) - : effective_max_decompressed_size; + const std::size_t allocation_size = content_size_is_known ? static_cast(frame_content_size) : effective_max_decompressed_size; uncompressed_data.resize(allocation_size); - const std::size_t decompressed_size = ZSTD_decompress( - uncompressed_data.data(), - uncompressed_data.size(), - compressed_data.data(), - compressed_data.size()); + const std::size_t decompressed_size + = ZSTD_decompress(uncompressed_data.data(), uncompressed_data.size(), compressed_data.data(), compressed_data.size()); if (ZSTD_isError(decompressed_size)) { if (ZSTD_getErrorCode(decompressed_size) == ZSTD_error_checksum_wrong) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + return Error::fail(Error::Code::CorruptData, "Zstandard checksum mismatch"); } - if (!content_size_is_known - && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + if (!content_size_is_known && ZSTD_getErrorCode(decompressed_size) == ZSTD_error_dstSize_tooSmall) { + return Error::fail(Error::Code::ResourceExhausted, "decompressed data exceeds the configured size limit"); } - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + return Error::fail(Error::Code::CorruptData, std::string("Zstandard decompression failed: ") + ZSTD_getErrorName(decompressed_size)); } if (content_size_is_known && decompressed_size != uncompressed_data.size()) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + return Error::fail(Error::Code::CorruptData, "Zstandard decompressed size does not match its frame header"); } uncompressed_data.resize(decompressed_size); } - if (checksum_algorithm == ChecksumAlgorithm::Crc32c - && crc32c_checksum(uncompressed_data) != checksum) { - return std::unexpected(Error{ErrorCode::ChecksumMismatch}); + if (checksum_algorithm == ChecksumAlgorithm::Crc32c && crc32c_checksum(uncompressed_data) != checksum) { + return Error::fail(Error::Code::CorruptData, "CRC32C checksum mismatch"); } return uncompressed_data; } diff --git a/src/terrainlib/io/compression.h b/src/terrainlib/io/compression.h index 104f9edd..5c5b45a2 100644 --- a/src/terrainlib/io/compression.h +++ b/src/terrainlib/io/compression.h @@ -9,11 +9,13 @@ #include +#include "Error.h" + namespace io::envelope { using Bytes = std::vector; -inline constexpr std::size_t default_max_decompressed_size = std::size_t{1} << 30; +inline constexpr std::size_t default_max_decompressed_size = std::size_t { 1 } << 30; enum class ChecksumAlgorithm : std::uint8_t { None, @@ -24,28 +26,7 @@ enum class ChecksumAlgorithm : std::uint8_t { enum class CompressionAlgorithm : std::uint8_t { None, ZstdBestCompressionWithChecksum, -}; - -enum class ErrorCode : std::uint8_t { - SerializationFailed, - DeserializationFailed, - InvalidMagic, - WrongClassName, - UnsupportedClassVersion, - UnsupportedChecksumAlgorithm, - UnsupportedCompressionAlgorithm, - InvalidAlgorithmCombination, - ChecksumMismatch, - CompressionFailed, - DecompressionFailed, - SizeLimitExceeded, -}; - -struct Error { - ErrorCode code; - std::errc serialization_error{}; - - constexpr bool operator==(const Error &) const = default; + ZstdDefaultCompressionWithChecksum, }; struct CompressedData { @@ -53,13 +34,10 @@ struct CompressedData { std::string checksum; }; -std::expected compress_with_checksum( - const Bytes &uncompressed_data, - CompressionAlgorithm compression_algorithm, - ChecksumAlgorithm checksum_algorithm); +Expected compress_with_checksum( + const Bytes& uncompressed_data, CompressionAlgorithm compression_algorithm, ChecksumAlgorithm checksum_algorithm); -std::expected checked_decompress( - const Bytes &compressed_data, +Expected checked_decompress(const Bytes& compressed_data, CompressionAlgorithm compression_algorithm, ChecksumAlgorithm checksum_algorithm, std::string_view checksum, diff --git a/src/terrainlib/io/conversion.h b/src/terrainlib/io/conversion.h new file mode 100644 index 00000000..620ba9f8 --- /dev/null +++ b/src/terrainlib/io/conversion.h @@ -0,0 +1,196 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "Error.h" + +namespace io::conversion { + +namespace detail { + + inline std::unexpected allocation_error(const std::source_location location = std::source_location::current()) noexcept + { + return Error::fail(Error::Code::ResourceExhausted, "allocate raster conversion output", location); + } + + template + inline constexpr int cv_depth = -1; + + template <> + inline constexpr int cv_depth = CV_8U; + template <> + inline constexpr int cv_depth = CV_8S; + template <> + inline constexpr int cv_depth = CV_16U; + template <> + inline constexpr int cv_depth = CV_16S; + template <> + inline constexpr int cv_depth = CV_32S; + template <> + inline constexpr int cv_depth = CV_32F; + template <> + inline constexpr int cv_depth = CV_64F; + + template + struct PixelTraits { + using channel_type = std::remove_cv_t; + static constexpr int channels = 1; + static constexpr bool supported = cv_depth >= 0; + + static channel_type& channel(Pixel& pixel, const int) { return pixel; } + + static const channel_type& channel(const Pixel& pixel, const int) { return pixel; } + }; + + template + struct PixelTraits> { + using channel_type = Channel; + static constexpr int channels = static_cast(Length); + static constexpr bool supported = channels >= 2 && channels <= 4 && cv_depth >= 0; + + static channel_type& channel(glm::vec& pixel, const int index) { return pixel[static_cast(index)]; } + + static const channel_type& channel(const glm::vec& pixel, const int index) + { + return pixel[static_cast(index)]; + } + }; + + template + using Traits = PixelTraits>; + + template + inline constexpr int cv_type = Traits::supported ? CV_MAKETYPE(cv_depth::channel_type>, Traits::channels) : -1; + + template + inline constexpr bool has_packed_layout = Traits::supported + && std::is_trivially_copyable_v && sizeof(Pixel) == sizeof(typename Traits::channel_type) * Traits::channels; + + template + void copy_mat_to_raster(const cv::Mat& source, radix::Raster& destination) + { + using PixelInfo = Traits; + using Channel = typename PixelInfo::channel_type; + + const auto row_bytes = static_cast(source.cols) * sizeof(Pixel); + if constexpr (has_packed_layout) { + for (int row = 0; row < source.rows; ++row) { + std::memcpy(destination.data() + static_cast(row) * source.cols, source.ptr(row), row_bytes); + } + } else { + for (int row = 0; row < source.rows; ++row) { + const Channel* source_row = source.ptr(row); + for (int column = 0; column < source.cols; ++column) { + Pixel& pixel = destination.pixel({ + static_cast(column), + static_cast(row), + }); + for (int channel = 0; channel < PixelInfo::channels; ++channel) { + PixelInfo::channel(pixel, channel) = source_row[column * PixelInfo::channels + channel]; + } + } + } + } + } + + template + void copy_raster_to_mat(const radix::Raster& source, cv::Mat& destination) + { + using PixelInfo = Traits; + using Channel = typename PixelInfo::channel_type; + + const auto row_bytes = static_cast(destination.cols) * sizeof(Pixel); + if constexpr (has_packed_layout) { + for (int row = 0; row < destination.rows; ++row) { + std::memcpy(destination.ptr(row), source.data() + static_cast(row) * destination.cols, row_bytes); + } + } else { + for (int row = 0; row < destination.rows; ++row) { + Channel* destination_row = destination.ptr(row); + for (int column = 0; column < destination.cols; ++column) { + const Pixel& pixel = source.pixel({ + static_cast(column), + static_cast(row), + }); + for (int channel = 0; channel < PixelInfo::channels; ++channel) { + destination_row[column * PixelInfo::channels + channel] = PixelInfo::channel(pixel, channel); + } + } + } + } + } + +} // namespace detail + +template +Expected> to_raster(const cv::Mat& source) +{ + if constexpr (!detail::Traits::supported) { + return Error::fail(Error::Code::Unsupported, "unsupported raster pixel type"); + } else { + if (source.empty()) { + return Error::fail(Error::Code::InvalidInput, "cannot convert an empty OpenCV matrix to a raster"); + } + if (source.dims != 2) { + return Error::fail(Error::Code::Unsupported, "unsupported OpenCV matrix dimension count " + std::to_string(source.dims)); + } + if (source.type() != detail::cv_type) { + return Error::fail(Error::Code::InvalidInput, + "OpenCV matrix type " + std::to_string(source.type()) + " does not match expected type " + std::to_string(detail::cv_type)); + } + if (source.cols < 0 || source.rows < 0 || static_cast(source.cols) > std::numeric_limits::max() + || static_cast(source.rows) > std::numeric_limits::max()) { + return Error::fail(Error::Code::InvalidInput, "OpenCV matrix dimensions are outside the raster range"); + } + + try { + radix::Raster destination({ + static_cast(source.cols), + static_cast(source.rows), + }); + detail::copy_mat_to_raster(source, destination); + return destination; + } catch (const std::bad_alloc&) { + return detail::allocation_error(); + } catch (const cv::Exception& error) { + return Error::fail(Error::Code::Internal, "OpenCV raster conversion failed: " + error.msg); + } + } +} + +template +Expected to_mat(const radix::Raster& source) +{ + if constexpr (!detail::Traits::supported) { + return Error::fail(Error::Code::Unsupported, "unsupported raster pixel type"); + } else { + if (source.width() == 0 || source.height() == 0) { + return Error::fail(Error::Code::InvalidInput, "cannot convert an empty raster to an OpenCV matrix"); + } + if (source.width() > static_cast(std::numeric_limits::max()) + || source.height() > static_cast(std::numeric_limits::max())) { + return Error::fail(Error::Code::InvalidInput, "raster dimensions are outside the OpenCV matrix range"); + } + + try { + cv::Mat destination(static_cast(source.height()), static_cast(source.width()), detail::cv_type); + detail::copy_raster_to_mat(source, destination); + return destination; + } catch (const std::bad_alloc&) { + return detail::allocation_error(); + } catch (const cv::Exception& error) { + return Error::fail(Error::Code::Internal, "OpenCV raster conversion failed: " + error.msg); + } + } +} + +} // namespace io::conversion diff --git a/src/terrainlib/io/envelope.h b/src/terrainlib/io/envelope.h index 4b36065b..cf0e770e 100644 --- a/src/terrainlib/io/envelope.h +++ b/src/terrainlib/io/envelope.h @@ -32,12 +32,9 @@ template struct FixedString { char value[Size]; - constexpr FixedString(const char (&text)[Size]) - { - std::copy_n(text, Size, value); - } + constexpr FixedString(const char (&text)[Size]) { std::copy_n(text, Size, value); } - constexpr auto operator<=>(const FixedString &) const = default; + constexpr auto operator<=>(const FixedString&) const = default; }; template @@ -48,52 +45,50 @@ struct Version { namespace detail { -template -struct FindVersion; + template + struct FindVersion; -template -struct FindVersion { - using type = std::conditional_t::type>; -}; + template + struct FindVersion { + using type = std::conditional_t::type>; + }; -template -struct FindVersion { - using type = void; -}; + template + struct FindVersion { + using type = void; + }; -template -consteval bool versions_are_strictly_increasing() -{ - constexpr std::uint32_t numbers[] = {Versions::number...}; - for (std::size_t index = 1; index < sizeof...(Versions); ++index) { - if (numbers[index - 1] >= numbers[index]) { - return false; + template + consteval bool versions_are_strictly_increasing() + { + constexpr std::uint32_t numbers[] = { Versions::number... }; + for (std::size_t index = 1; index < sizeof...(Versions); ++index) { + if (numbers[index - 1] >= numbers[index]) { + return false; + } } + return true; } - return true; -} -template -concept ConvertsFromPrevious = requires(From previous) { - { To::from_previous(std::move(previous)) } -> std::same_as; -}; + template + concept ConvertsFromPrevious = requires(From previous) { + { To::from_previous(std::move(previous)) } -> std::same_as; + }; -template -consteval bool conversions_are_valid(std::index_sequence) -{ - return (ConvertsFromPrevious< - typename std::tuple_element_t::payload_type, - typename std::tuple_element_t::payload_type> + template + consteval bool conversions_are_valid(std::index_sequence) + { + return (ConvertsFromPrevious::payload_type, + typename std::tuple_element_t::payload_type> && ...); -} + } } // namespace detail template struct PayloadSchema { static_assert(sizeof...(Versions) > 0, "a payload schema requires at least one version"); - static_assert(detail::versions_are_strictly_increasing(), - "payload versions must be strictly increasing"); + static_assert(detail::versions_are_strictly_increasing(), "payload versions must be strictly increasing"); using version_tuple = std::tuple; static constexpr std::size_t version_count = sizeof...(Versions); @@ -104,37 +99,33 @@ struct PayloadSchema { using latest_version_descriptor = version_at; using latest_type = typename latest_version_descriptor::payload_type; - static constexpr std::string_view class_name{ClassName.value, sizeof(ClassName.value) - 1}; + static constexpr std::string_view class_name { ClassName.value, sizeof(ClassName.value) - 1 }; static constexpr std::uint32_t latest_version = latest_version_descriptor::number; template - static constexpr bool supports_version = - !std::is_void_v::type>; + static constexpr bool supports_version = !std::is_void_v::type>; - static constexpr bool supports_version_number(const std::uint32_t number) - { - return ((number == Versions::number) || ...); - } + static constexpr bool supports_version_number(const std::uint32_t number) { return ((number == Versions::number) || ...); } template using payload_type = typename detail::FindVersion::type::payload_type; - static_assert(version_count == 1 - || detail::conversions_are_valid( - std::make_index_sequence{}), - "each payload version must provide from_previous for the preceding version"); + static_assert(version_count == 1 || detail::conversions_are_valid(std::make_index_sequence {}), + "each payload version must provide from_previous for the preceding version"); }; template -std::expected serialize( - const typename Schema::template payload_type &payload, - CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdBestCompressionWithChecksum, +Expected serialize(const typename Schema::template payload_type& payload, + CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, + ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); + +template +Expected serialize(const typename Schema::latest_type& payload, + CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib); template -std::expected deserialize( - std::span bytes, - std::size_t max_decompressed_size = default_max_decompressed_size); +Expected deserialize(std::span bytes, std::size_t max_decompressed_size = default_max_decompressed_size); } // namespace io::envelope diff --git a/src/terrainlib/io/envelope.inl b/src/terrainlib/io/envelope.inl index c9055894..f71348d0 100644 --- a/src/terrainlib/io/envelope.inl +++ b/src/terrainlib/io/envelope.inl @@ -8,28 +8,28 @@ namespace io::envelope { namespace detail { template -std::expected serialize_to_bytes(const Value &value) +Expected serialize_to_bytes(const Value& value) { Bytes bytes; zpp::bits::out output(bytes, zpp::bits::alloc_limit()); const zpp::bits::errc result = output(value); if (zpp::bits::failure(result)) { - return std::unexpected(Error{ErrorCode::SerializationFailed, result.code}); + const auto cause = std::make_error_code(result.code); + const auto code = result.code == std::errc::not_enough_memory ? Error::Code::ResourceExhausted : Error::Code::Internal; + return Error::fail(code, "serialize envelope data", cause); } return bytes; } template -std::expected deserialize_from_bytes(const std::span bytes) +Expected deserialize_from_bytes(const std::span bytes) { Value value{}; zpp::bits::in input(bytes, zpp::bits::alloc_limit()); const zpp::bits::errc result = input(value); if (zpp::bits::failure(result) || input.position() != bytes.size()) { - return std::unexpected(Error{ - ErrorCode::DeserializationFailed, - zpp::bits::failure(result) ? result.code : std::errc::bad_message, - }); + const auto cause = std::make_error_code(zpp::bits::failure(result) ? result.code : std::errc::bad_message); + return Error::fail(Error::Code::CorruptData, "deserialize envelope data", cause); } return value; } @@ -46,7 +46,7 @@ typename Schema::latest_type convert_to_latest(Current current) } template -std::expected deserialize_version( +Expected deserialize_version( const std::uint32_t class_version, const std::span payload_bytes) { @@ -54,7 +54,7 @@ std::expected deserialize_version( if (class_version == CurrentVersion::number) { auto payload = deserialize_from_bytes(payload_bytes); if (!payload) { - return std::unexpected(payload.error()); + return Error::propagate(std::move(payload), "deserializing versioned envelope payload"); } return convert_to_latest(std::move(*payload)); } @@ -62,14 +62,15 @@ std::expected deserialize_version( if constexpr (Index + 1 < Schema::version_count) { return deserialize_version(class_version, payload_bytes); } else { - return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + return Error::fail(Error::Code::Unsupported, + "unsupported envelope class version " + std::to_string(class_version)); } } } // namespace detail template -std::expected serialize( +Expected serialize( const typename Schema::template payload_type &payload, const CompressionAlgorithm compression_algorithm, const ChecksumAlgorithm checksum_algorithm) @@ -79,12 +80,12 @@ std::expected serialize( auto payload_bytes = detail::serialize_to_bytes(payload); if (!payload_bytes) { - return std::unexpected(payload_bytes.error()); + return Error::propagate(std::move(payload_bytes), "serializing envelope payload"); } auto compressed = compress_with_checksum(*payload_bytes, compression_algorithm, checksum_algorithm); if (!compressed) { - return std::unexpected(compressed.error()); + return Error::propagate(std::move(compressed), "compressing envelope payload"); } const Envelope envelope{ @@ -101,29 +102,43 @@ std::expected serialize( } template -std::expected deserialize( +Expected serialize( + const typename Schema::latest_type &payload, + const CompressionAlgorithm compression_algorithm, + const ChecksumAlgorithm checksum_algorithm) +{ + return serialize( + payload, + compression_algorithm, + checksum_algorithm); +} + +template +Expected deserialize( const std::span bytes, const std::size_t max_decompressed_size) { auto envelope = detail::deserialize_from_bytes(bytes); if (!envelope) { - return std::unexpected(envelope.error()); + return Error::propagate(std::move(envelope), "reading envelope header"); } if (envelope->magic != magic) { - return std::unexpected(Error{ErrorCode::InvalidMagic}); + return Error::fail(Error::Code::CorruptData, "invalid envelope magic"); } if (envelope->class_name != Schema::class_name) { - return std::unexpected(Error{ErrorCode::WrongClassName}); + return Error::fail(Error::Code::CorruptData, + "unexpected envelope class \"" + envelope->class_name + "\", expected \"" + std::string(Schema::class_name) + "\""); } if (!Schema::supports_version_number(envelope->class_version)) { - return std::unexpected(Error{ErrorCode::UnsupportedClassVersion}); + return Error::fail(Error::Code::Unsupported, + "unsupported envelope class version " + std::to_string(envelope->class_version)); } const std::size_t effective_max_decompressed_size = std::min(max_decompressed_size, default_max_decompressed_size); if (envelope->uncompressed_size > std::numeric_limits::max() || envelope->uncompressed_size > effective_max_decompressed_size) { - return std::unexpected(Error{ErrorCode::SizeLimitExceeded}); + return Error::fail(Error::Code::ResourceExhausted, "envelope payload exceeds the configured size limit"); } auto payload_bytes = checked_decompress( @@ -133,13 +148,18 @@ std::expected deserialize( envelope->checksum, static_cast(envelope->uncompressed_size)); if (!payload_bytes) { - if (payload_bytes.error().code == ErrorCode::SizeLimitExceeded) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + if (payload_bytes.error().code() == Error::Code::ResourceExhausted) { + return Error::propagate(std::move(payload_bytes), + Error::Code::CorruptData, "decompressed payload exceeds the size declared by the envelope"); + } + if (payload_bytes.error().code() == Error::Code::InvalidInput) { + return Error::propagate(std::move(payload_bytes), + Error::Code::CorruptData, "envelope declares an invalid compression and checksum combination"); } - return std::unexpected(payload_bytes.error()); + return Error::propagate(std::move(payload_bytes), "decompressing envelope payload"); } if (payload_bytes->size() != envelope->uncompressed_size) { - return std::unexpected(Error{ErrorCode::DecompressionFailed}); + return Error::fail(Error::Code::CorruptData, "decompressed payload size does not match the envelope declaration"); } return detail::deserialize_version(envelope->class_version, *payload_bytes); diff --git a/src/terrainlib/io/envelope_file.h b/src/terrainlib/io/envelope_file.h new file mode 100644 index 00000000..daa5d5c4 --- /dev/null +++ b/src/terrainlib/io/envelope_file.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "io/bytes.h" +#include "io/envelope.h" + +namespace io::envelope { + +template +Expected read_from_path( + const std::filesystem::path& path, const std::size_t max_decompressed_size = default_max_decompressed_size) +{ + auto file_bytes = ::io::read_bytes_from_path(path); + if (!file_bytes) { + return Error::propagate(std::move(file_bytes), "reading envelope file \"" + path.string() + "\""); + } + const auto bytes = std::as_bytes(std::span { *file_bytes }); + auto result = deserialize(bytes, max_decompressed_size); + if (!result) { + return Error::propagate(std::move(result), "decoding envelope file \"" + path.string() + "\""); + } + return std::move(*result); +} + +template +Expected write_to_path(const typename Schema::latest_type& payload, + const std::filesystem::path& path, + const bool make_dirs = true, + const CompressionAlgorithm compression_algorithm = CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, + const ChecksumAlgorithm checksum_algorithm = ChecksumAlgorithm::HandledByCompressionLib) +{ + auto serialized = serialize(payload, compression_algorithm, checksum_algorithm); + if (!serialized) { + return Error::propagate(std::move(serialized), "encoding envelope file \"" + path.string() + "\""); + } + const auto bytes = std::span { + reinterpret_cast(serialized->data()), + serialized->size(), + }; + auto result = ::io::write_bytes_to_path(bytes, path, make_dirs); + if (!result) { + return Error::propagate(std::move(result), "writing envelope file \"" + path.string() + "\""); + } + return {}; +} + +} // namespace io::envelope diff --git a/src/terrainlib/io/serialize.h b/src/terrainlib/io/serialize.h deleted file mode 100644 index ffda2d1f..00000000 --- a/src/terrainlib/io/serialize.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "io/Error.h" - -namespace io { - -template -std::expected, Error> write_to_bytes(const T &value); -template -std::expected read_from_bytes(const std::span bytes); - -template -std::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs = true); -template -std::expected read_from_path(const std::filesystem::path &path); - -} - -#include "io/serialize.inl" diff --git a/src/terrainlib/io/serialize.inl b/src/terrainlib/io/serialize.inl deleted file mode 100644 index 80324d03..00000000 --- a/src/terrainlib/io/serialize.inl +++ /dev/null @@ -1,88 +0,0 @@ -#pragma once - -#include - -#include "io/bytes.h" -#include "io/utils.h" -#include "type_utils.h" - -namespace io { - -template -std::expected, Error> write_to_bytes(const T &value) { - std::vector data; - zpp::bits::out out(data); - const auto result = out(value); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while writing {}: {}", type_name(), error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. - case std::errc::message_size: // message size is beyond the user defined allocation limits. - case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return std::unexpected(Error::OutOfMemory); - case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. - case std::errc::bad_message: // attempt to read a variant of unrecognized type. - case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return std::unexpected(Error::Serialize); - case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. - case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. - UNREACHABLE(); - default: - UNREACHABLE(); - } - } - return data; -} - -template -std::expected read_from_bytes(const std::span bytes) { - zpp::bits::in in(bytes); - T value; - const auto result = in(value); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while reading {}: {}", type_name(), error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: // growing buffer would grow beyond the allocation limits or overflow. - case std::errc::message_size: // message size is beyond the user defined allocation limits. - case std::errc::result_out_of_range: // attempting to write or read from a too short buffer. - return std::unexpected(Error::OutOfMemory); - case std::errc::value_too_large: // varint (variable length integer) encoding is beyond the representation limits. - case std::errc::bad_message: // attempt to read a variant of unrecognized type. - case std::errc::invalid_argument: // attempting to serialize null pointer or a value-less variant. - return std::unexpected(Error::Deserialize); - case std::errc::protocol_error: // attempt to deserialize an invalid protocol message. - case std::errc::not_supported: // attempt to call an RPC that is not listed as supported. - UNREACHABLE(); - default: - UNREACHABLE(); - } - } - return value; -} - -template -std::expected read_from_path(const std::filesystem::path &path) { - const auto result = read_bytes_from_path(path); - if (!result.has_value()) { - return std::unexpected(result.error()); - } - const std::vector bytes = result.value(); - return read_from_bytes(bytes); -} - -template -std::expected write_to_path(const T &value, const std::filesystem::path &path, bool make_dirs) { - const auto result = write_to_bytes(value); - if (!result.has_value()) { - return std::unexpected(result.error()); - } - - const std::vector bytes = result.value(); - return write_bytes_to_path(bytes, path, make_dirs); -} - -} // namespace io diff --git a/src/terrainlib/io/utils.cpp b/src/terrainlib/io/utils.cpp index 85b5c77f..0d8f6298 100644 --- a/src/terrainlib/io/utils.cpp +++ b/src/terrainlib/io/utils.cpp @@ -2,10 +2,14 @@ namespace io::utils { -std::filesystem::path create_parent_directories(const std::filesystem::path &path) { - const std::filesystem::path parent_path = std::filesystem::absolute(path).parent_path(); - std::filesystem::create_directories(parent_path); - return parent_path; +std::error_code create_parent_directories(const std::filesystem::path &path) { + std::error_code error; + const std::filesystem::path parent_path = std::filesystem::absolute(path, error).parent_path(); + if (error) { + return error; + } + std::filesystem::create_directories(parent_path, error); + return error; } } // namespace io::utils diff --git a/src/terrainlib/io/utils.h b/src/terrainlib/io/utils.h index 8b6b4e69..aa369d69 100644 --- a/src/terrainlib/io/utils.h +++ b/src/terrainlib/io/utils.h @@ -1,9 +1,10 @@ #pragma once #include +#include namespace io::utils { -std::filesystem::path create_parent_directories(const std::filesystem::path &path); +[[nodiscard]] std::error_code create_parent_directories(const std::filesystem::path &path); } // namespace io::utils diff --git a/src/terrainlib/mesh/EncodedMesh.h b/src/terrainlib/mesh/EncodedMesh.h index 4e9e4610..fd0927a6 100644 --- a/src/terrainlib/mesh/EncodedMesh.h +++ b/src/terrainlib/mesh/EncodedMesh.h @@ -3,8 +3,6 @@ #include #include -#include - namespace mesh { class Encoded { @@ -20,7 +18,6 @@ class Encoded { bool operator!=(const Header &) const = default; }; - using serialize = zpp::bits::members<5>; Header header; std::vector triangles; std::vector positions; diff --git a/src/terrainlib/mesh/codec/Gltf.h b/src/terrainlib/mesh/codec/Gltf.h new file mode 100644 index 00000000..61e5aa35 --- /dev/null +++ b/src/terrainlib/mesh/codec/Gltf.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/io/gltf.h" +#include "store/Codec.h" + +namespace mesh::codec { + +enum class GltfContainer { + Binary, + Json, +}; + +class Gltf final : public store::Codec { +public: + explicit Gltf(const GltfContainer container) + : m_container(container) + { + } + + std::vector paths(const std::filesystem::path& node_path) const override + { + return { add_extension(node_path, m_container == GltfContainer::Binary ? ".glb" : ".gltf") }; + } + + Expected read(const std::filesystem::path& node_path) const override + { + const std::filesystem::path path = paths(node_path).front(); + auto result = mesh::io::gltf::load_from_path(path); + if (!result) { + return Error::propagate(std::move(result), "read glTF mesh \"" + path.string() + "\""); + } + return std::move(*result); + } + + Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override + { + const std::filesystem::path path = paths(node_path).front(); + try { + auto result = mesh::io::gltf::save_to_path(mesh, path); + if (!result) { + return Error::propagate(std::move(result), "write glTF mesh \"" + path.string() + "\""); + } + return {}; + } catch (const std::bad_alloc&) { + return Error::fail(Error::Code::ResourceExhausted, "write glTF mesh \"" + path.string() + "\": out of memory"); + } catch (const std::exception& error) { + return Error::fail(Error::Code::Internal, "write glTF mesh \"" + path.string() + "\": " + error.what()); + } catch (...) { + return Error::fail(Error::Code::Internal, "write glTF mesh \"" + path.string() + "\": unknown exception"); + } + } + + GltfContainer container() const { return m_container; } + +private: + GltfContainer m_container; +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/SfMesh.h b/src/terrainlib/mesh/codec/SfMesh.h new file mode 100644 index 00000000..3c0de760 --- /dev/null +++ b/src/terrainlib/mesh/codec/SfMesh.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +#include "io/envelope_file.h" +#include "mesh/SimpleMesh.h" +#include "mesh/codec/SfMeshFormat.h" +#include "store/Codec.h" + +namespace mesh::codec { + +class SfMesh final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override { return { add_extension(node_path, ".sfmesh") }; } + + Expected read(const std::filesystem::path& node_path) const override + { + const std::filesystem::path path = paths(node_path).front(); + auto payload = ::io::envelope::read_from_path(path); + if (!payload) { + return Error::propagate(std::move(payload), "read SF mesh"); + } + if (auto valid = mesh::sf::validate(*payload); !valid) { + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate SF mesh read from \"" + path.string() + "\""); + } + auto decoded = mesh::sf::decode_payload(std::move(*payload)); + if (!decoded) { + return Error::propagate(std::move(decoded), "decode SF mesh \"" + path.string() + "\""); + } + return std::move(*decoded); + } + + Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh) const override { return write(node_path, mesh, {}); } + + Expected write(const std::filesystem::path& node_path, const mesh::Simple& mesh, const mesh::EncodeOptions options) const + { + const std::filesystem::path path = paths(node_path).front(); + auto payload = mesh::sf::encode_payload(mesh, options); + if (!payload) { + return Error::propagate(std::move(payload), "encode SF mesh \"" + path.string() + "\""); + } + if (auto valid = mesh::sf::validate(*payload); !valid) { + return Error::propagate(std::move(valid), "validate SF mesh for \"" + path.string() + "\""); + } + auto result = ::io::envelope::write_to_path(*payload, path); + if (!result) { + return Error::propagate(std::move(result), "write SF mesh"); + } + return {}; + } +}; + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/codec/SfMeshFormat.h b/src/terrainlib/mesh/codec/SfMeshFormat.h new file mode 100644 index 00000000..d97de5cd --- /dev/null +++ b/src/terrainlib/mesh/codec/SfMeshFormat.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include + +#include "Error.h" +#include "io/envelope.h" +#include "mesh/EncodedMesh.h" +#include "mesh/SimpleMesh.h" +#include "mesh/encode.h" + +namespace mesh::sf { + +namespace v1 { + + struct Payload { + std::uint32_t vertex_count; + std::uint32_t face_count; + std::vector triangles; + std::vector positions; + std::vector uvs; + std::vector texture; + }; + +} // namespace v1 + +using Schema = ::io::envelope::PayloadSchema<"mesh.SfMesh", ::io::envelope::Version<1, v1::Payload>>; +using Payload = Schema::latest_type; + +inline Expected validate(const Payload& payload) +{ + if (payload.vertex_count > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Position)) { + return Error::fail(Error::Code::InvalidInput, "SF mesh vertex count exceeds the allocation limit"); + } + if (payload.face_count > ::io::envelope::default_max_decompressed_size / sizeof(mesh::Simple::Triangle)) { + return Error::fail(Error::Code::InvalidInput, "SF mesh face count exceeds the allocation limit"); + } + if (payload.vertex_count == 0 && (!payload.positions.empty() || !payload.uvs.empty())) { + return Error::fail(Error::Code::InvalidInput, "empty mesh contains vertex data"); + } + if (payload.vertex_count != 0 && payload.positions.empty()) { + return Error::fail(Error::Code::InvalidInput, "non-empty mesh contains no position data"); + } + if (payload.face_count == 0 && !payload.triangles.empty()) { + return Error::fail(Error::Code::InvalidInput, "empty mesh contains triangle data"); + } + if (payload.face_count != 0 && payload.triangles.empty()) { + return Error::fail(Error::Code::InvalidInput, "non-empty mesh contains no triangle data"); + } + return {}; +} + +inline Expected encode_payload(const mesh::Simple& mesh, const mesh::EncodeOptions options = {}) +{ + auto encoded = mesh::encode(mesh, options); + if (!encoded) { + return Error::fail(Error::Code::InvalidInput, "could not encode SF mesh payload"); + } + return Payload { + .vertex_count = encoded->header.vertex_count, + .face_count = encoded->header.face_count, + .triangles = std::move(encoded->triangles), + .positions = std::move(encoded->positions), + .uvs = std::move(encoded->uvs), + .texture = std::move(encoded->texture), + }; +} + +inline Expected decode_payload(Payload payload) +{ + const mesh::Encoded encoded{ + .header = { + .version = 1, + .n_dims = mesh::Simple::n_dims, + .component_type = mesh::component_type_id(), + .vertex_count = payload.vertex_count, + .face_count = payload.face_count, + }, + .triangles = std::move(payload.triangles), + .positions = std::move(payload.positions), + .uvs = std::move(payload.uvs), + .texture = std::move(payload.texture), + }; + auto decoded = mesh::decode(encoded); + if (!decoded) { + return Error::fail(Error::Code::CorruptData, "could not decode SF mesh payload"); + } + return std::move(*decoded); +} + +} // namespace mesh::sf diff --git a/src/terrainlib/mesh/codec/from_extension.h b/src/terrainlib/mesh/codec/from_extension.h new file mode 100644 index 00000000..58d91153 --- /dev/null +++ b/src/terrainlib/mesh/codec/from_extension.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +#include + +#include "mesh/codec/Gltf.h" +#include "mesh/codec/SfMesh.h" +#include "store/Codec.h" + +namespace mesh::codec { + +inline Expected>> from_extension(const std::string_view extension) +{ + if (extension == ".sfmesh") { + return std::make_unique(); + } + if (extension == ".glb") { + return std::make_unique(GltfContainer::Binary); + } + if (extension == ".gltf") { + return std::make_unique(GltfContainer::Json); + } + return Error::fail(Error::Code::Unsupported, "unsupported mesh codec selector: " + std::string(extension)); +} + +} // namespace mesh::codec diff --git a/src/terrainlib/mesh/io.cpp b/src/terrainlib/mesh/io.cpp index 467e0537..a80cc30f 100644 --- a/src/terrainlib/mesh/io.cpp +++ b/src/terrainlib/mesh/io.cpp @@ -1,25 +1,28 @@ #include "mesh/io.h" #include "log.h" #include "mesh/io/gltf.h" -#include "mesh/io/terrain.h" +#include "mesh/codec/SfMesh.h" #include "mesh/validate.h" namespace mesh::io { -std::expected load_from_path( +Expected load_from_path( const std::filesystem::path &path, const LoadOptions& options) { const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { return gltf::load_from_path(path, options); - } else if (extension == ".terrain") { - return terrain::load_from_path(path, options); + } else if (extension == ".sfmesh") { + std::filesystem::path node_path = path; + node_path.replace_extension(); + const mesh::codec::SfMesh codec; + return codec.read(node_path); } else { - return std::unexpected(LoadMeshErrorKind::UnsupportedFormat); + return Error::fail(Error::Code::Unsupported, "unsupported mesh input format: " + extension.string()); } } -std::expected save_to_path( +Expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options) { @@ -30,10 +33,13 @@ std::expected save_to_path( const std::filesystem::path extension = path.extension(); if (extension == ".glb" || extension == ".gltf") { return gltf::save_to_path(mesh, path, options); - } else if (extension == ".terrain") { - return terrain::save_to_path(mesh, path, options); + } else if (extension == ".sfmesh") { + std::filesystem::path node_path = path; + node_path.replace_extension(); + const mesh::codec::SfMesh codec; + return codec.write(node_path, mesh, mesh::EncodeOptions{.texture_format = options.texture_format}); } else { - return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); + return Error::fail(Error::Code::Unsupported, "unsupported mesh output format: " + extension.string()); } } diff --git a/src/terrainlib/mesh/io.h b/src/terrainlib/mesh/io.h index ea474c73..d05242ec 100644 --- a/src/terrainlib/mesh/io.h +++ b/src/terrainlib/mesh/io.h @@ -2,19 +2,17 @@ #include -#include - +#include "Error.h" #include "mesh/SimpleMesh.h" #include "mesh/io/options.h" -#include "mesh/io/error.h" namespace mesh::io { -std::expected load_from_path( +Expected load_from_path( const std::filesystem::path &path, const LoadOptions& options = {}); -std::expected save_to_path( +Expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options = {}); diff --git a/src/terrainlib/mesh/io/error.h b/src/terrainlib/mesh/io/error.h deleted file mode 100644 index 40df1267..00000000 --- a/src/terrainlib/mesh/io/error.h +++ /dev/null @@ -1,142 +0,0 @@ -#pragma once - -#include - -namespace mesh::io { - -enum class LoadMeshErrorKind { - UnsupportedFormat, - FileNotFound, - InvalidFormat, - OutOfMemory -}; - -class LoadMeshError { -public: - LoadMeshError() = default; - constexpr LoadMeshError(LoadMeshErrorKind kind) - : kind(kind) {} - - operator LoadMeshErrorKind() const { - return this->kind; - } - constexpr bool operator==(LoadMeshError other) const { - return this->kind == other.kind; - } - constexpr bool operator!=(LoadMeshError other) const { - return this->kind != other.kind; - } - constexpr bool operator==(LoadMeshErrorKind other) const { - return this->kind == other; - } - constexpr bool operator!=(LoadMeshErrorKind other) const { - return this->kind != other; - } - - const char* description() const { - switch (kind) { - case LoadMeshErrorKind::UnsupportedFormat: - return "format not supported"; - case LoadMeshErrorKind::FileNotFound: - return "file not found"; - case LoadMeshErrorKind::InvalidFormat: - return "invalid file format"; - case LoadMeshErrorKind::OutOfMemory: - return "out of memory"; - default: - return "undefined error"; - } - } - -private: - LoadMeshErrorKind kind; -}; - -enum class SaveMeshErrorKind { - UnsupportedFormat, - OpenFile, - WriteFile, - OutOfMemory -}; - -class SaveMeshError { -public: - SaveMeshError() = default; - constexpr SaveMeshError(SaveMeshErrorKind kind) - : kind(kind) {} - - operator SaveMeshErrorKind() const { - return this->kind; - } - constexpr bool operator==(SaveMeshError other) const { - return this->kind == other.kind; - } - constexpr bool operator!=(SaveMeshError other) const { - return this->kind != other.kind; - } - constexpr bool operator==(SaveMeshErrorKind other) const { - return this->kind == other; - } - constexpr bool operator!=(SaveMeshErrorKind other) const { - return this->kind != other; - } - - const char* description() const { - switch (kind) { - case SaveMeshErrorKind::UnsupportedFormat: - return "format not supported"; - case SaveMeshErrorKind::OpenFile: - return "failed to open output file"; - case SaveMeshErrorKind::WriteFile: - return "failed to write to output file"; - case SaveMeshErrorKind::OutOfMemory: - return "out of memory"; - default: - return "undefined error"; - } - } - -private: - SaveMeshErrorKind kind; -}; - -} // namespace mesh::io - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const mesh::io::SaveMeshError &error, FormatContext &ctx) const { - return fmt::format_to(ctx.out(), "{}", error.description()); - } -}; - -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const mesh::io::LoadMeshError &error, FormatContext &ctx) const { - return fmt::format_to(ctx.out(), "{}", error.description()); - } -}; - -#include -#include -inline std::ostream &operator<<(std::ostream &os, const mesh::io::SaveMeshError &error) { - fmt::print(os, "{}", error); - return os; -} - -inline std::ostream &operator<<(std::ostream &os, const mesh::io::LoadMeshError &error) { - fmt::print(os, "{}", error); - return os; -} diff --git a/src/terrainlib/mesh/io/gltf.cpp b/src/terrainlib/mesh/io/gltf.cpp index 318716a6..25ed2c84 100644 --- a/src/terrainlib/mesh/io/gltf.cpp +++ b/src/terrainlib/mesh/io/gltf.cpp @@ -1,7 +1,11 @@ +#include +#include #include #include #include -#include +#include +#include +#include #define CGLTF_IMPLEMENTATION #define CGLTF_WRITE_IMPLEMENTATION @@ -10,13 +14,11 @@ #undef CGLTF_IMPLEMENTATION #undef CGLTF_WRITE_IMPLEMENTATION +#include "io/utils.h" #include "log.h" -#include "mesh/io/utils.h" #include "mesh/io/gltf.h" #include "mesh/io/texture.h" -using namespace mesh::io::utils; - namespace mesh::io::gltf { namespace { @@ -178,7 +180,7 @@ std::string data_uri_encode(unsigned char const *bytes_to_encode, unsigned int i return "data:application/octet-stream;base64," + base64_encode(bytes_to_encode, in_len); } -std::optional load_texture_from_material(const cgltf_material &material) { +Expected> load_texture_from_material(const cgltf_material &material) { if (!material.has_pbr_metallic_roughness || material.pbr_metallic_roughness.base_color_texture.texture == nullptr) { LOG_WARN("mesh material has no texture"); return std::nullopt; @@ -193,52 +195,184 @@ std::optional load_texture_from_material(const cgltf_material &material } const std::span raw_texture{cgltf_buffer_view_data(albedo_image.buffer_view), albedo_image.buffer_view->size}; - cv::Mat texture = mesh::io::read_texture_from_encoded_bytes(raw_texture); - return texture; -} - -#define GET_OR_INVALID_FORMAT(var, opt) \ - do { \ - if (!(opt).has_value()) { \ - return std::unexpected(LoadMeshErrorKind::InvalidFormat); \ - } else { \ - var = opt.value(); \ - } \ - } while (false) - -template -static std::optional> get_single_element(const char *name, cgltf_size count, T const *items) { - if (count == 0) { - LOG_ERROR("file contains no {}", name); - return std::nullopt; - } - if (count > 1) { - LOG_WARN("file contains more than one {}", name); - return std::nullopt; + try { + cv::Mat texture = mesh::io::read_texture_from_encoded_bytes(raw_texture); + if (texture.empty()) { + return Error::fail(Error::Code::CorruptData, "could not decode embedded glTF texture"); + } + return std::optional { std::move(texture) }; + } catch (const cv::Exception& error) { + if (error.code == cv::Error::StsNoMem) { + return Error::fail(Error::Code::ResourceExhausted, "decode embedded glTF texture: out of memory"); + } + return Error::fail(Error::Code::CorruptData, "could not decode embedded glTF texture: " + error.msg); } - - const T &ref = items[0]; - - return ref; } -LoadMeshError map_cgltf_error(cgltf_result result) { +Error load_cgltf_error(const cgltf_result result, const std::filesystem::path& path) { switch (result) { case cgltf_result::cgltf_result_data_too_short: case cgltf_result::cgltf_result_unknown_format: case cgltf_result::cgltf_result_invalid_json: case cgltf_result::cgltf_result_invalid_gltf: + return Error::make(Error::Code::CorruptData, "decode invalid glTF file", path); case cgltf_result::cgltf_result_legacy_gltf: - return LoadMeshErrorKind::InvalidFormat; + return Error::make(Error::Code::Unsupported, "decode legacy glTF file", path); case cgltf_result::cgltf_result_file_not_found: + return Error::make(Error::Code::NotFound, "open glTF file", path); case cgltf_result::cgltf_result_io_error: - return LoadMeshErrorKind::FileNotFound; + return Error::make(Error::Code::Io, "read glTF file", path); case cgltf_result::cgltf_result_out_of_memory: - return LoadMeshErrorKind::OutOfMemory; - default: - UNREACHABLE(); - break; + return Error::make(Error::Code::ResourceExhausted, "load glTF file: out of memory", path); + case cgltf_result::cgltf_result_invalid_options: + case cgltf_result::cgltf_result_success: + case cgltf_result::cgltf_result_max_enum: + return Error::make(Error::Code::Internal, "unexpected glTF loader result for \"" + path.string() + "\""); + } + return Error::make(Error::Code::Internal, "unknown glTF loader result for \"" + path.string() + "\""); +} + +Expected> serialize_json(const cgltf_options& options, const cgltf_data& data) +try { + const cgltf_size expected_size = cgltf_write(&options, nullptr, 0, &data); + if (expected_size == 0) { + return Error::fail(Error::Code::Internal, "glTF serialization produced no output"); + } + + std::vector json; + if (expected_size > json.max_size()) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the supported size"); } + json.resize(expected_size); + const cgltf_size actual_size = cgltf_write(&options, reinterpret_cast(json.data()), json.size(), &data); + if (actual_size != expected_size) { + return Error::fail(Error::Code::Internal, "glTF serialization size changed between passes"); + } + json.resize(actual_size - 1); + return json; +} catch (const std::bad_alloc&) { + return Error::fail(Error::Code::ResourceExhausted, "allocate glTF serialization output"); +} catch (const std::length_error&) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the supported size"); +} + +bool write_all(std::ofstream& file, std::span bytes) +{ + constexpr std::size_t max_write_size = std::size_t { 1 } << 30; + while (!bytes.empty()) { + const std::size_t write_size = (std::min)(bytes.size(), max_write_size); + file.write(reinterpret_cast(bytes.data()), static_cast(write_size)); + if (!file.good()) { + return false; + } + bytes = bytes.subspan(write_size); + } + return true; +} + +Expected write_serialized( + const std::filesystem::path& path, + const std::span json, + const cgltf_data& data, + const bool binary_output) +try { + constexpr std::uint32_t glb_magic = 0x46546C67; + constexpr std::uint32_t glb_version = 2; + constexpr std::uint32_t json_chunk_magic = 0x4E4F534A; + constexpr std::uint32_t binary_chunk_magic = 0x004E4942; + constexpr std::size_t header_size = 12; + constexpr std::size_t chunk_header_size = 8; + constexpr std::size_t glb_size_limit = (std::numeric_limits::max)(); + + std::size_t json_padding = 0; + std::size_t binary_size = 0; + std::size_t binary_padding = 0; + std::size_t padded_json_size = 0; + std::size_t padded_binary_size = 0; + std::size_t total_size = json.size(); + if (binary_output) { + json_padding = (4 - json.size() % 4) % 4; + binary_size = data.bin == nullptr ? 0 : data.bin_size; + binary_padding = (4 - binary_size % 4) % 4; + if (json.size() > glb_size_limit - 3 || binary_size > glb_size_limit - 3) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the GLB size limit"); + } + padded_json_size = json.size() + json_padding; + padded_binary_size = binary_size + binary_padding; + if (padded_json_size > glb_size_limit - header_size - chunk_header_size) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the GLB size limit"); + } + total_size = header_size + chunk_header_size + padded_json_size; + if (binary_size != 0) { + if (padded_binary_size > glb_size_limit - chunk_header_size + || chunk_header_size + padded_binary_size > glb_size_limit - total_size) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the GLB size limit"); + } + total_size += chunk_header_size + padded_binary_size; + } + } + + const std::error_code directory_error = ::io::utils::create_parent_directories(path); + if (directory_error) { + return Error::fail(Error::Code::Io, "create parent directories for", path, directory_error); + } + + std::ofstream file(path, std::ios::binary); + if (!file.is_open()) { + return Error::fail(Error::Code::Io, "open glTF file for writing", path); + } + + if (!binary_output) { + if (!write_all(file, json)) { + return Error::fail(Error::Code::Io, "write glTF JSON to", path); + } + } else { + const auto put_u32 = [](const std::span bytes, const std::size_t offset, const std::uint32_t value) { + bytes[offset] = static_cast(value); + bytes[offset + 1] = static_cast(value >> 8); + bytes[offset + 2] = static_cast(value >> 16); + bytes[offset + 3] = static_cast(value >> 24); + }; + std::array header {}; + put_u32(header, 0, glb_magic); + put_u32(header, 4, glb_version); + put_u32(header, 8, static_cast(total_size)); + put_u32(header, 12, static_cast(padded_json_size)); + put_u32(header, 16, json_chunk_magic); + constexpr std::array json_padding_bytes { 0x20, 0x20, 0x20 }; + constexpr std::array binary_padding_bytes {}; + + if (!write_all(file, header) + || !write_all(file, json) + || !write_all(file, std::span(json_padding_bytes).first(json_padding))) { + return Error::fail(Error::Code::Io, "write glTF JSON chunk to", path); + } + if (binary_size != 0) { + std::array binary_header {}; + put_u32(binary_header, 0, static_cast(padded_binary_size)); + put_u32(binary_header, 4, binary_chunk_magic); + const std::span binary_bytes(static_cast(data.bin), binary_size); + if (!write_all(file, binary_header) + || !write_all(file, binary_bytes) + || !write_all(file, std::span(binary_padding_bytes).first(binary_padding))) { + return Error::fail(Error::Code::Io, "write glTF binary chunk to", path); + } + } + } + + file.flush(); + if (!file.good()) { + return Error::fail(Error::Code::Io, "flush glTF file", path); + } + file.close(); + if (file.fail()) { + return Error::fail(Error::Code::Io, "close glTF file", path); + } + return {}; +} catch (const std::bad_alloc&) { + return Error::fail(Error::Code::ResourceExhausted, "allocate glTF writer state"); +} catch (const std::length_error&) { + return Error::fail(Error::Code::ResourceExhausted, "serialized glTF exceeds the supported size"); } /// Calculates the size of the data of a vector in bytes. @@ -265,60 +399,65 @@ static std::string image_ext_to_mime(std::string_view extension) { } } -std::expected load_raw_from_path(const std::filesystem::path &path) { +Expected load_raw_from_path(const std::filesystem::path &path) { cgltf_options options = {}; cgltf_data *data = NULL; const std::string path_str = path.string(); const char *path_ptr = path_str.c_str(); cgltf_result result = cgltf_parse_file(&options, path_ptr, &data); if (result != cgltf_result::cgltf_result_success) { - return std::unexpected(result); + return Error::propagate(load_cgltf_error(result, path), "parse glTF file"); } + RawMesh raw_mesh(data, cgltf_free); - result = cgltf_load_buffers(&options, data, path_ptr); + result = cgltf_load_buffers(&options, raw_mesh.get(), path_ptr); if (result != cgltf_result::cgltf_result_success) { - cgltf_free(data); - return std::unexpected(result); + return Error::propagate(load_cgltf_error(result, path), "load glTF buffers"); } - result = cgltf_validate(data); + result = cgltf_validate(raw_mesh.get()); if (result != cgltf_result_success) { - cgltf_free(data); - return std::unexpected(result); + return Error::propagate(load_cgltf_error(result, path), "validate glTF data"); } - return RawMesh(data, cgltf_free); + return raw_mesh; } -std::expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { +Expected load_from_raw(const RawMesh &raw, const LoadOptions& /* options */) { LOG_TRACE("Loading mesh from gltf data"); const cgltf_data &data = *raw; - const auto mesh_opt = get_single_element("mesh", data.meshes_count, data.meshes); - if (!mesh_opt.has_value()) { - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + if (data.meshes_count == 0) { + return Error::fail(Error::Code::CorruptData, "glTF file contains no mesh"); + } + if (data.meshes_count > 1) { + return Error::fail(Error::Code::Unsupported, "loading glTF files with multiple meshes is not supported"); } - const cgltf_mesh &mesh = mesh_opt.value(); + const cgltf_mesh &mesh = data.meshes[0]; - const auto mesh_primitive_opt = get_single_element("mesh primitive", mesh.primitives_count, mesh.primitives); - if (!mesh_primitive_opt.has_value()) { - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + if (mesh.primitives_count == 0) { + return Error::fail(Error::Code::CorruptData, "glTF mesh contains no primitive"); + } + if (mesh.primitives_count > 1) { + return Error::fail(Error::Code::Unsupported, "loading glTF meshes with multiple primitives is not supported"); } - const cgltf_primitive &mesh_primitive = mesh_primitive_opt.value(); + const cgltf_primitive &mesh_primitive = mesh.primitives[0]; if (mesh_primitive.type != cgltf_primitive_type::cgltf_primitive_type_triangles) { LOG_ERROR("mesh has invalid primitive type"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::Unsupported, "loading non-triangle glTF primitives is not supported"); } // indices if (mesh_primitive.indices == nullptr) { LOG_ERROR("mesh primitive has no indices"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::Unsupported, "loading non-indexed glTF primitives is not supported"); } cgltf_accessor &index_accessor = *mesh_primitive.indices; std::vector indices; - DEBUG_ASSERT(index_accessor.count % 3 == 0); + if (index_accessor.count % 3 != 0) { + return Error::fail(Error::Code::CorruptData, "glTF mesh index count is not divisible by three"); + } indices.resize(index_accessor.count / 3); cgltf_accessor_unpack_indices( &index_accessor, @@ -330,13 +469,13 @@ std::expected load_from_raw(const RawMesh &raw, const cgltf_attribute *position_attr = find_attribute_with_type(mesh_primitive.attributes, mesh_primitive.attributes_count, cgltf_attribute_type_position); if (position_attr == nullptr) { LOG_ERROR("mesh has no position attribute"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::CorruptData, "glTF mesh contains no position attribute"); } cgltf_accessor &position_accessor = *position_attr->data; if (position_accessor.type != cgltf_type_vec3) { LOG_WARN("mesh positions are not vec3"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::CorruptData, "glTF mesh positions are not three-component vectors"); } std::vector positions; positions.resize(position_accessor.count); @@ -351,7 +490,7 @@ std::expected load_from_raw(const RawMesh &raw, const cgltf_accessor &uv_accessor = *uv_attr->data; if (uv_accessor.type != cgltf_type_vec2) { LOG_WARN("mesh uvss are not vec2"); - return std::unexpected(LoadMeshErrorKind::InvalidFormat); + return Error::fail(Error::Code::CorruptData, "glTF mesh UVs are not two-component vectors"); } uvs.resize(uv_accessor.count); cgltf_accessor_unpack_floats(&uv_accessor, reinterpret_cast(uvs.data()), uvs.size() * 2); @@ -374,14 +513,18 @@ std::expected load_from_raw(const RawMesh &raw, const std::optional texture; if (mesh_primitive.material != nullptr) { - texture = load_texture_from_material(*mesh_primitive.material); + auto loaded_texture = load_texture_from_material(*mesh_primitive.material); + if (!loaded_texture) { + return Error::propagate(std::move(loaded_texture), "load glTF material texture"); + } + texture = std::move(*loaded_texture); } return SimpleMesh(indices, positionsd, uvsd, texture); } /// Saves the mesh as a .gltf or .glb file at the given path. -std::expected save_to_path( +Expected save_to_path( const SimpleMesh &terrain_mesh, const std::filesystem::path &path, const SaveOptions& options) { @@ -423,7 +566,23 @@ std::expected save_to_path( const bool has_texture = terrain_mesh.has_texture(); std::vector texture_bytes; if (has_texture) { - texture_bytes = mesh::io::write_texture_to_encoded_buffer(terrain_mesh.texture.value(), options.texture_format); + try { + if (!cv::haveImageWriter(options.texture_format)) { + return Error::fail(Error::Code::Unsupported, "unsupported glTF texture format: " + options.texture_format); + } + texture_bytes = mesh::io::write_texture_to_encoded_buffer(terrain_mesh.texture.value(), options.texture_format); + } catch (const cv::Exception& error) { + if (error.code == cv::Error::StsNoMem) { + return Error::fail(Error::Code::ResourceExhausted, "encode glTF texture: out of memory"); + } + if (error.code == cv::Error::StsUnsupportedFormat || error.code == cv::Error::StsNotImplemented) { + return Error::fail(Error::Code::Unsupported, "unsupported glTF texture encoding: " + error.msg); + } + return Error::fail(Error::Code::InvalidInput, "could not encode glTF texture: " + error.msg); + } + if (!terrain_mesh.texture->empty() && texture_bytes.empty()) { + return Error::fail(Error::Code::InvalidInput, "could not encode glTF texture"); + } } // Create a single buffer that holds all binary data (indices, vertices, textures) @@ -707,24 +866,32 @@ std::expected save_to_path( } // ********************* Save the GLTF data to a file ********************* // - create_parent_directories(path); cgltf_options gltf_options = {}; if (binary_output) { gltf_options.type = cgltf_file_type_glb; } - if (cgltf_write_file(&gltf_options, path.string().c_str(), &data) != cgltf_result_success) { - throw std::runtime_error("Failed to save GLTF file"); + auto serialized = serialize_json(gltf_options, data); + if (!serialized) { + return Error::propagate(std::move(serialized), "serialize glTF output"); + } + auto written = write_serialized(path, *serialized, data, binary_output); + if (!written) { + return Error::propagate(std::move(written), "write glTF output"); } return {}; } -std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { - std::expected raw_mesh = load_raw_from_path(path); +Expected load_from_path(const std::filesystem::path &path, const LoadOptions &options) { + auto raw_mesh = load_raw_from_path(path); if (!raw_mesh) { - return std::unexpected(map_cgltf_error(raw_mesh.error())); + return Error::propagate(std::move(raw_mesh), "load raw glTF data"); + } + auto mesh = load_from_raw(*raw_mesh, options); + if (!mesh) { + return Error::propagate(std::move(mesh), "decode glTF mesh \"" + path.string() + "\""); } - return load_from_raw(*raw_mesh, options); + return std::move(*mesh); } } // namespace mesh::io::gltf diff --git a/src/terrainlib/mesh/io/gltf.h b/src/terrainlib/mesh/io/gltf.h index 20333e93..02b5bf20 100644 --- a/src/terrainlib/mesh/io/gltf.h +++ b/src/terrainlib/mesh/io/gltf.h @@ -1,33 +1,31 @@ #pragma once #include -#include #include +#include #include -#include +#include "Error.h" #include "mesh/SimpleMesh.h" -#include "mesh/io/error.h" #include "mesh/io/options.h" namespace mesh::io::gltf { using RawMesh = std::unique_ptr; -std::expected load_from_path( +Expected load_from_path( const std::filesystem::path &path, const LoadOptions &options = {}); -std::expected load_from_raw( +Expected load_from_raw( const RawMesh &mesh, const LoadOptions &options = {}); -std::expected save_to_path( +Expected save_to_path( const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -// std::expected save_to_raw(const RawMesh &mesh, const SaveOptions &options = {}); -std::expected load_raw_from_path(const std::filesystem::path &path); +Expected load_raw_from_path(const std::filesystem::path &path); } // namespace mesh::io::gltf diff --git a/src/terrainlib/mesh/io/options.h b/src/terrainlib/mesh/io/options.h index e7c2ec27..332ade5e 100644 --- a/src/terrainlib/mesh/io/options.h +++ b/src/terrainlib/mesh/io/options.h @@ -11,7 +11,7 @@ struct LoadOptions { struct SaveOptions { std::string texture_format = ".jpeg"; - std::string name = "Terrain"; + std::string name = "Octree"; std::unordered_map metadata = {}; }; diff --git a/src/terrainlib/mesh/io/terrain.cpp b/src/terrainlib/mesh/io/terrain.cpp deleted file mode 100644 index d443d6f8..00000000 --- a/src/terrainlib/mesh/io/terrain.cpp +++ /dev/null @@ -1,172 +0,0 @@ -#include - -#include - -#include "io/bytes.h" -#include "log.h" -#include "mesh/io/terrain.h" -#include "mesh/EncodedMesh.h" -#include "mesh/encode.h" - -namespace mesh::io::terrain { - -namespace { -LoadMeshError load_error_from_io_error(::io::Error error) { - switch (error) { - case ::io::Error::Value::OpenFile: - return LoadMeshErrorKind::FileNotFound; - case ::io::Error::Value::DetermineSize: - case ::io::Error::Value::ReadBytes: - return LoadMeshErrorKind::InvalidFormat; - default: - return LoadMeshErrorKind::InvalidFormat; - } -} - -SaveMeshError save_error_from_io_error(::io::Error error) { - switch (error) { - case ::io::Error::Value::OpenFile: - return SaveMeshErrorKind::OpenFile; - case ::io::Error::Value::WriteBytes: - return SaveMeshErrorKind::WriteFile; - default: - return SaveMeshErrorKind::WriteFile; - } -} - -std::expected write_bytes_to_path( - const std::span bytes, const std::filesystem::path &path) { - const auto result = ::io::write_bytes_to_path(bytes, path); - if (!result.has_value()) { - return std::unexpected(save_error_from_io_error(result.error())); - } - return {}; -} - -std::expected, LoadMeshError> read_bytes_from_path(const std::filesystem::path &path) { - const auto result = ::io::read_bytes_from_path(path); - if (!result.has_value()) { - return std::unexpected(load_error_from_io_error(result.error())); - } - return result.value(); -} - -std::expected, SaveMeshError> save_encoded_to_buffer(const mesh::Encoded &mesh) { - LOG_TRACE("Serializing mesh to buffer"); - - // TODO: this ignores the texture format in SaveOptions - std::vector data; - zpp::bits::out out(data); - auto result = out(mesh); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("Error while writing mesh: {}", error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: - case std::errc::message_size: - case std::errc::result_out_of_range: - return std::unexpected(SaveMeshErrorKind::OutOfMemory); - break; - default: - UNREACHABLE(); - break; - } - } - - return data; -} - -std::expected load_encoded_from_buffer(const std::span bytes) { - LOG_TRACE("Deserializing mesh from buffer"); - - zpp::bits::in in(bytes); - mesh::Encoded mesh; - auto result = in(mesh); - if (zpp::bits::failure(result)) { - std::error_code error_code = std::make_error_code(result); - LOG_ERROR("error while reading mesh: {}", error_code.message()); - - switch (result) { - case std::errc::no_buffer_space: - case std::errc::message_size: - return std::unexpected(LoadMeshErrorKind::OutOfMemory); - case std::errc::value_too_large: - case std::errc::bad_message: - case std::errc::protocol_error: - case std::errc::result_out_of_range: - return std::unexpected(LoadMeshErrorKind::InvalidFormat); - case std::errc::not_supported: - case std::errc::invalid_argument: - UNREACHABLE(); - break; - default: - throw std::runtime_error("unexpected error"); - } - } - - return mesh; -} -} - -std::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions& options) { - const auto encode_result = mesh::encode(mesh, mesh::EncodeOptions{ - .texture_format = options.texture_format}); - if (!encode_result.has_value()) { - return std::unexpected(SaveMeshErrorKind::UnsupportedFormat); - } - const mesh::Encoded encoded = encode_result.value(); - - const auto deser_result = save_encoded_to_buffer(encoded); - if (!deser_result.has_value()) { - return std::unexpected(deser_result.error()); - } - const std::vector buffer = deser_result.value(); - - return buffer; -} - -std::expected load_from_buffer(const std::span bytes, const LoadOptions & /* options */) { - const auto deser_result = load_encoded_from_buffer(bytes); - if (!deser_result.has_value()) { - return std::unexpected(deser_result.error()); - } - const mesh::Encoded encoded = deser_result.value(); - - const auto decode_result = mesh::decode(encoded); - if (!decode_result.has_value()) { - return std::unexpected(LoadMeshErrorKind::InvalidFormat); - } - const mesh::Simple mesh = decode_result.value(); - - return mesh; -} - -std::expected load_from_path(const std::filesystem::path &path, const LoadOptions & /* options */) { - const auto bytes_result = read_bytes_from_path(path); - if (!bytes_result.has_value()) { - return std::unexpected(bytes_result.error()); - } - const std::vector bytes = bytes_result.value(); - - return load_from_buffer(bytes); -} - -std::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions& options) { - LOG_TRACE("Saving mesh as high precision terrain"); - - const auto result = save_to_buffer(mesh, options); - if (!result.has_value()) { - return std::unexpected(result.error()); - } - const std::vector bytes = result.value(); - - const auto write_result = write_bytes_to_path(bytes, path); - if (!write_result.has_value()) { - return std::unexpected(write_result.error()); - } - - return {}; -} - -} // namespace mesh::io::terrain diff --git a/src/terrainlib/mesh/io/terrain.h b/src/terrainlib/mesh/io/terrain.h deleted file mode 100644 index fd097a24..00000000 --- a/src/terrainlib/mesh/io/terrain.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "mesh/SimpleMesh.h" -#include "mesh/io/error.h" -#include "mesh/io/options.h" - -namespace mesh::io::terrain { - -std::expected load_from_path(const std::filesystem::path &path, const LoadOptions &options = {}); -std::expected load_from_buffer(const std::span buffer, const LoadOptions &options = {}); - -std::expected save_to_path(const SimpleMesh &mesh, const std::filesystem::path &path, const SaveOptions &options = {}); -std::expected, SaveMeshError> save_to_buffer(const SimpleMesh &mesh, const SaveOptions &options = {}); - -} // namespace mesh::io::terrain diff --git a/src/terrainlib/mesh/io/texture.h b/src/terrainlib/mesh/io/texture.h index c6849014..e110878e 100644 --- a/src/terrainlib/mesh/io/texture.h +++ b/src/terrainlib/mesh/io/texture.h @@ -3,7 +3,6 @@ #include #include #include -#include #include @@ -21,33 +20,3 @@ std::vector write_texture_to_encoded_buffer(const cv::Mat &image, const std::vector write_texture_to_encoded_buffer(const ImageAndExt &item); } - -#include - -namespace mesh::io { - -template -auto serialize(Archive &archive, const mesh::io::ImageAndExt &item) { - const std::vector encoded = mesh::io::write_texture_to_encoded_buffer(item.image, item.ext); - return archive(item.ext, encoded); -} - -template -auto serialize(Archive &archive, mesh::io::ImageAndExt &item) { - std::vector encoded; - - auto result = archive(item.ext, encoded); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - item.image = mesh::io::read_texture_from_encoded_bytes(encoded); - return result; - } -} - -} // namespace mesh::io diff --git a/src/terrainlib/mesh/storage.h b/src/terrainlib/mesh/storage.h new file mode 100644 index 00000000..8a84830d --- /dev/null +++ b/src/terrainlib/mesh/storage.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +#include "mesh/SimpleMesh.h" +#include "mesh/codec/from_extension.h" +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "octree/storage/open_runtime.h" +#include "store/IndexedStorage.h" + +namespace mesh::storage { + +inline constexpr std::string_view payload_class = "mesh.Simple3d"; + +using Storage = store::Storage; +using IndexedStorage = store::IndexedStorage; +using OpenOptions = octree::storage::OpenOptions; + +inline Expected open_index(const std::filesystem::path& path) +{ + return store::open_index(path, octree::storage::index_format(), payload_class, mesh::codec::from_extension); +} + +inline Expected open_folder(const std::filesystem::path& path, OpenOptions options = {}) +{ + return octree::storage::open_folder(path, std::string(payload_class), ".sfmesh", mesh::codec::from_extension, std::move(options)); +} + +inline Expected open_folder_indexed(const std::filesystem::path& path, OpenOptions options = {}) +{ + return octree::storage::open_folder_indexed(path, std::string(payload_class), ".sfmesh", mesh::codec::from_extension, std::move(options)); +} + +} // namespace mesh::storage diff --git a/src/terrainlib/octree/Id.h b/src/terrainlib/octree/Id.h index 44518c71..f1619cb3 100644 --- a/src/terrainlib/octree/Id.h +++ b/src/terrainlib/octree/Id.h @@ -98,7 +98,7 @@ class Id_ { return Coords(max_coord_on_level(level)); } - constexpr Id_() = default; // zpp::bits requires a default constructor + constexpr Id_() = default; template requires(sizeof...(Coordinates) == Dimensions && (std::convertible_to && ...)) constexpr Id_(const Level level, Coordinates... coordinates) : Id_(level, Coords{coordinates...}) {} @@ -399,51 +399,3 @@ struct hash> { } }; } // namespace std - -#include -namespace octree { -namespace { -constexpr zpp::bits::errc success() { - return zpp::bits::errc(std::errc()); -} -} // namespace - -template < - size_t Dimensions, - size_t MaxLevel, - typename LevelT, - typename IndexT, - typename CoordT> -constexpr auto serialize(auto &archive, octree::Id_ &id) { - using Id = octree::Id_; - typename Id::Level level; - typename Id::Index index; - auto result = archive(level, index); - using Result = std::remove_cvref_t; - if constexpr (!std::is_same_v && !std::is_same_v) { - return result; - } else { - if (zpp::bits::failure(result)) { - return result; - } - - auto maybe_id = Id::try_make(level, index); - if (!maybe_id) { - return zpp::bits::errc(std::errc::bad_message); - } - - id = *maybe_id; - return success(); - } -} - -template < - size_t Dimensions, - size_t MaxLevel, - typename LevelT, - typename IndexT, - typename CoordT> -constexpr auto serialize(auto &archive, const octree::Id_ &id) { - return archive(id.level(), id.index_on_level()); -} -} // namespace octree diff --git a/src/terrainlib/octree/IndexMap.cpp b/src/terrainlib/octree/IndexMap.cpp deleted file mode 100644 index 8bb7e53c..00000000 --- a/src/terrainlib/octree/IndexMap.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include - -#include "octree/IndexMap.h" -#include "log.h" - -namespace octree { - -std::optional IndexMap::get(Id id) const { - const NodeStatus* status = this->get_raw(id); - if (status != nullptr) { - return *status; - } else { - return std::nullopt; - } -} - -bool IndexMap::add(Id id) { - NodeStatus* status = this->get_raw(id); - if (status != nullptr) { - switch (*status) { - case NodeStatus::Inner: - case NodeStatus::Leaf: - // Nothing to do - return false; - case NodeStatus::Virtual: - this->set_raw(id, NodeStatus::Inner); - return true; - } - } - - const auto parent_opt = id.parent(); - if (!parent_opt.has_value()) { - DEBUG_ASSERT(id.is_root()); - if (this->empty()) { - this->set_raw(id, NodeStatus::Leaf); - return true; - } else { - UNREACHABLE(); - return true; - } - } - - const auto parent = parent_opt.value(); - const NodeStatus* parent_status = this->get_raw(parent); - if (parent_status == nullptr) { - this->add(parent); - this->set_raw(parent, NodeStatus::Virtual); - this->set_raw(id, NodeStatus::Leaf); - return true; - } - - switch (*parent_status) { - case NodeStatus::Leaf: - this->set_raw(parent, NodeStatus::Inner); - this->set_raw(id, NodeStatus::Leaf); - break; - case NodeStatus::Inner: - case NodeStatus::Virtual: - this->set_raw(id, NodeStatus::Leaf); - break; - } - - return true; -} - -bool IndexMap::remove(Id id) { - NodeStatus* status = this->get_raw(id); - if (status == nullptr) { - return false; - } - - switch (*status) { - case NodeStatus::Inner: - *status = NodeStatus::Virtual; - return true; - case NodeStatus::Virtual: - // Nothing to do - return false; - case NodeStatus::Leaf: - this->remove_raw(id); - this->update_parent_after_remove(id); - return true; - } - - UNREACHABLE(); -} - -bool IndexMap::is_present(Id id) const { - auto it = this->_index.find(id); - return it != this->_index.end(); -} -bool IndexMap::is_absent(Id id) const { - return !this->is_present(id); -} -bool IndexMap::is(NodeStatus status, Id id) const { - return this->get(id) == status; -} - -void IndexMap::clear() { - this->_index.clear(); -} -bool IndexMap::empty() const { - return this->_index.empty(); -} -size_t IndexMap::size() const { - return this->_index.size(); -} - -IndexMap::const_iterator IndexMap::begin() const { - return this->_index.begin(); -} -IndexMap::const_iterator IndexMap::end() const { - return this->_index.end(); -} -IndexMap::const_iterator IndexMap::cbegin() const { - return this->_index.cbegin(); -} -IndexMap::const_iterator IndexMap::cend() const { - return this->_index.cend(); -} - -NodeStatus* IndexMap::get_raw(Id id) { - if (auto it = this->_index.find(id); it != this->_index.end()) { - return &it->second; - } - return nullptr; -} -const NodeStatus* IndexMap::get_raw(Id id) const { - if (auto it = this->_index.find(id); it != this->_index.end()) { - return &it->second; - } - return nullptr; -} -void IndexMap::set_raw(Id id, NodeStatus status) { - this->_index[id] = status; -} -void IndexMap::remove_raw(Id id) { - this->_index.erase(id); -} - -void IndexMap::update_parent_after_remove(Id id) { - const auto parent_opt = id.parent(); - if (!parent_opt.has_value()) { - return; - } - - const auto parent = parent_opt.value(); - NodeStatus* parent_status = this->get_raw(parent); - DEBUG_ASSERT(parent_status != nullptr); - - // We know there are children since we just removed one - DEBUG_ASSERT(parent.has_children()); - const auto siblings = parent.children().value(); - for (const auto& sibling : siblings) { - if (this->is_present(sibling) && sibling != id) { - return; - } - } - - switch (*parent_status) { - case NodeStatus::Inner: - *parent_status = NodeStatus::Leaf; - break; - case NodeStatus::Virtual: - this->remove_raw(parent); - this->update_parent_after_remove(parent); - break; - case NodeStatus::Leaf: - UNREACHABLE(); - break; - } -} - -} diff --git a/src/terrainlib/octree/IndexMap.h b/src/terrainlib/octree/IndexMap.h deleted file mode 100644 index c5dbf535..00000000 --- a/src/terrainlib/octree/IndexMap.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/NodeStatus.h" - -namespace octree { - -class IndexMap { -public: - using Container = std::unordered_map; - using iterator = Container::iterator; - using const_iterator = Container::const_iterator; - - std::optional get(Id id) const; - bool add(Id id); - bool remove(Id id); - - bool is_present(Id id) const; - bool is_absent(Id id) const; - bool is(NodeStatus status, Id id) const; - - void clear(); - bool empty() const; - size_t size() const; - - const_iterator begin() const; - const_iterator end() const; - const_iterator cbegin() const; - const_iterator cend() const; - - NodeStatus* get_raw(Id id); - const NodeStatus* get_raw(Id id) const; - void set_raw(Id id, NodeStatus status); - void remove_raw(Id id); - - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; - -private: - Container _index; - void update_parent_after_remove(Id id); -}; - -} // namespace octree diff --git a/src/terrainlib/octree/NodeStatus.h b/src/terrainlib/octree/NodeStatus.h deleted file mode 100644 index 9e00d97c..00000000 --- a/src/terrainlib/octree/NodeStatus.h +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once - -#include - -#include "log.h" - -namespace octree { - -class NodeStatus { -public: - using Underlying = uint8_t; - enum Value : Underlying { - Leaf = 0, - Inner = 1, - Virtual = 2 // not present on disk but has children - }; - - constexpr NodeStatus() = default; - constexpr NodeStatus(Value value) : _value(value) {} - - constexpr operator Value() const { - return this->_value; - } - explicit operator bool() const = delete; - constexpr bool operator==(NodeStatus other) const { - return this->_value == other._value; - } - constexpr bool operator!=(NodeStatus other) const { - return !(*this == other); - } - constexpr bool operator==(Value other_value) const { - return this->_value == other_value; - } - constexpr bool operator!=(Value other_value) const { - return this->_value != other_value; - } - - std::string to_string() const; - -private: - Value _value; - -public: - using serialize = zpp::bits::members<1>; - friend zpp::bits::access; -}; - -} - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const octree::NodeStatus &status, FormatContext &ctx) const { - switch (status) { - case octree::NodeStatus::Leaf: - return fmt::format_to(ctx.out(), "Leaf"); - case octree::NodeStatus::Inner: - return fmt::format_to(ctx.out(), "Inner"); - case octree::NodeStatus::Virtual: - return fmt::format_to(ctx.out(), "Virtual"); - default: - UNREACHABLE(); - } - } -}; - -#include -#include -inline std::string octree::NodeStatus::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const octree::NodeStatus &status) { - fmt::print(os, "{}", status); - return os; -} diff --git a/src/terrainlib/octree/NodeStatusOrMissing.h b/src/terrainlib/octree/NodeStatusOrMissing.h deleted file mode 100644 index 0868cb93..00000000 --- a/src/terrainlib/octree/NodeStatusOrMissing.h +++ /dev/null @@ -1,138 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/NodeStatus.h" - -namespace octree { - -class NodeStatusOrMissing { -public: - using Underlying = NodeStatus::Underlying; - enum Value : Underlying { - Leaf = 0, - Inner = 1, - Virtual = 2, // not present on disk but has children - Missing = 3 - }; - - constexpr NodeStatusOrMissing() = default; - constexpr NodeStatusOrMissing(Value value) : _value(value) {} - NodeStatusOrMissing(NodeStatus opt) : _value(from_status(opt)) {} - NodeStatusOrMissing(std::optional opt) : _value(from_optional_status(opt)) {} - - constexpr operator Value() const { - return this->_value; - } - constexpr operator std::optional() const { - return this->to_optional_status(); - } - explicit operator bool() const = delete; - constexpr bool operator==(NodeStatusOrMissing other) const { - return this->_value == other._value; - } - constexpr bool operator!=(NodeStatusOrMissing other) const { - return !(*this == other); - } - constexpr bool operator==(Value other_value) const { - return this->_value == other_value; - } - constexpr bool operator!=(Value other_value) const { - return this->_value != other_value; - } - - std::string to_string() const; - -private: - constexpr static NodeStatusOrMissing from_status(NodeStatus status) noexcept { - return static_cast(static_cast(status)); - } - constexpr static NodeStatusOrMissing from_optional_status(std::optional opt) noexcept { - if (opt.has_value()) { - return from_status(opt.value()); - } else { - return NodeStatusOrMissing::Missing; - } - } - constexpr std::optional to_optional_status() const noexcept { - if (this->_value == NodeStatusOrMissing::Missing) { - return std::nullopt; - } else { - return static_cast(this->_value); - } - } - -public: - Value _value; -}; - -} - -#include -template <> -struct fmt::formatter { - template - constexpr auto parse(ParseContext &ctx) { - return ctx.begin(); - } - - template - auto format(const octree::NodeStatusOrMissing &status, FormatContext &ctx) const { - switch (status) { - case octree::NodeStatusOrMissing::Leaf: - return fmt::format_to(ctx.out(), "Leaf"); - case octree::NodeStatusOrMissing::Inner: - return fmt::format_to(ctx.out(), "Inner"); - case octree::NodeStatusOrMissing::Virtual: - return fmt::format_to(ctx.out(), "Virtual"); - case octree::NodeStatusOrMissing::Missing: - return fmt::format_to(ctx.out(), "Missing"); - default: - UNREACHABLE(); - } - } -}; - -#include -#include -inline std::string octree::NodeStatusOrMissing::to_string() const { - return fmt::format("{}", *this); -} -inline std::ostream &operator<<(std::ostream &os, const octree::NodeStatusOrMissing &status) { - fmt::print(os, "{}", status); - return os; -} - -namespace octree { -namespace { -template -consteval bool _verify_enum_values(std::index_sequence) { - constexpr auto ns_values = magic_enum::enum_values(); - constexpr auto nsom_values = magic_enum::enum_values(); - - return ((magic_enum::enum_underlying(ns_values[Is]) == - magic_enum::enum_underlying(nsom_values[Is])) && - ...); -} - -consteval bool _verify_enum() { - static_assert( - std::is_same_v, - magic_enum::underlying_type_t>, - "Mismatched underlying type"); - - constexpr auto ns_values = magic_enum::enum_values(); - constexpr auto nsom_values = magic_enum::enum_values(); - - static_assert(nsom_values.size() == ns_values.size() + 1, "Mismatched enum counts"); - - static_assert(_verify_enum_values(std::make_index_sequence()), - "Enum value mismatch"); - - return true; -} -static_assert(_verify_enum()); -} -} diff --git a/src/terrainlib/octree/Storage.h b/src/terrainlib/octree/Storage.h deleted file mode 100644 index e6776555..00000000 --- a/src/terrainlib/octree/Storage.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/MeshStorage.h" -#include "octree/storage/RawStorage.h" -#include "octree/storage/Storage.h" -#include "octree/storage/open.h" - -namespace octree { - -using Storage = MeshStorage; -using IndexedStorage = IndexedMeshStorage; - -} diff --git a/src/terrainlib/octree/StoreTraits.h b/src/terrainlib/octree/StoreTraits.h new file mode 100644 index 00000000..67334210 --- /dev/null +++ b/src/terrainlib/octree/StoreTraits.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +#include "octree/Id.h" + +namespace octree { + +struct StoreTraits { + using Key = Id; + using Hasher = std::hash; + + static constexpr Key root() { return Key::root(); } + static constexpr auto parent(const Key& key) { return key.parent(); } + static constexpr auto children(const Key& key) { return key.children(); } + static constexpr bool is_valid(const Key& key) { return key.level() <= Key::max_level() && key.index_on_level() <= Key::max_index_on_level(key.level()); } + static std::string key_to_string(const Key& key) { return key.to_string(); } +}; + +} // namespace octree diff --git a/src/terrainlib/octree/disk/IndexFile.h b/src/terrainlib/octree/disk/IndexFile.h deleted file mode 100644 index 478be69f..00000000 --- a/src/terrainlib/octree/disk/IndexFile.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/IndexMap.h" - -namespace octree::disk::v1 { - -inline constexpr std::string_view index_file_name() { - return "terrain.index"; -} -inline constexpr std::string_view index_extension() { - return ".index"; -} - -struct IndexFile { - using serialize = zpp::bits::members<3>; - - std::string layout_strategy_id; - std::string preferred_extension; - IndexMap map; -}; -} diff --git a/src/terrainlib/octree/disk/Layout.h b/src/terrainlib/octree/disk/Layout.h deleted file mode 100644 index ba56f40b..00000000 --- a/src/terrainlib/octree/disk/Layout.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" - -namespace octree::disk { - -class Layout { -public: - Layout(std::filesystem::path base_path, std::unique_ptr strategy, std::string extension_with_dot) - : _base_path(std::move(base_path)), _strategy(std::move(strategy)), _extension_with_dot(std::move(extension_with_dot)) { - DEBUG_ASSERT(!_extension_with_dot.empty() && _extension_with_dot[0] == '.'); - } - - std::filesystem::path get_node_path(Id id) const { - return this->_base_path / this->_strategy->get_relative_node_path(id, this->_extension_with_dot); - } - std::optional get_id_from_node_path(const std::filesystem::path& path) const { - const std::filesystem::path relative_path = std::filesystem::relative(path, _base_path); - return this->_strategy->get_id_from_relative_node_path(relative_path); - } - - const std::filesystem::path& base_path() const { - return this->_base_path; - } - const layout::Strategy &strategy() const { - return *this->_strategy; - } - std::string_view extension_with_dot() const { - return this->_extension_with_dot; - } - std::string_view extension() const { - return this->extension_with_dot().substr(1); - } - -private: - std::filesystem::path _base_path; - std::unique_ptr _strategy; - std::string _extension_with_dot; -}; - -} diff --git a/src/terrainlib/octree/disk/layout/Strategy.h b/src/terrainlib/octree/disk/layout/Strategy.h deleted file mode 100644 index a1d5fb70..00000000 --- a/src/terrainlib/octree/disk/layout/Strategy.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "octree/Id.h" - -namespace octree::disk::layout { - -class Strategy { -public: - virtual ~Strategy() = default; - - virtual std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const = 0; - virtual std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const = 0; -}; - -} diff --git a/src/terrainlib/octree/disk/layout/StrategyRegister.h b/src/terrainlib/octree/disk/layout/StrategyRegister.h deleted file mode 100644 index 026a44f7..00000000 --- a/src/terrainlib/octree/disk/layout/StrategyRegister.h +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include "octree/disk/layout/Strategy.h" -#include "log.h" - -namespace octree::disk::layout { - -class StrategyRegister { -public: - using Factory = std::function()>; - - static StrategyRegister &instance() { - static StrategyRegister inst; - return inst; - } - - template - void register_strategy(const std::string &id) { - this->_factories[id] = [] { return std::make_unique(); }; - this->_type_to_id[std::type_index(typeid(T))] = id; - } - - std::unique_ptr create(const std::string &id) const { - auto it = this->_factories.find(id); - if (it != this->_factories.end()) { - return (it->second)(); - } - return nullptr; - } - - const std::unordered_map &factories() const { - return this->_factories; - } - - template - std::string get_id() const { - return this->_get_id(typeid(T)); - } - - std::string get_id(const Strategy &s) const { - return this->_get_id(typeid(s)); - } - -private: - std::unordered_map _factories; - std::unordered_map _type_to_id; - - // Private ctor/dtor to enforce singleton - StrategyRegister() = default; - ~StrategyRegister() = default; - StrategyRegister(const StrategyRegister &) = delete; - StrategyRegister &operator=(const StrategyRegister &) = delete; - - std::string _get_id(std::type_index type_index) const { - auto it = this->_type_to_id.find(type_index); - if (it != this->_type_to_id.end()) { - return it->second; - } - LOG_ERROR_AND_EXIT("Encountered octree layout strategy not registered"); - } -}; - -} // namespace octree::disk::layout - -#include "octree/disk/layout/strategy/Flat.h" -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" - -namespace { -template -struct Registrar { - Registrar(const std::string &id) { - octree::disk::layout::StrategyRegister::instance().register_strategy(id); - } -}; - -const Registrar reg_flat("flat"); -const Registrar reg_lacd("level_and_coordinate_directories"); -} diff --git a/src/terrainlib/octree/disk/layout/strategy/Default.h b/src/terrainlib/octree/disk/layout/strategy/Default.h deleted file mode 100644 index 45c7842d..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/Default.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -#include "octree/disk/layout/strategy/LevelAndCoordinateDirectories.h" - -namespace octree::disk::layout::strategy { - -using Default = LevelAndCoordinateDirectories; - -inline std::unique_ptr make_default() { - return std::make_unique(); -} - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/disk/layout/strategy/Flat.h b/src/terrainlib/octree/disk/layout/strategy/Flat.h deleted file mode 100644 index 0a11a941..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/Flat.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" -#include "string_utils.h" - -namespace octree::disk::layout::strategy { - -class Flat : public Strategy { -public: - std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const override { - return fmt::format("{}-{}{}", id.level(), id.index_on_level(), extension_with_dot); - } - - std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const override { - DEBUG_ASSERT(relative_path.is_relative()); - - const std::string _filestem = relative_path.stem().string(); - const std::string_view filestem = _filestem; - - const size_t dash_pos = filestem.find('-'); - if (dash_pos == std::string::npos) { - return std::nullopt; - } - const std::string_view level_str = filestem.substr(0, dash_pos); - const std::string_view index_str = filestem.substr(dash_pos + 1); - - const auto level_opt = from_chars(level_str); - if (!level_opt) { - return std::nullopt; - } - - const auto index_opt = from_chars(index_str); - if (!index_opt) { - return std::nullopt; - } - - const Id::Level level = *level_opt; - const Id::Index index = *index_opt; - return Id::try_make(level, index); - } -}; - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h deleted file mode 100644 index 6a5ffa68..00000000 --- a/src/terrainlib/octree/disk/layout/strategy/LevelAndCoordinateDirectories.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/layout/Strategy.h" -#include "string_utils.h" - -namespace octree::disk::layout::strategy { - -class LevelAndCoordinateDirectories : public Strategy { -public: - std::filesystem::path get_relative_node_path(Id id, std::string_view extension_with_dot) const override { - const Id::Coords coords = id.coords(); - return fmt::format("{}/{}/{}/{}{}", id.level(), coords.x, coords.y, coords.z, extension_with_dot); - } - std::optional get_id_from_relative_node_path(const std::filesystem::path &relative_path) const override { - DEBUG_ASSERT(relative_path.is_relative()); - - auto it = relative_path.begin(); - if (std::distance(it, relative_path.end()) < 4) { - return std::nullopt; - } - - const auto level_opt = from_chars(it->native()); - if (!level_opt) { - return std::nullopt; - } - const Id::Level level = *level_opt; - ++it; - - const auto x_opt = from_chars(it->native()); - if (!x_opt) { - return std::nullopt; - } - const Id::Coord x = *x_opt; - ++it; - - const auto y_opt = from_chars(it->native()); - if (!y_opt) { - return std::nullopt; - } - const Id::Coord y = *y_opt; - ++it; - - const auto z_opt = from_chars(it->stem().native()); - if (!z_opt) { - return std::nullopt; - } - const Id::Coord z = *z_opt; - - return Id::try_make(level, {x, y, z}); - } -}; - -} // namespace octree::disk::layout::strategy diff --git a/src/terrainlib/octree/storage/CopyError.h b/src/terrainlib/octree/storage/CopyError.h deleted file mode 100644 index 7bd6c278..00000000 --- a/src/terrainlib/octree/storage/CopyError.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include -#include - -namespace octree { - -enum class CopyErrorKind { - FileNotFound, - CreateLink, - CreateDirectories, - RemoveOld, - Read, - Write -}; - -class CopyError { -public: - constexpr CopyError(CopyErrorKind kind) - : kind(kind) {} - - operator CopyErrorKind() const { - return this->kind; - } - constexpr bool operator==(CopyError other) const { - return this->kind == other.kind; - } - constexpr bool operator!=(CopyError other) const { - return this->kind != other.kind; - } - - std::string description() const { - switch (kind) { - case CopyErrorKind::FileNotFound: - return "file not found"; - case CopyErrorKind::CreateLink: - return "cannot create hard link"; - case CopyErrorKind::CreateDirectories: - return "cannot create directories"; - case CopyErrorKind::RemoveOld: - return "cannot remove current file"; - case CopyErrorKind::Read: - return "cannot read file"; - case CopyErrorKind::Write: - return "cannot write file"; - default: - return "undefined error"; - } - } - - friend std::ostream &operator<<(std::ostream &os, const CopyError &err) { - return os << err.description(); - } - -private: - CopyErrorKind kind; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/IndexFile.h b/src/terrainlib/octree/storage/IndexFile.h new file mode 100644 index 00000000..c1288e73 --- /dev/null +++ b/src/terrainlib/octree/storage/IndexFile.h @@ -0,0 +1,200 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "io/envelope_file.h" +#include "octree/StoreTraits.h" +#include "octree/store_layout/Mappings.h" +#include "store/Index.h" +#include "store/IndexFormat.h" + +namespace octree::storage { + +inline constexpr std::string_view metadata_file_name = "octree.storemeta"; +inline constexpr std::string_view index_file_name = "octree.storeindex"; + +namespace v1 { + + struct StoreMetadata { + std::string layout_id; + std::string payload_class; + std::string codec_selector; + }; + + struct IndexEntry { + std::uint8_t level; + std::uint64_t index; + std::uint8_t status; + }; + + struct StoreIndex { + std::vector entries; + }; + +} // namespace v1 + +using StoreMetadataSchema = io::envelope::PayloadSchema<"octree.StoreMetadata", io::envelope::Version<1, v1::StoreMetadata>>; +using StoreIndexSchema = io::envelope::PayloadSchema<"octree.StoreIndex", io::envelope::Version<1, v1::StoreIndex>>; + +using StoreMetadata = StoreMetadataSchema::latest_type; +using StoreIndex = StoreIndexSchema::latest_type; + +inline Expected validate(const StoreMetadata& metadata) +{ + if (metadata.layout_id.empty()) { + return Error::fail(Error::Code::InvalidInput, "layout ID is empty"); + } + if (metadata.payload_class.empty()) { + return Error::fail(Error::Code::InvalidInput, "payload class is empty"); + } + if (metadata.codec_selector.empty()) { + return Error::fail(Error::Code::InvalidInput, "codec selector is empty"); + } + return {}; +} + +inline StoreIndex encode_index(const store::Index& index) +{ + StoreIndex result; + result.entries.reserve(index.size()); + for (const auto& [id, status] : index) { + result.entries.push_back({ + .level = id.level(), + .index = id.index_on_level(), + .status = static_cast(static_cast(status)), + }); + } + return result; +} + +inline Expected> decode_index(const StoreIndex& encoded) +{ + store::Index result; + std::unordered_set seen; + for (const v1::IndexEntry& entry : encoded.entries) { + const auto id = Id::try_make(static_cast(entry.level), static_cast(entry.index)); + if (!id) { + return Error::fail(Error::Code::CorruptData, "index contains an invalid octree ID"); + } + if (!seen.insert(*id).second) { + return Error::fail(Error::Code::CorruptData, "index contains a duplicate octree ID"); + } + if (entry.status > static_cast(store::NodeStatus::Virtual)) { + return Error::fail(Error::Code::CorruptData, "index contains an invalid node status"); + } + auto set = result.set_raw(*id, store::NodeStatus { static_cast(entry.status) }); + if (!set) { + return Error::propagate(std::move(set), Error::Code::CorruptData, "add decoded node to store index"); + } + } + + for (const auto& [id, status] : result) { + const auto children = StoreTraits::children(id); + bool has_child = false; + if (children) { + for (const Id& child : *children) { + auto child_status = result.get(child); + if (!child_status) { + return Error::propagate(std::move(child_status), Error::Code::CorruptData, "look up child while validating store index"); + } + has_child = has_child || child_status->has_value(); + } + } + if ((status == store::NodeStatus::Leaf && has_child) || ((status == store::NodeStatus::Inner || status == store::NodeStatus::Virtual) && !has_child)) { + return Error::fail(Error::Code::CorruptData, "index contains inconsistent node topology"); + } + + const auto parent = StoreTraits::parent(id); + if (!parent) { + if (id != StoreTraits::root()) { + return Error::fail(Error::Code::CorruptData, "index contains a non-root node without a parent"); + } + continue; + } + auto parent_status = result.get(*parent); + if (!parent_status) { + return Error::propagate(std::move(parent_status), Error::Code::CorruptData, "look up parent while validating store index"); + } + if (!parent_status->has_value() || parent_status->value() == store::NodeStatus::Leaf) { + return Error::fail(Error::Code::CorruptData, "index contains a node without a valid indexed parent"); + } + } + return result; +} + +inline Expected read_store_metadata(const std::filesystem::path& base_path) +{ + const std::filesystem::path path = base_path / metadata_file_name; + auto metadata = io::envelope::read_from_path(path); + if (!metadata) { + return metadata; + } + if (auto valid = validate(*metadata); !valid) { + return Error::propagate(std::move(valid), Error::Code::CorruptData, "validate store metadata read from \"" + path.string() + "\""); + } + return std::move(*metadata); +} + +inline Expected> read_index_file(const std::filesystem::path& path) +{ + auto metadata = read_store_metadata(path.parent_path()); + if (!metadata) { + return Error::propagate(std::move(metadata), "read octree store metadata"); + } + + auto encoded_index = io::envelope::read_from_path(path); + if (!encoded_index) { + return Error::propagate(std::move(encoded_index), "read octree store index \"" + path.string() + "\""); + } + auto index = decode_index(*encoded_index); + if (!index) { + return Error::propagate(std::move(index), "decode store index read from \"" + path.string() + "\""); + } + return store::IndexMetadata { + std::move(*index), + std::move(metadata->layout_id), + std::move(metadata->payload_class), + std::move(metadata->codec_selector), + }; +} + +inline Expected write_index_file(const std::filesystem::path& path, const store::IndexMetadata& metadata) +{ + const StoreMetadata store_metadata { + .layout_id = metadata.layout_id, + .payload_class = metadata.payload_class, + .codec_selector = metadata.codec_selector, + }; + if (auto valid = validate(store_metadata); !valid) { + return Error::propagate(std::move(valid), "validate store metadata for writing"); + } + const std::filesystem::path metadata_path = path.parent_path() / metadata_file_name; + if (auto written = io::envelope::write_to_path(store_metadata, metadata_path); !written) { + return Error::propagate(std::move(written), "write octree store metadata " + metadata_path.string()); + } + + const StoreIndex encoded_index = encode_index(metadata.index); + if (auto written = io::envelope::write_to_path(encoded_index, path); !written) { + return Error::propagate(std::move(written), "write octree store index " + path.string()); + } + return {}; +} + +inline store::path_layout::Mapping default_mapping() { return store_layout::level_and_coordinate_directories(); } + +inline store::IndexFormat index_format() +{ + return { + index_file_name, + read_index_file, + write_index_file, + store_layout::from_id, + default_mapping, + }; +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/storage/IndexedStorage.h b/src/terrainlib/octree/storage/IndexedStorage.h deleted file mode 100644 index 3ba94e8f..00000000 --- a/src/terrainlib/octree/storage/IndexedStorage.h +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once - -#include - -#include - -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/disk/Layout.h" -#include "octree/storage/Storage.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" -#include "octree/storage/helpers.h" - -namespace octree { - -template Codec = DefaultCodecFor> -class IndexedStorage_ : public Storage_ { -public: - explicit IndexedStorage_(Storage_ inner) noexcept - : Storage_(std::move(inner)) { - this->ensure_indexed(); - DEBUG_ASSERT(this->is_indexed()); - } - explicit IndexedStorage_(RawStorage_ inner, IndexMap map) noexcept - : Storage_(std::move(inner), std::move(map)) { - DEBUG_ASSERT(this->is_indexed()); - } - - IndexedStorage_ &operator=(const IndexedStorage_ &) = delete; - IndexedStorage_(const IndexedStorage_ &) = delete; - IndexedStorage_(IndexedStorage_ &&) = default; - IndexedStorage_ &operator=(IndexedStorage_ &&) = default; - - ~IndexedStorage_() override { - if (this->is_index_dirty()) { - LOG_WARN("Index was not saved upon IndexedStorage_ destruction, use save_index()."); - } - } - - const IndexMap& index() const noexcept { - DEBUG_ASSERT(this->is_indexed()); - return Storage_::index().value(); - } - void update_index() noexcept { - Storage_::update_index(); - } - std::expected save_index() const noexcept { - if (!this->is_index_dirty()) { - return {}; - } - - auto result = helpers::save_index_map(this->index(), this->layout()); - if (result.has_value()) { - this->set_index_dirty(false); - } - return result; - } - -private: - IndexMap& index_mut() noexcept { - DEBUG_ASSERT(this->is_indexed()); - return Storage_::index_mut().value(); - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/MeshStorage.h b/src/terrainlib/octree/storage/MeshStorage.h deleted file mode 100644 index 46b69813..00000000 --- a/src/terrainlib/octree/storage/MeshStorage.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include "mesh/SimpleMesh.h" -#include "octree/storage/Storage.h" -#include "octree/storage/IndexedStorage.h" - -namespace octree { - -using MeshStorage = Storage_; -using IndexedMeshStorage = IndexedStorage_; - -} diff --git a/src/terrainlib/octree/storage/RawStorage.h b/src/terrainlib/octree/storage/RawStorage.h deleted file mode 100644 index 39482250..00000000 --- a/src/terrainlib/octree/storage/RawStorage.h +++ /dev/null @@ -1,118 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "octree/Id.h" -#include "octree/disk/Layout.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/CopyError.h" -#include "octree/storage/defaults.h" - -namespace octree { - -template Codec = DefaultCodecFor> -class RawStorage_ { -public: - using value_type = T; - using codec_type = Codec; - using load_error = typename Codec::load_error; - using save_error = typename Codec::save_error; - - explicit RawStorage_(disk::Layout layout) noexcept : _layout(std::move(layout)) {} - - ~RawStorage_() = default; - RawStorage_ &operator=(const RawStorage_ &) = delete; - RawStorage_(const RawStorage_ &) = delete; - RawStorage_(RawStorage_ &&) = default; - RawStorage_ &operator=(RawStorage_ &&) = default; - - std::expected load(const Id &id) const noexcept { - const auto path = this->path_for(id); - return Codec::load_from_path(path); - } - - std::expected save(const Id &id, const T &node) const noexcept { - const auto path = this->path_for(id); - return Codec::save_to_path(node, path); - } - - std::expected copy_to(const Id &id, RawStorage_ &target) const noexcept { - return target.copy_from(id, *this); - } - - std::expected copy_from(const Id &id, const RawStorage_ &source) noexcept { - if (!source.has(id)) { - return std::unexpected(CopyErrorKind::FileNotFound); - } - - const auto source_path = source.path_for(id); - const auto target_path = this->path_for(id); - if (source_path.extension() != target_path.extension()) { - // TODO: should this error instead? - const auto load_result = source.load(id); - if (!load_result.has_value()) { - return std::unexpected(CopyErrorKind::Read); - } - const value_type node = load_result.value(); - - const auto save_result = this->save(id, node); - if (!save_result.has_value()) { - return std::unexpected(CopyErrorKind::Write); - } - return {}; - } - - std::error_code ec; - if (std::filesystem::remove(target_path, ec)) { - if (ec) { - return std::unexpected(CopyErrorKind::RemoveOld); - } - } - - std::filesystem::create_directories(target_path.parent_path(), ec); - if (ec) { - return std::unexpected(CopyErrorKind::CreateDirectories); - } - - std::filesystem::create_hard_link(source_path, target_path, ec); - if (ec) { - return std::unexpected(CopyErrorKind::CreateLink); - } - - return {}; - } - - bool remove(const Id &id) const noexcept { - const auto path = this->path_for(id); - std::error_code ec; - const auto removed = std::filesystem::remove(path, ec); - return !ec && removed; - } - - bool has(const Id &id) const noexcept { - const auto path = this->path_for(id); - std::error_code ec; - const auto exists = std::filesystem::exists(path, ec); - return !ec && exists; - } - - std::filesystem::path path_for(const Id &id) const noexcept { - return this->_layout.get_node_path(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_layout.base_path(); - } - - const disk::Layout &layout() const noexcept { - return this->_layout; - } - -private: - disk::Layout _layout; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/Storage.h b/src/terrainlib/octree/storage/Storage.h deleted file mode 100644 index 782a9c03..00000000 --- a/src/terrainlib/octree/storage/Storage.h +++ /dev/null @@ -1,312 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -#include "mesh/io.h" -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/strategy/Default.h" -#include "octree/NodeStatusOrMissing.h" -#include "octree/storage/RawStorage.h" -#include "octree/storage/cache/ICache.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" -#include "octree/storage/helpers.h" - -namespace octree { - -namespace detail { -struct MaybeIndex { - std::optional map; - mutable bool dirty = false; - - explicit MaybeIndex() : map(std::nullopt) {} - explicit MaybeIndex(IndexMap map) : map(std::move(map)) {} - - MaybeIndex(const MaybeIndex &) = default; - MaybeIndex &operator=(const MaybeIndex &) = default; - - MaybeIndex(MaybeIndex &&other) noexcept - : map(std::move(other.map)), dirty(std::exchange(other.dirty, false)) { - other.map.reset(); - } - - MaybeIndex &operator=(MaybeIndex &&other) noexcept { - this->map = std::move(other.map); - this->dirty = std::exchange(other.dirty, false); - other.map.reset(); - return *this; - } - - bool add(const Id &id) noexcept { - if (!map.has_value()) { - return false; - } - - const bool changed = map->add(id); - dirty = dirty || changed; - return changed; - } - - bool remove(const Id &id) noexcept { - if (!map.has_value()) { - return false; - } - - const bool changed = map->remove(id); - dirty = dirty || changed; - return changed; - } - - bool contains(const Id &id, const bool def = false) const noexcept { - if (this->map.has_value()) { - const octree::NodeStatusOrMissing status = this->map->get(id); - return status == octree::NodeStatusOrMissing::Leaf || status == NodeStatusOrMissing::Inner; - } - return def; - } -}; - -template -struct MaybeCache : public cache::ICache { - std::optional>> cache; - - explicit MaybeCache() : cache(std::nullopt) {} - explicit MaybeCache(std::unique_ptr> cache) : cache(std::move(cache)) {} - - std::optional get(const Id& id) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->get(id); - } - return std::nullopt; - } - - bool put(const Id &id, const T &value) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->put(id, value); - } - return false; - } - - bool remove(const Id &id) noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->remove(id); - } - return false; - } - - bool contains(const Id &id) const noexcept override { - if (this->cache.has_value()) { - return this->cache->get()->contains(id); - } - return false; - } -}; -} - -struct StorageSettings { - bool allow_overwrite = false; -}; - -template Codec = DefaultCodecFor> -class Storage_ { -public: - using value_type = T; - using codec_type = Codec; - using load_error = typename Codec::load_error; - using save_error = typename Codec::save_error; - - explicit Storage_(RawStorage_ inner) - : _inner(std::move(inner)) {} - explicit Storage_(RawStorage_ inner, IndexMap index) - : _inner(std::move(inner)), _index(detail::MaybeIndex(std::move(index))) {} - - Storage_ &operator=(const Storage_ &) = delete; - Storage_(const Storage_ &) = delete; - Storage_(Storage_ &&) = default; - Storage_ &operator=(Storage_ &&) = default; - - virtual ~Storage_() { - if (this->_index.map.has_value() && this->_index.dirty) { - auto result = helpers::save_index_map(this->_index.map.value(), this->_inner.layout()); - if (!result.has_value()) { - LOG_ERROR("Failed to automatically save index when closing storage: {}", result.error()); - } - } - } - - std::expected load(const Id &id) const noexcept { - if (const auto value_opt = this->_cache.get(id)) { - return value_opt.value(); - } - - if (!this->_index.contains(id, true)) { - return std::unexpected(Codec::file_not_found()); - } - - const auto result = this->_inner.load(id); - if (result.has_value()) { - this->_cache.put(id, result.value()); - } - return result; - } - - std::expected save(const Id &id, const value_type &value) noexcept { - if (this->check_overwrite(id)) { - LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); - } - - const auto result = this->_inner.save(id, value); - if (result.has_value()) { - this->_cache.put(id, value); - this->_index.add(id); - } - return result; - } - - std::expected copy_from(const Id &id, const Storage_ &source) noexcept { - if (!source._index.contains(id, true)) { - return std::unexpected(CopyErrorKind::FileNotFound); - } - - if (this->check_overwrite(id)) { - // TODO: proper error - LOG_ERROR_AND_EXIT("tried to overwrite value when not allowed"); - } - - const auto result = this->_inner.copy_from(id, source._inner); - if (result.has_value()) { - this->_index.add(id); - } - return result; - } - - std::expected copy_to(const Id &id, Storage_ &target) const noexcept { - return target.copy_from(id, *this); - } - - bool remove(const Id &id) noexcept { - this->_cache.remove(id); - this->_index.remove(id); - return this->_inner.remove(id); - } - - bool has(const Id &id) const noexcept { - if (this->is_indexed()) { - return this->_index.contains(id, false); - } else { - return this->_inner.has(id); - } - } - - std::filesystem::path path_for(const Id &id) const noexcept { - return this->_inner.path_for(id); - } - - std::filesystem::path base_path() const noexcept { - return this->_inner.base_path(); - } - - bool is_indexed() const noexcept { - return this->_index.map.has_value(); - } - - void ensure_indexed() noexcept { - if (this->is_indexed()) { - return; - } - - LOG_TRACE("Index not present, creating empty index"); - IndexMap map; - helpers::update_index_map(map, this->_inner.layout()); - LOG_TRACE("Index created with {} entries", map.size()); - this->_index.map = std::move(map); - } - - std::optional> index() const noexcept { - // I miss Option::as_ref - if (this->_index.map.has_value()) { - return this->_index.map.value(); - } else { - return std::nullopt; - } - } - - std::optional>> &cache() noexcept { - return this->_cache.cache; - } - - std::optional>> cache() const noexcept { - if (this->_cache.cache.has_value()) { - return *this->_cache.cache.value(); - } else { - return std::nullopt; - } - } - - std::expected save_or_create_index() noexcept { - if (this->is_indexed() && !this->_index.dirty) { - return {}; - } - - this->ensure_indexed(); - - auto result = helpers::save_index_map(this->index().value(), this->_inner.layout()); - if (result.has_value()) { - this->_index.dirty = false; - } - return result; - } - - const octree::disk::Layout &layout() const noexcept { - return this->_inner.layout(); - } - - const StorageSettings& settings() const noexcept { - return this->_settings; - } - StorageSettings& settings() noexcept { - return this->_settings; - } - -protected: - bool check_overwrite(const Id &id) noexcept { - return !this->_settings.allow_overwrite && this->has(id); - } - - std::optional &index_mut() noexcept { - return this->_index.map; - } - - bool is_index_dirty() const noexcept { - return this->_index.dirty; - } - void set_index_dirty(const bool val = true) const noexcept { - this->_index.dirty = val; - } - - void update_index() noexcept { - if (!this->is_indexed()) { - return; - } - - helpers::update_index_map(this->index_mut().value(), this->_inner.layout()); - this->_index.dirty = false; - } - -private: - RawStorage_ _inner; - detail::MaybeIndex _index = detail::MaybeIndex(); - mutable detail::MaybeCache _cache = detail::MaybeCache(); - StorageSettings _settings = {}; -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/cache/Dummy.h b/src/terrainlib/octree/storage/cache/Dummy.h deleted file mode 100644 index ecf6cff8..00000000 --- a/src/terrainlib/octree/storage/cache/Dummy.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include - -#include "octree/Id.h" -#include "octree/storage/cache/ICache.h" - -namespace octree::cache { - - template -class Dummy : public ICache { -public: - std::optional get(const Id &) noexcept override { - return std::nullopt; - } - bool put(const Id &, const T &) noexcept override { - return false; - } - bool remove(const Id &) noexcept override { - return false; - } - bool contains(const Id &) const noexcept override { - return false; - } -}; - -} diff --git a/src/terrainlib/octree/storage/cache/ICache.h b/src/terrainlib/octree/storage/cache/ICache.h deleted file mode 100644 index bd0258bc..00000000 --- a/src/terrainlib/octree/storage/cache/ICache.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include - -#include "octree/Id.h" - -namespace octree::cache { - -template -class ICache { -public: - virtual ~ICache() = default; - - virtual std::optional get(const Id& id) noexcept = 0; - virtual bool put(const Id &id, const T &value) noexcept = 0; - virtual bool remove(const Id &id) noexcept = 0; - virtual bool contains(const Id &id) const noexcept = 0; -}; - -} diff --git a/src/terrainlib/octree/storage/cache/LruCache.h b/src/terrainlib/octree/storage/cache/LruCache.h deleted file mode 100644 index 1c705e05..00000000 --- a/src/terrainlib/octree/storage/cache/LruCache.h +++ /dev/null @@ -1,71 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "octree/Id.h" -#include "octree/RawStorage.h" -#include "octree/cache/ICache.h" - -namespace octree::cache { - -// TODO: UNTESTED -// Do not use this cache for NodeLoader's ancestor lookup until cached ancestors -// are clipped to the requested node bounds. NodeLoader currently returns a -// cached ancestor unchanged, producing incorrect geometry for child requests. -template -class Lru_ : public ICache { -public: - explicit Lru_(const size_t capacity) : _capacity(capacity) {} - - std::optional get(const Key& key) { - auto it = this->_map.find(key); - if (it == this->_map.end()) return std::nullopt; - - // Move to front (most recently used) - this->_usage.splice(this->_usage.begin(), this->_usage, it->second.second); - return it->second.first; - } - - void put(const Key& key, const Value& value) { - auto it = this->_map.find(key); - if (it != this->_map.end()) { - // Update value and move to front - it->second.first = value; - this->_usage.splice(this->_usage.begin(), this->_usage, it->second.second); - } else { - // Insert new entry - if (this->_map.size() == _capacity) { - // Evict least recently used - const Key& lru_key = this->_usage.back(); - this->_map.erase(lru_key); - this->_usage.pop_back(); - } - - this->_usage.push_front(key); - this->_map[key] = { value, this->_usage.begin() }; - } - } - - void remove(const Key& key) { - auto it = this->_map.find(key); - if (it != this->_map.end()) { - this->_usage.erase(it->second.second); - this->_map.erase(it); - } - } - - bool contains(const Key& key) const { - return this->_map.contains(key); - } - -private: - size_t _capacity; - std::list _usage; - std::unordered_map::iterator>> _map; -}; - -using Lru = Lru; - -} diff --git a/src/terrainlib/octree/storage/codec/Codec.h b/src/terrainlib/octree/storage/codec/Codec.h deleted file mode 100644 index f31b6f45..00000000 --- a/src/terrainlib/octree/storage/codec/Codec.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -template -concept CodecFor = - requires(const std::filesystem::path &path, const T &value) { - typename Codec::value_type; - typename Codec::load_error; - typename Codec::save_error; - - requires std::same_as; - - { Codec::load_from_path(path) } noexcept - -> std::same_as>; - - { Codec::save_to_path(value, path) } noexcept - -> std::same_as>; - - { Codec::file_not_found() } noexcept - -> std::same_as; - - { Codec::supported_extensions() } -> std::ranges::range; - - requires std::convertible_to< - std::ranges::range_value_t, - std::string_view>; - }; diff --git a/src/terrainlib/octree/storage/codec/MeshCodec.h b/src/terrainlib/octree/storage/codec/MeshCodec.h deleted file mode 100644 index 7a15400a..00000000 --- a/src/terrainlib/octree/storage/codec/MeshCodec.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "mesh/io.h" - -namespace octree { - -struct MeshCodec { - using value_type = mesh::Simple; - using load_error = mesh::io::LoadMeshError; - using save_error = mesh::io::SaveMeshError; - - static std::expected load_from_path(const std::filesystem::path& path) noexcept { - return mesh::io::load_from_path(path); - } - - static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { - return mesh::io::save_to_path(value, path); - } - - static load_error file_not_found() noexcept { - return mesh::io::LoadMeshErrorKind::FileNotFound; - } - - static std::vector supported_extensions() { - return {".terrain", ".glb", ".gltf"}; - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h b/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h deleted file mode 100644 index 3ea68ba9..00000000 --- a/src/terrainlib/octree/storage/codec/ReadOnlyCodec.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include - -#include "log.h" - -namespace octree { - -// Wraps a codec so saving always fails loudly instead of writing, for storages that must stay -// read-only (e.g. a view that aliases another storage's files). -template -struct ReadOnlyCodec : Codec { - static std::expected save_to_path( - const typename Codec::value_type &, const std::filesystem::path &) noexcept { - LOG_ERROR_AND_EXIT("Attempted to write through a read-only codec"); - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h b/src/terrainlib/octree/storage/codec/ZppBitsCodec.h deleted file mode 100644 index dc4acd5a..00000000 --- a/src/terrainlib/octree/storage/codec/ZppBitsCodec.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "Codec.h" -#include "io/serialize.h" -#include "io/Error.h" - -namespace octree { - -template -struct ZppBitsCodec { - using value_type = T; - using load_error = io::Error; - using save_error = io::Error; - - static std::expected load_from_path(const std::filesystem::path& path) noexcept { - return io::read_from_path(path); - } - - static std::expected save_to_path(const value_type& value, const std::filesystem::path& path) noexcept { - return io::write_to_path(value, path); - } - - static load_error file_not_found() noexcept { - return io::Error::Value::OpenFile; - } - - static std::vector supported_extensions() { - return {".bin"}; - } -}; - -} // namespace octree diff --git a/src/terrainlib/octree/storage/defaults.h b/src/terrainlib/octree/storage/defaults.h deleted file mode 100644 index fb4981c4..00000000 --- a/src/terrainlib/octree/storage/defaults.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "mesh/SimpleMesh.h" -#include "codec/Codec.h" -#include "codec/MeshCodec.h" -#include "codec/ZppBitsCodec.h" - -namespace octree { - -using DefaultT = mesh::Simple; - -template -struct DefaultCodec { - using type = ZppBitsCodec; -}; - -template <> -struct DefaultCodec { - using type = MeshCodec; -}; - -template -using DefaultCodecFor = typename DefaultCodec>::type; - -} // namespace octree \ No newline at end of file diff --git a/src/terrainlib/octree/storage/helpers.cpp b/src/terrainlib/octree/storage/helpers.cpp deleted file mode 100644 index ed817515..00000000 --- a/src/terrainlib/octree/storage/helpers.cpp +++ /dev/null @@ -1,136 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include "octree/storage/helpers.h" -#include "io/serialize.h" -#include "octree/NodeStatus.h" -#include "octree/IndexMap.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/Strategy.h" -#include "octree/disk/layout/StrategyRegister.h" - -namespace octree::helpers { -namespace { -template -std::optional find_max_key(const std::unordered_map& map) { - if (map.empty()) { - return std::nullopt; - } - - auto max_it = map.begin(); - for (auto it = std::next(max_it); it != map.end(); it++) { - if (it->second > max_it->second) { - max_it = it; - } - } - - return max_it->first; -} -} - -std::optional guess_layout_strategy( - const std::filesystem::path &base_path, - size_t max_files_to_check) { - - if (!std::filesystem::is_directory(base_path)) { - return std::nullopt; - } - - std::vector candidate_paths; - std::unordered_map extension_counters; - for (const auto &entry : std::filesystem::recursive_directory_iterator(base_path)) { - if (max_files_to_check == 0) { - break; - } - if (!entry.is_regular_file() && !entry.is_symlink()) { - continue; - } - - const auto ext = entry.path().extension(); - if (ext == disk::v1::index_extension()) { - continue; - } - extension_counters[ext]++; - - candidate_paths.emplace_back(std::filesystem::relative(entry.path(), base_path)); - max_files_to_check--; - } - std::optional most_common_ext = find_max_key(extension_counters); - - std::unique_ptr best_strategy; - size_t best_match_count = 0; - const auto &factories = disk::layout::StrategyRegister::instance().factories(); - for (const auto &[_, make_strategy] : factories) { - auto strategy = make_strategy(); // assume returns unique_ptr - - size_t match_count = 0; - for (const auto &rel_path : candidate_paths) { - if (strategy->get_id_from_relative_node_path(rel_path).has_value()) { - match_count++; - } - } - - if (match_count > best_match_count) { - best_match_count = match_count; - best_strategy = std::move(strategy); - } - } - - if (best_strategy && most_common_ext.has_value()) { - return LayoutWithoutBase(std::move(best_strategy), most_common_ext.value()); - } - - return std::nullopt; -} - -std::expected save_index_map(const IndexMap& index, const disk::Layout& layout) { - const auto index_path = layout.base_path() / disk::v1::index_file_name(); - LOG_TRACE("Saving octree storage index to {}", index_path); - - disk::v1::IndexFile index_file; - index_file.map = index; - index_file.preferred_extension = layout.extension_with_dot(); - index_file.layout_strategy_id = disk::layout::StrategyRegister::instance().get_id(layout.strategy()); - - const auto result = io::write_to_path(index_file, index_path); - if (!result.has_value()) { - LOG_ERROR("Failed to save octree storage index to {}", index_path); - return std::unexpected(result.error()); - } - return {}; -} - -void update_index_map(IndexMap &index, const disk::Layout &layout) { - index.clear(); - const auto &base_path = layout.base_path(); - - std::filesystem::create_directories(base_path); - - for (const auto &entry : std::filesystem::recursive_directory_iterator(base_path)) { - if (!entry.is_regular_file() && !entry.is_symlink()) { - continue; - } - - const auto ext = entry.path().extension(); - if (ext == disk::v1::index_extension()) { - continue; - } - - auto id_opt = layout.get_id_from_node_path(entry.path()); - if (!id_opt) { - continue; - } - - const Id id = *id_opt; - index.add(id); - } -} - -} diff --git a/src/terrainlib/octree/storage/helpers.h b/src/terrainlib/octree/storage/helpers.h deleted file mode 100644 index 288096cf..00000000 --- a/src/terrainlib/octree/storage/helpers.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "io/Error.h" -#include "octree/IndexMap.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/Layout.h" -#include "octree/disk/layout/Strategy.h" - -namespace octree::helpers { - -struct LayoutWithoutBase { - std::unique_ptr strategy; - std::string extension_with_dot; -}; - -std::optional guess_layout_strategy( - const std::filesystem::path &base_path, - size_t max_files_to_check = 100); -std::expected save_index_map(const IndexMap &index, const disk::Layout &layout); -void update_index_map(IndexMap &index, const disk::Layout &layout); - -} diff --git a/src/terrainlib/octree/storage/open.h b/src/terrainlib/octree/storage/open.h deleted file mode 100644 index b21ede08..00000000 --- a/src/terrainlib/octree/storage/open.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include "io/Error.h" -#include "octree/disk/layout/strategy/Default.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/Storage.h" -#include "octree/storage/codec/Codec.h" -#include "octree/storage/defaults.h" - -namespace octree { - -struct OpenOptions { - std::unique_ptr default_layout_strategy = {}; - std::optional preferred_extension_with_dot = {}; -}; - -template Codec = DefaultCodecFor> -std::expected, io::Error> open_index(const std::filesystem::path &index_path); -template Codec = DefaultCodecFor> -Storage_ open_folder( - const std::filesystem::path &base_path, - const bool create_index = false, - OpenOptions options = {}); -template Codec = DefaultCodecFor> -IndexedStorage_ open_folder_indexed( - const std::filesystem::path &base_path, - OpenOptions options = {}); -} - -#include "open.inl" - diff --git a/src/terrainlib/octree/storage/open.inl b/src/terrainlib/octree/storage/open.inl deleted file mode 100644 index c60c3699..00000000 --- a/src/terrainlib/octree/storage/open.inl +++ /dev/null @@ -1,111 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include "io/serialize.h" -#include "log.h" -#include "io/Error.h" -#include "octree/disk/IndexFile.h" -#include "octree/disk/layout/StrategyRegister.h" -#include "octree/storage/IndexedStorage.h" -#include "octree/storage/helpers.h" -#include "octree/storage/defaults.h" -#include "range_utils.h" - -namespace octree { - -template Codec> -std::expected, io::Error> open_index(const std::filesystem::path &index_path) { - LOG_TRACE("Opening storage index {}", index_path); - - const auto result = io::read_from_path(index_path); - if (!result.has_value()) { - LOG_TRACE("Failed to open storage index due to {}", result.error()); - return std::unexpected(result.error()); - } - auto index_file = result.value(); - LOG_TRACE("Successfully read storage index with {} entries.", index_file.map.size()); - auto index_map = std::move(index_file.map); - const auto base_path = index_path.parent_path(); - auto layout_strategy = disk::layout::StrategyRegister::instance().create(index_file.layout_strategy_id); - disk::Layout layout(base_path, std::move(layout_strategy), index_file.preferred_extension); - RawStorage_ raw_storage(std::move(layout)); - return IndexedStorage_(std::move(raw_storage), std::move(index_map)); -} - -template Codec> -Storage_ open_folder( - const std::filesystem::path &base_path, - bool create_index, - OpenOptions options) { - LOG_TRACE("Opening storage folder {}", base_path); - - if (!std::filesystem::is_directory(base_path)) { - if (std::filesystem::exists(base_path)) { - LOG_ERROR_AND_EXIT("Base path {} exists but is not a directory", base_path); - } - - LOG_TRACE("Base path {} does not exist, creating it", base_path); - std::filesystem::create_directories(base_path); - } - - const std::filesystem::path index_path = base_path / disk::v1::index_file_name(); - auto storage_opt = open_index(index_path); - if (storage_opt.has_value()) { - LOG_TRACE("Loaded existing index"); - return Storage_(std::move(storage_opt.value())); - } - - auto layout_info_opt = helpers::guess_layout_strategy(base_path); - if (layout_info_opt.has_value()) { - const octree::helpers::LayoutWithoutBase &layout_info = layout_info_opt.value(); - LOG_TRACE("Guessed layout strategy of dataset as {} and extension as {}", - disk::layout::StrategyRegister::instance().get_id(*layout_info.strategy), - layout_info.extension_with_dot); - } else { - auto default_layout_strategy = std::move(options.default_layout_strategy); - if (!default_layout_strategy) { - default_layout_strategy = disk::layout::strategy::make_default(); - } - std::string default_extension_with_dot; - const std::vector supported_extensions = Codec::supported_extensions(); - if (options.preferred_extension_with_dot.has_value()) { - ASSERT(contains(supported_extensions, options.preferred_extension_with_dot.value())); - default_extension_with_dot = options.preferred_extension_with_dot.value(); - } else { - default_extension_with_dot = supported_extensions[0]; - } - LOG_WARN("Unable to determine layout of dataset, using layout strategy {} and extension {}", - disk::layout::StrategyRegister::instance().get_id(*default_layout_strategy), - default_extension_with_dot); - layout_info_opt = octree::helpers::LayoutWithoutBase(std::move(default_layout_strategy), default_extension_with_dot); - } - - disk::Layout layout(base_path, std::move(layout_info_opt->strategy), layout_info_opt->extension_with_dot); - if (!create_index) { - return Storage_(RawStorage_(std::move(layout))); - } - - IndexMap map; - helpers::update_index_map(map, layout); - if (!map.empty()) { - const auto save_result = helpers::save_index_map(map, layout); - if (!save_result.has_value()) { - LOG_ERROR_AND_EXIT("Failed to create storage index in {}: {}", base_path, save_result.error()); - } - } - return Storage_(RawStorage_(std::move(layout)), std::move(map)); -} - -template Codec> -IndexedStorage_ open_folder_indexed( - const std::filesystem::path &base_path, - OpenOptions options) { - return IndexedStorage_(open_folder(base_path, true, std::move(options))); -} - -} // namespace octree diff --git a/src/terrainlib/octree/storage/open_runtime.h b/src/terrainlib/octree/storage/open_runtime.h new file mode 100644 index 00000000..f8cee12b --- /dev/null +++ b/src/terrainlib/octree/storage/open_runtime.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "octree/StoreTraits.h" +#include "octree/storage/IndexFile.h" +#include "store/open.h" + +namespace octree::storage { + +struct OpenOptions { + std::optional> default_mapping = std::nullopt; + std::optional preferred_extension = std::nullopt; +}; + +template +Expected> open_folder(const std::filesystem::path& base_path, + std::string payload_class, + std::string default_codec_selector, + CodecResolver resolve_codec, + OpenOptions options = {}) +{ + std::error_code error; + const bool base_exists = std::filesystem::exists(base_path, error); + if (error) { + return Error::fail(Error::Code::Io, "check existence of", base_path, error); + } + if (base_exists && !std::filesystem::is_directory(base_path, error)) { + return Error::fail(Error::Code::Io, "check directory", base_path, error ? error : std::make_error_code(std::errc::not_a_directory)); + } + if (!base_exists) { + std::filesystem::create_directories(base_path, error); + if (error) { + return Error::fail(Error::Code::Io, "create directories", base_path, error); + } + } + + const auto format = index_format(); + const std::filesystem::path index_path = base_path / format.index_filename; + const std::filesystem::path metadata_path = base_path / metadata_file_name; + const bool index_exists = std::filesystem::exists(index_path, error); + if (error) { + return Error::fail(Error::Code::Io, "check existence of", index_path, error); + } + const bool metadata_exists = std::filesystem::exists(metadata_path, error); + if (error) { + return Error::fail(Error::Code::Io, "check existence of", metadata_path, error); + } + if (index_exists != metadata_exists) { + return Error::fail(Error::Code::CorruptData, + "octree dataset metadata and index must either both exist or both be absent: " + (index_exists ? metadata_path : index_path).string()); + } + if (index_exists) { + auto indexed = store::open_index(index_path, format, payload_class, resolve_codec); + if (!indexed) { + return Error::propagate(std::move(indexed), "open indexed octree storage at \"" + base_path.string() + "\""); + } + return store::Storage(std::move(*indexed)); + } + + const store::path_layout::Mapping mapping = options.default_mapping.value_or(format.default_mapping()); + std::string codec_selector = options.preferred_extension.value_or(std::move(default_codec_selector)); + auto codec = resolve_codec(codec_selector); + if (!codec) { + return Error::propagate(std::move(codec), "resolve octree storage codec \"" + codec_selector + "\""); + } + + return store::make_storage(base_path, format, mapping, std::move(payload_class), std::move(codec_selector), std::move(*codec)); +} + +template +Expected> open_folder_indexed(const std::filesystem::path& base_path, + std::string payload_class, + std::string default_codec_selector, + CodecResolver resolve_codec, + OpenOptions options = {}) +{ + auto result = open_folder(base_path, std::move(payload_class), std::move(default_codec_selector), std::move(resolve_codec), std::move(options)); + if (!result) { + return Error::propagate(std::move(result)); + } + return store::IndexedStorage(std::move(*result)); +} + +} // namespace octree::storage diff --git a/src/terrainlib/octree/store_layout/Flat.h b/src/terrainlib/octree/store_layout/Flat.h new file mode 100644 index 00000000..f7f6944e --- /dev/null +++ b/src/terrainlib/octree/store_layout/Flat.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "octree/Id.h" +#include "string_utils.h" + +namespace octree::store_layout { + +inline std::filesystem::path flat_key_to_node_path(const Id& id) { return fmt::format("{}-{}", id.level(), id.index_on_level()); } + +inline std::optional flat_node_path_to_key(const std::filesystem::path& node_path) +{ + if (node_path.empty() || node_path.is_absolute() || node_path.has_parent_path() || node_path.has_extension()) { + return std::nullopt; + } + + const std::string text = node_path.string(); + const size_t separator = text.find('-'); + if (separator == std::string::npos || separator != text.rfind('-')) { + return std::nullopt; + } + const auto level = from_chars(std::string_view(text).substr(0, separator)); + const auto index = from_chars(std::string_view(text).substr(separator + 1)); + if (!level.has_value() || !index.has_value()) { + return std::nullopt; + } + return Id::try_make(level.value(), index.value()); +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h new file mode 100644 index 00000000..2dac849b --- /dev/null +++ b/src/terrainlib/octree/store_layout/LevelAndCoordinateDirectories.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +#include + +#include "octree/Id.h" +#include "string_utils.h" + +namespace octree::store_layout { + +inline std::filesystem::path level_and_coordinate_key_to_node_path(const Id& id) +{ + const Id::Coords coordinates = id.coords(); + return fmt::format("{}/{}/{}/{}", id.level(), coordinates.x, coordinates.y, coordinates.z); +} + +inline std::optional level_and_coordinate_node_path_to_key(const std::filesystem::path& node_path) +{ + if (node_path.empty() || node_path.is_absolute() || std::distance(node_path.begin(), node_path.end()) != 4) { + return std::nullopt; + } + + auto part = node_path.begin(); + const auto level = from_chars(part->string()); + ++part; + const auto x = from_chars(part->string()); + ++part; + const auto y = from_chars(part->string()); + ++part; + if (part->has_extension()) { + return std::nullopt; + } + const auto z = from_chars(part->string()); + if (!level.has_value() || !x.has_value() || !y.has_value() || !z.has_value()) { + return std::nullopt; + } + return Id::try_make(level.value(), { x.value(), y.value(), z.value() }); +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/store_layout/Mappings.h b/src/terrainlib/octree/store_layout/Mappings.h new file mode 100644 index 00000000..ce2bb437 --- /dev/null +++ b/src/terrainlib/octree/store_layout/Mappings.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include + +#include "octree/store_layout/Flat.h" +#include "octree/store_layout/LevelAndCoordinateDirectories.h" +#include "store/path_layout.h" + +namespace octree::store_layout { + +inline store::path_layout::Mapping flat() { return { "flat", flat_key_to_node_path, flat_node_path_to_key }; } + +inline store::path_layout::Mapping level_and_coordinate_directories() +{ + return { + "level_and_coordinate_directories", + level_and_coordinate_key_to_node_path, + level_and_coordinate_node_path_to_key, + }; +} + +inline std::array, 2> all() { return { flat(), level_and_coordinate_directories() }; } + +inline std::optional> from_id(const std::string_view id) +{ + for (const auto mapping : all()) { + if (mapping.id == id) { + return mapping; + } + } + return std::nullopt; +} + +} // namespace octree::store_layout diff --git a/src/terrainlib/octree/traverse.h b/src/terrainlib/octree/traverse.h deleted file mode 100644 index b912a0fa..00000000 --- a/src/terrainlib/octree/traverse.h +++ /dev/null @@ -1,81 +0,0 @@ -#pragma once - -#include -#include - -#include "octree/Id.h" -#include "octree/Storage.h" -#include "log.h" - -namespace octree { - -enum class TraversalOrder { - DepthFirst, - BreadthFirst -}; - -constexpr bool always_refine(const Id &) { - return true; -} - -template < - typename VisitFn, - typename RefineFn = std::function> -void traverse( - const IndexMap &index, - VisitFn &&visit_fn, - RefineFn &&refine_fn = always_refine, - const Id &root = Id::root(), - TraversalOrder order = TraversalOrder::DepthFirst) { - if (!index.is_present(root)) { - return; - } - - if (order == TraversalOrder::DepthFirst) { - std::function dfs; - dfs = [&](const Id& current) { - auto current_status_opt = index.get(current); - if (!current_status_opt) { - return; - } - const auto current_status = current_status_opt.value(); - - visit_fn(current, current_status); - - if (current.has_children() && refine_fn(current)) { - const auto children = current.children().value(); - for (const auto& child : children) { - dfs(child); - } - } - }; - dfs(root); - } else if (order == TraversalOrder::BreadthFirst) { - std::queue queue; - queue.push(root); - - while (!queue.empty()) { - Id current = queue.front(); - queue.pop(); - - auto current_status_opt = index.get(current); - if (!current_status_opt) { - continue; - } - const auto current_status = current_status_opt.value(); - - visit_fn(current, current_status); - - if (current.has_children() && refine_fn(current)) { - const auto children = current.children().value(); - for (const auto& child : children) { - queue.push(child); - } - } - } - } else { - UNREACHABLE(); - } -} - -} // namespace octree diff --git a/src/terrainlib/pch.h b/src/terrainlib/pch.h index dfbd471c..a936589c 100644 --- a/src/terrainlib/pch.h +++ b/src/terrainlib/pch.h @@ -35,7 +35,6 @@ #include #include #include -#include // Internal headers #include "log.h" diff --git a/src/terrainlib/raster_store/StoreTraits.h b/src/terrainlib/raster_store/StoreTraits.h new file mode 100644 index 00000000..111f941b --- /dev/null +++ b/src/terrainlib/raster_store/StoreTraits.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace raster_store { + +struct StoreTraits { + using Key = radix::tile::Id; + using Hasher = Key::Hasher; + + static constexpr unsigned max_zoom_level = std::numeric_limits::digits; + + static Key root() { return { 0, { 0, 0 } }; } + static std::optional parent(const Key& key) + { + if (key.zoom_level == 0) { + return std::nullopt; + } + return key.parent(); + } + static std::optional> children(const Key& key) + { + if (key.zoom_level >= max_zoom_level) { + return std::nullopt; + } + return key.children(); + } + static bool is_valid(const Key& key) + { + if (key.zoom_level > max_zoom_level) { + return false; + } + if (key.zoom_level == max_zoom_level) { + return true; + } + const uint32_t extent = uint32_t { 1 } << key.zoom_level; + return key.coords.x < extent && key.coords.y < extent; + } + static std::string key_to_string(const Key& key) { return to_string(key); } +}; + +} // namespace raster_store diff --git a/src/terrainlib/sf/finalize_storage.h b/src/terrainlib/sf/finalize_storage.h new file mode 100644 index 00000000..f6ff379b --- /dev/null +++ b/src/terrainlib/sf/finalize_storage.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include + +#include "mesh/storage.h" +#include "sf/validate_index.h" + +namespace sf { + +inline Expected finalize_storage(mesh::storage::Storage& storage) +{ + auto save_result = storage.save_or_create_index(); + if (!save_result) { + return save_result; + } + + const auto index = storage.index(); + DEBUG_ASSERT(index.has_value()); + return validate_index(index->get()); +} + +} // namespace sf diff --git a/src/terrainlib/sf/validate_index.h b/src/terrainlib/sf/validate_index.h new file mode 100644 index 00000000..09cb4083 --- /dev/null +++ b/src/terrainlib/sf/validate_index.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "Error.h" +#include "octree/StoreTraits.h" +#include "store/Index.h" + +namespace sf { + +inline Expected validate_index(const store::Index& index) +{ + for (const auto& [key, status] : index) { + if (status == store::NodeStatus::Inner) { + return Error::fail(Error::Code::CorruptData, "Structura Fundamentalis topology contains Inner node " + key.to_string()); + } + } + return {}; +} + +} // namespace sf diff --git a/src/terrainlib/store/Codec.h b/src/terrainlib/store/Codec.h new file mode 100644 index 00000000..56958e1a --- /dev/null +++ b/src/terrainlib/store/Codec.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include + +#include "Error.h" + +namespace store { + +template +class Codec { +public: + virtual ~Codec() = default; + + virtual std::vector paths(const std::filesystem::path& node_path) const = 0; + + virtual Expected read(const std::filesystem::path&) const { return Error::fail(Error::Code::Unsupported, "codec does not support reading"); } + + virtual Expected write(const std::filesystem::path&, const NodeData&) const + { + return Error::fail(Error::Code::Unsupported, "codec does not support writing"); + } + +protected: + static std::filesystem::path add_extension(std::filesystem::path path, const std::string_view extension) + { + path += extension; + return path; + } +}; + +} // namespace store diff --git a/src/terrainlib/store/Index.h b/src/terrainlib/store/Index.h new file mode 100644 index 00000000..d1e99098 --- /dev/null +++ b/src/terrainlib/store/Index.h @@ -0,0 +1,229 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "Error.h" +#include "log.h" +#include "store/NodeStatus.h" +#include "store/Traits.h" +#include + +namespace store { + +template +class Index { +public: + using Key = typename Traits::Key; + using Container = std::unordered_map; + using iterator = typename Container::iterator; + using const_iterator = typename Container::const_iterator; + + Expected> get(const Key& key) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + const NodeStatus* status = get_raw_unchecked(key); + if (status == nullptr) { + return std::nullopt; + } + return *status; + } + + Expected add(const Key& key) + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + + NodeStatus* status = get_raw_unchecked(key); + if (status != nullptr) { + switch (*status) { + case NodeStatus::Inner: + case NodeStatus::Leaf: + return false; + case NodeStatus::Virtual: + set_raw_unchecked(key, NodeStatus::Inner); + return true; + } + } + + const auto parent = Traits::parent(key); + if (!parent.has_value()) { + DEBUG_ASSERT(key == Traits::root()); + if (empty()) { + set_raw_unchecked(key, NodeStatus::Leaf); + return true; + } + UNREACHABLE(); + } + + if (!Traits::is_valid(parent.value())) { + return store::invalid_key_error(parent.value()); + } + NodeStatus* parent_status = get_raw_unchecked(parent.value()); + if (parent_status == nullptr) { + auto parent_result = add(parent.value()); + if (!parent_result) { + return parent_result; + } + set_raw_unchecked(parent.value(), NodeStatus::Virtual); + set_raw_unchecked(key, NodeStatus::Leaf); + return true; + } + + switch (*parent_status) { + case NodeStatus::Leaf: + set_raw_unchecked(parent.value(), NodeStatus::Inner); + set_raw_unchecked(key, NodeStatus::Leaf); + break; + case NodeStatus::Inner: + case NodeStatus::Virtual: + set_raw_unchecked(key, NodeStatus::Leaf); + break; + } + return true; + } + + Expected remove(const Key& key) + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + + NodeStatus* status = get_raw_unchecked(key); + if (status == nullptr) { + return false; + } + switch (*status) { + case NodeStatus::Inner: + *status = NodeStatus::Virtual; + return true; + case NodeStatus::Virtual: + return false; + case NodeStatus::Leaf: + remove_raw_unchecked(key); + return update_parent_after_remove(key); + } + UNREACHABLE(); + } + + Expected is_present(const Key& key) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + return m_index.contains(key); + } + Expected is_absent(const Key& key) const + { + auto present = is_present(key); + if (!present) { + return present; + } + return !present.value(); + } + Expected is(const NodeStatus status, const Key& key) const + { + auto result = get(key); + if (!result) { + return Error::propagate(std::move(result)); + } + return result.value() == status; + } + + Expected get_raw(const Key& key) + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + return get_raw_unchecked(key); + } + Expected get_raw(const Key& key) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + return get_raw_unchecked(key); + } + Expected set_raw(const Key& key, const NodeStatus status) + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + set_raw_unchecked(key, status); + return {}; + } + Expected remove_raw(const Key& key) + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + remove_raw_unchecked(key); + return {}; + } + + void clear() { m_index.clear(); } + bool empty() const { return m_index.empty(); } + size_t size() const { return m_index.size(); } + + const_iterator begin() const { return m_index.begin(); } + const_iterator end() const { return m_index.end(); } + const_iterator cbegin() const { return m_index.cbegin(); } + const_iterator cend() const { return m_index.cend(); } + +private: + NodeStatus* get_raw_unchecked(const Key& key) + { + const auto iterator = m_index.find(key); + return iterator == m_index.end() ? nullptr : &iterator->second; + } + const NodeStatus* get_raw_unchecked(const Key& key) const + { + const auto iterator = m_index.find(key); + return iterator == m_index.end() ? nullptr : &iterator->second; + } + void set_raw_unchecked(const Key& key, const NodeStatus status) { m_index[key] = status; } + void remove_raw_unchecked(const Key& key) { m_index.erase(key); } + + Expected update_parent_after_remove(const Key& key) + { + const auto parent = Traits::parent(key); + if (!parent.has_value()) { + return true; + } + if (!Traits::is_valid(parent.value())) { + return store::invalid_key_error(parent.value()); + } + + NodeStatus* parent_status = get_raw_unchecked(parent.value()); + DEBUG_ASSERT(parent_status != nullptr); + const auto siblings = Traits::children(parent.value()); + DEBUG_ASSERT(siblings.has_value()); + for (const Key& sibling : siblings.value()) { + if (sibling != key && get_raw_unchecked(sibling) != nullptr) { + return true; + } + } + + switch (*parent_status) { + case NodeStatus::Inner: + *parent_status = NodeStatus::Leaf; + break; + case NodeStatus::Virtual: + remove_raw_unchecked(parent.value()); + return update_parent_after_remove(parent.value()); + case NodeStatus::Leaf: + UNREACHABLE(); + } + return true; + } + + Container m_index; +}; + +} // namespace store diff --git a/src/terrainlib/store/IndexFormat.h b/src/terrainlib/store/IndexFormat.h new file mode 100644 index 00000000..71c04944 --- /dev/null +++ b/src/terrainlib/store/IndexFormat.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "Error.h" +#include "store/Index.h" +#include "store/path_layout.h" + +namespace store { + +template +struct IndexMetadata { + Index index; + std::string layout_id; + std::string payload_class; + std::string codec_selector; +}; + +template +struct IndexFormat { + std::string_view index_filename; + + Expected> (*read)(const std::filesystem::path& index_path); + Expected (*write)(const std::filesystem::path& index_path, const IndexMetadata& metadata); + std::optional> (*mapping_from_id)(std::string_view id); + path_layout::Mapping (*default_mapping)(); +}; + +} // namespace store diff --git a/src/terrainlib/store/IndexedStorage.h b/src/terrainlib/store/IndexedStorage.h new file mode 100644 index 00000000..6aac6138 --- /dev/null +++ b/src/terrainlib/store/IndexedStorage.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include "store/Storage.h" + +namespace store { + +template +class IndexedStorage : public Storage { +public: + using Base = Storage; + using typename Base::Persistence; + + explicit IndexedStorage(Storage storage) + : Base(std::move(storage)) + { + this->ensure_indexed(); + } + + IndexedStorage(RawStorage raw, Index index, Persistence persistence) + : Base(std::move(raw), std::move(index), std::move(persistence)) + { + } + + IndexedStorage(const IndexedStorage&) = delete; + IndexedStorage& operator=(const IndexedStorage&) = delete; + IndexedStorage(IndexedStorage&&) noexcept = default; + IndexedStorage& operator=(IndexedStorage&&) noexcept = default; + + const Index& index() const { return this->index_ref(); } +}; + +} // namespace store diff --git a/src/terrainlib/store/NodeStatus.h b/src/terrainlib/store/NodeStatus.h new file mode 100644 index 00000000..4cd17241 --- /dev/null +++ b/src/terrainlib/store/NodeStatus.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include +#include + +#include "log.h" +#include +#include + +namespace store { + +class NodeStatus { +public: + using Underlying = uint8_t; + enum Value : Underlying { + Leaf = 0, + Inner = 1, + Virtual = 2, + }; + + constexpr NodeStatus() = default; + constexpr NodeStatus(const Value value) + : m_value(value) + { + } + + constexpr operator Value() const { return m_value; } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatus other) const { return m_value == other.m_value; } + constexpr bool operator!=(const NodeStatus other) const { return !(*this == other); } + constexpr bool operator==(const Value other) const { return m_value == other; } + constexpr bool operator!=(const Value other) const { return m_value != other; } + + std::string to_string() const; + +private: + Value m_value; +}; + +} // namespace store + +template <> +struct fmt::formatter { + template + constexpr auto parse(ParseContext& context) + { + return context.begin(); + } + + template + auto format(const store::NodeStatus& status, FormatContext& context) const + { + switch (status) { + case store::NodeStatus::Leaf: + return fmt::format_to(context.out(), "Leaf"); + case store::NodeStatus::Inner: + return fmt::format_to(context.out(), "Inner"); + case store::NodeStatus::Virtual: + return fmt::format_to(context.out(), "Virtual"); + default: + UNREACHABLE(); + } + } +}; + +inline std::string store::NodeStatus::to_string() const { return fmt::format("{}", *this); } + +inline std::ostream& operator<<(std::ostream& stream, const store::NodeStatus& status) +{ + fmt::print(stream, "{}", status); + return stream; +} diff --git a/src/terrainlib/store/NodeStatusOrMissing.h b/src/terrainlib/store/NodeStatusOrMissing.h new file mode 100644 index 00000000..1b69fd28 --- /dev/null +++ b/src/terrainlib/store/NodeStatusOrMissing.h @@ -0,0 +1,127 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "log.h" +#include "store/NodeStatus.h" + +namespace store { + +class NodeStatusOrMissing { +public: + using Underlying = NodeStatus::Underlying; + enum Value : Underlying { + Leaf = 0, + Inner = 1, + Virtual = 2, + Missing = 3, + }; + + constexpr NodeStatusOrMissing() = default; + constexpr NodeStatusOrMissing(const Value value) + : m_value(value) + { + } + NodeStatusOrMissing(const NodeStatus status) + : m_value(from_status(status)) + { + } + NodeStatusOrMissing(const std::optional status) + : m_value(from_optional_status(status)) + { + } + + constexpr operator Value() const { return m_value; } + constexpr operator std::optional() const { return to_optional_status(); } + explicit operator bool() const = delete; + constexpr bool operator==(const NodeStatusOrMissing other) const { return m_value == other.m_value; } + constexpr bool operator!=(const NodeStatusOrMissing other) const { return !(*this == other); } + constexpr bool operator==(const Value other) const { return m_value == other; } + constexpr bool operator!=(const Value other) const { return m_value != other; } + + std::string to_string() const; + +private: + static constexpr NodeStatusOrMissing from_status(const NodeStatus status) noexcept { return static_cast(static_cast(status)); } + static constexpr NodeStatusOrMissing from_optional_status(const std::optional status) noexcept + { + return status.has_value() ? from_status(status.value()) : NodeStatusOrMissing(NodeStatusOrMissing::Missing); + } + constexpr std::optional to_optional_status() const noexcept + { + if (m_value == Missing) { + return std::nullopt; + } + return static_cast(m_value); + } + +public: + Value m_value; +}; + +namespace detail { + template + consteval bool verify_node_status_values(std::index_sequence) + { + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + return ((magic_enum::enum_underlying(status_values[Indices]) == magic_enum::enum_underlying(optional_values[Indices])) && ...); + } + + consteval bool verify_node_status_enums() + { + static_assert(std::is_same_v, magic_enum::underlying_type_t>); + constexpr auto status_values = magic_enum::enum_values(); + constexpr auto optional_values = magic_enum::enum_values(); + static_assert(optional_values.size() == status_values.size() + 1); + static_assert(verify_node_status_values(std::make_index_sequence())); + return true; + } + + static_assert(verify_node_status_enums()); +} // namespace detail + +} // namespace store + +template <> +struct fmt::formatter { + template + constexpr auto parse(ParseContext& context) + { + return context.begin(); + } + + template + auto format(const store::NodeStatusOrMissing& status, FormatContext& context) const + { + switch (status) { + case store::NodeStatusOrMissing::Leaf: + return fmt::format_to(context.out(), "Leaf"); + case store::NodeStatusOrMissing::Inner: + return fmt::format_to(context.out(), "Inner"); + case store::NodeStatusOrMissing::Virtual: + return fmt::format_to(context.out(), "Virtual"); + case store::NodeStatusOrMissing::Missing: + return fmt::format_to(context.out(), "Missing"); + default: + UNREACHABLE(); + } + } +}; + +inline std::string store::NodeStatusOrMissing::to_string() const { return fmt::format("{}", *this); } + +inline std::ostream& operator<<(std::ostream& stream, const store::NodeStatusOrMissing& status) +{ + fmt::print(stream, "{}", status); + return stream; +} diff --git a/src/terrainlib/store/RawStorage.h b/src/terrainlib/store/RawStorage.h new file mode 100644 index 00000000..22e90011 --- /dev/null +++ b/src/terrainlib/store/RawStorage.h @@ -0,0 +1,172 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "store/Codec.h" +#include "store/Traits.h" +#include "store/path_layout.h" + +namespace store { + +template +class RawStorage { +public: + using Key = typename Traits::Key; + using value_type = NodeData; + + RawStorage(path_layout::Resolver layout, std::unique_ptr> codec) + : m_layout(std::move(layout)) + , m_codec(std::move(codec)) + { + } + + RawStorage(const RawStorage&) = delete; + RawStorage& operator=(const RawStorage&) = delete; + RawStorage(RawStorage&&) noexcept = default; + RawStorage& operator=(RawStorage&&) noexcept = default; + + Expected load(const Key& key) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + return m_codec->read(m_layout.node_path(key)); + } + + Expected save(const Key& key, const NodeData& data) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + return m_codec->write(m_layout.node_path(key), data); + } + + Expected> paths(const Key& key) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + return m_codec->paths(m_layout.node_path(key)); + } + + Expected has(const Key& key) const + { + auto node_paths = paths(key); + if (!node_paths) { + return Error::propagate(std::move(node_paths)); + } + if (node_paths->empty()) { + return false; + } + for (const auto& path : node_paths.value()) { + std::error_code error; + const bool exists = std::filesystem::exists(path, error); + if (error) { + return Error::fail(Error::Code::Io, "check existence of", path, error); + } + if (!exists) { + return false; + } + } + return true; + } + + Expected remove(const Key& key) const + { + auto node_paths = paths(key); + if (!node_paths) { + return Error::propagate(std::move(node_paths)); + } + bool removed = false; + for (const auto& path : node_paths.value()) { + std::error_code error; + removed = std::filesystem::remove(path, error) || removed; + if (error) { + return Error::fail(Error::Code::Io, "remove", path, error); + } + } + return removed; + } + + Expected copy_from(const Key& key, const RawStorage& source) const + { + return copy_from(key, source, []() -> Expected { return {}; }); + } + + template + Expected copy_from(const Key& key, const RawStorage& source, BeforeModify&& before_modify) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + auto source_exists = source.has(key); + if (!source_exists) { + return Error::propagate(std::move(source_exists)); + } + if (!source_exists.value()) { + return Error::fail(Error::Code::NotFound, "source node " + Traits::key_to_string(key) + " is missing"); + } + + const std::filesystem::path probe("__codec_probe__/node"); + if (m_codec->paths(probe) != source.m_codec->paths(probe)) { + auto loaded = source.m_codec->read(source.m_layout.node_path(key)); + if (!loaded) { + return Error::propagate(std::move(loaded), "read source node while copying"); + } + const auto prepared = before_modify(); + if (!prepared) { + return prepared; + } + auto written = m_codec->write(m_layout.node_path(key), loaded.value()); + if (!written) { + return Error::propagate(std::move(written), "write target node while copying"); + } + return {}; + } + + const auto source_paths = source.m_codec->paths(source.m_layout.node_path(key)); + const auto target_paths = m_codec->paths(m_layout.node_path(key)); + if (source_paths.size() != target_paths.size()) { + return Error::fail(Error::Code::Internal, "matching codec probes produced different actual path counts"); + } + if (source_paths == target_paths) { + return {}; + } + const auto prepared = before_modify(); + if (!prepared) { + return prepared; + } + for (size_t index = 0; index < source_paths.size(); ++index) { + std::error_code error; + std::filesystem::remove(target_paths[index], error); + if (error) { + return Error::fail(Error::Code::Io, "remove", target_paths[index], error); + } + std::filesystem::create_directories(target_paths[index].parent_path(), error); + if (error) { + return Error::fail(Error::Code::Io, "create directories", target_paths[index].parent_path(), error); + } + std::filesystem::create_hard_link(source_paths[index], target_paths[index], error); + if (error) { + return Error::fail(Error::Code::Io, "create hard link", source_paths[index], target_paths[index], error); + } + } + return {}; + } + + const path_layout::Resolver& layout() const { return m_layout; } + const Codec& codec() const { return *m_codec; } + +private: + path_layout::Resolver m_layout; + std::unique_ptr> m_codec; +}; + +} // namespace store diff --git a/src/terrainlib/store/Storage.h b/src/terrainlib/store/Storage.h new file mode 100644 index 00000000..2beb046e --- /dev/null +++ b/src/terrainlib/store/Storage.h @@ -0,0 +1,293 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "log.h" +#include "store/IndexFormat.h" +#include "store/RawStorage.h" + +namespace store { + +struct StorageSettings { + bool allow_overwrite = false; +}; + +template +class Storage { +public: + using Key = typename Traits::Key; + using key_type = Key; + using value_type = NodeData; + + struct Persistence { + IndexFormat format; + std::filesystem::path index_path; + std::string layout_id; + std::string payload_class; + std::string codec_selector; + }; + + explicit Storage(RawStorage raw) + : m_raw(std::move(raw)) + { + } + + Storage(RawStorage raw, Index index, Persistence persistence, const bool dirty = false) + : m_raw(std::move(raw)) + , m_index(std::move(index)) + , m_persistence(std::move(persistence)) + , m_dirty(dirty) + { + } + + virtual ~Storage() { finalize_displaced_state(); } + + Storage(const Storage&) = delete; + Storage& operator=(const Storage&) = delete; + + Storage(Storage&& other) noexcept + : m_raw(std::move(other.m_raw)) + , m_index(std::move(other.m_index)) + , m_persistence(std::move(other.m_persistence)) + , m_settings(other.m_settings) + , m_dirty(std::exchange(other.m_dirty, false)) + { + other.m_index.reset(); + other.m_persistence.reset(); + } + + Storage& operator=(Storage&& other) noexcept + { + if (this == &other) { + return *this; + } + finalize_displaced_state(); + m_raw = std::move(other.m_raw); + m_index = std::move(other.m_index); + m_persistence = std::move(other.m_persistence); + m_settings = other.m_settings; + m_dirty = std::exchange(other.m_dirty, false); + other.m_index.reset(); + other.m_persistence.reset(); + return *this; + } + + Expected load(const Key& key) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + if (m_index.has_value()) { + auto status = m_index->get(key); + if (!status) { + return Error::propagate(std::move(status), "look up node " + Traits::key_to_string(key) + " in storage index before loading"); + } + if (!status->has_value() || status->value() == NodeStatus::Virtual) { + return Error::fail(Error::Code::NotFound, "node " + Traits::key_to_string(key) + " is not physically present in the index"); + } + } + return m_raw.load(key); + } + + Expected save(const Key& key, const NodeData& data) + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + auto exists = has(key); + if (!exists) { + return Error::propagate(std::move(exists), "check whether node " + Traits::key_to_string(key) + " exists before saving"); + } + if (exists.value() && !m_settings.allow_overwrite) { + const auto node_paths = m_raw.paths(key).value(); + return Error::fail(Error::Code::AlreadyExists, "save node to", node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front()); + } + + auto result = m_raw.save(key, data); + if (!result) { + return result; + } + if (m_index.has_value()) { + auto added = m_index->add(key); + if (!added) { + return Error::propagate(std::move(added), "add saved node " + Traits::key_to_string(key) + " to storage index"); + } + m_dirty = m_dirty || added.value(); + } + return {}; + } + + Expected copy_from(const Key& key, const Storage& source) + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + auto exists = has(key); + if (!exists) { + return Error::propagate(std::move(exists), "check whether target node " + Traits::key_to_string(key) + " exists before copying"); + } + if (exists.value() && !m_settings.allow_overwrite) { + const auto node_paths = m_raw.paths(key).value(); + return Error::fail(Error::Code::AlreadyExists, "copy node to", node_paths.empty() ? m_raw.layout().node_path(key) : node_paths.front()); + } + const auto prepare_target = [&]() -> Expected { + if (!exists.value() || !m_index.has_value()) { + return {}; + } + auto removed = m_index->remove(key); + if (!removed) { + return Error::propagate(std::move(removed), "remove overwritten node " + Traits::key_to_string(key) + " from storage index"); + } + m_dirty = m_dirty || removed.value(); + return {}; + }; + + auto result = m_raw.copy_from(key, source.m_raw, prepare_target); + if (!result) { + return result; + } + if (m_index.has_value()) { + auto added = m_index->add(key); + if (!added) { + return Error::propagate(std::move(added), "add copied node " + Traits::key_to_string(key) + " to storage index"); + } + m_dirty = m_dirty || added.value(); + } + return {}; + } + + Expected remove(const Key& key) + { + auto removed = m_raw.remove(key); + if (!removed) { + return removed; + } + if (m_index.has_value()) { + auto index_removed = m_index->remove(key); + if (!index_removed) { + return Error::propagate(std::move(index_removed), "remove node " + Traits::key_to_string(key) + " from storage index"); + } + m_dirty = m_dirty || index_removed.value(); + } + return removed.value(); + } + + Expected has(const Key& key) const + { + if (!Traits::is_valid(key)) { + return store::invalid_key_error(key); + } + if (!m_index.has_value()) { + return m_raw.has(key); + } + auto status = m_index->get(key); + if (!status) { + return Error::propagate(std::move(status), "look up node " + Traits::key_to_string(key) + " in storage index"); + } + return status->has_value() && status->value() != NodeStatus::Virtual; + } + + Expected> paths(const Key& key) const { return m_raw.paths(key); } + + Expected path_for(const Key& key) const + { + auto node_paths = paths(key); + if (!node_paths) { + return Error::propagate(std::move(node_paths)); + } + return node_paths->empty() ? m_raw.layout().node_path(key) : node_paths->front(); + } + + const std::filesystem::path& base_path() const { return m_raw.layout().base_path(); } + const path_layout::Resolver& layout() const { return m_raw.layout(); } + const Codec& codec() const { return m_raw.codec(); } + std::optional codec_selector() const + { + if (!m_persistence.has_value()) { + return std::nullopt; + } + return m_persistence->codec_selector; + } + + bool is_indexed() const { return m_index.has_value(); } + void ensure_indexed() + { + if (!m_index.has_value()) { + m_index.emplace(); + m_dirty = true; + } + } + + std::optional>> index() const + { + if (!m_index.has_value()) { + return std::nullopt; + } + return std::cref(m_index.value()); + } + + Expected save_index() const + { + if (!m_dirty) { + return {}; + } + if (!m_index.has_value() || !m_persistence.has_value()) { + return Error::fail(Error::Code::Internal, "indexed storage has no persistence configuration"); + } + const IndexMetadata metadata { + m_index.value(), + m_persistence->layout_id, + m_persistence->payload_class, + m_persistence->codec_selector, + }; + auto result = m_persistence->format.write(m_persistence->index_path, metadata); + if (result) { + m_dirty = false; + } + return result; + } + + Expected save_or_create_index() + { + if (!m_index.has_value()) { + m_index.emplace(); + m_dirty = true; + } + return save_index(); + } + + const StorageSettings& settings() const { return m_settings; } + StorageSettings& settings() { return m_settings; } + +protected: + Index& index_mut() { return m_index.value(); } + const Index& index_ref() const { return m_index.value(); } + bool is_index_dirty() const { return m_dirty; } + void set_index_dirty(const bool dirty = true) const { m_dirty = dirty; } + +private: + void finalize_displaced_state() noexcept + { + if (!m_dirty || !m_index.has_value()) { + return; + } + const auto result = save_index(); + if (!result) { + LOG_ERROR("Failed to automatically save index: {}", result.error().to_string()); + } + } + + RawStorage m_raw; + std::optional> m_index; + std::optional m_persistence = std::nullopt; + StorageSettings m_settings; + mutable bool m_dirty = false; +}; + +} // namespace store diff --git a/src/terrainlib/store/ThreadSafeStorage.h b/src/terrainlib/store/ThreadSafeStorage.h new file mode 100644 index 00000000..13c30002 --- /dev/null +++ b/src/terrainlib/store/ThreadSafeStorage.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace store { + +template +class ThreadSafeStorage { + static_assert(!std::is_const_v, "ThreadSafeStorage requires a non-const Storage"); + +public: + using value_type = typename Storage::value_type; + using key_type = typename Storage::key_type; + + explicit ThreadSafeStorage(Storage&& storage) + : m_storage(std::move(storage)) + { + } + + ThreadSafeStorage(const ThreadSafeStorage&) = delete; + ThreadSafeStorage& operator=(const ThreadSafeStorage&) = delete; + ThreadSafeStorage(ThreadSafeStorage&&) = delete; + ThreadSafeStorage& operator=(ThreadSafeStorage&&) = delete; + + Storage release() && { return std::move(m_storage); } + + auto load(const key_type& key) const + { + std::shared_lock lock(m_mutex); + return m_storage.load(key); + } + + auto has(const key_type& key) const + { + std::shared_lock lock(m_mutex); + return m_storage.has(key); + } + + std::filesystem::path base_path() const noexcept { return m_storage.base_path(); } + + auto save(const key_type& key, const value_type& value) const + { + std::unique_lock lock(m_mutex); + return m_storage.save(key, value); + } + + auto save_or_create_index() const + { + std::unique_lock lock(m_mutex); + return m_storage.save_or_create_index(); + } + +private: + mutable Storage m_storage; + mutable std::shared_mutex m_mutex; +}; + +} // namespace store diff --git a/src/terrainlib/store/Traits.h b/src/terrainlib/store/Traits.h new file mode 100644 index 00000000..00fc23ae --- /dev/null +++ b/src/terrainlib/store/Traits.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + +#include "Error.h" + +namespace store { + +template +concept HierarchyTraits = requires(typename Traits::Key key) { + typename Traits::Key; + typename Traits::Hasher; + { Traits::root() } -> std::same_as; + Traits::parent(key); + Traits::children(key); + { Traits::is_valid(key) } -> std::same_as; + { Traits::key_to_string(key) } -> std::same_as; +}; + +template +std::unexpected invalid_key_error(const typename Traits::Key& key, const std::source_location location = std::source_location::current()) +{ + return Error::fail(Error::Code::InvalidInput, "invalid hierarchy key " + Traits::key_to_string(key), location); +} + +} // namespace store diff --git a/src/terrainlib/store/cache/Dummy.h b/src/terrainlib/store/cache/Dummy.h new file mode 100644 index 00000000..ab7c2990 --- /dev/null +++ b/src/terrainlib/store/cache/Dummy.h @@ -0,0 +1,17 @@ +#pragma once + +#include "store/cache/Interface.h" + +namespace store::cache { + +template +class Dummy final : public Interface { +public: + using Key = typename Traits::Key; + std::optional get(const Key&) noexcept override { return std::nullopt; } + bool put(const Key&, const NodeData&) noexcept override { return false; } + bool remove(const Key&) noexcept override { return false; } + bool contains(const Key&) const noexcept override { return false; } +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/cache/Interface.h b/src/terrainlib/store/cache/Interface.h new file mode 100644 index 00000000..7cc6ce73 --- /dev/null +++ b/src/terrainlib/store/cache/Interface.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "store/Traits.h" + +namespace store::cache { + +template +class Interface { +public: + using Key = typename Traits::Key; + virtual ~Interface() = default; + + virtual std::optional get(const Key& key) = 0; + virtual bool put(const Key& key, const NodeData& value) = 0; + virtual bool remove(const Key& key) noexcept = 0; + virtual bool contains(const Key& key) const noexcept = 0; +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/cache/Lru.h b/src/terrainlib/store/cache/Lru.h new file mode 100644 index 00000000..60b33d76 --- /dev/null +++ b/src/terrainlib/store/cache/Lru.h @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include + +#include "store/cache/Interface.h" + +namespace store::cache { + +template +class Lru final : public Interface { +public: + using Key = typename Traits::Key; + + explicit Lru(const size_t capacity) + : m_capacity(capacity) + { + } + + std::optional get(const Key& key) override + { + const auto iterator = m_values.find(key); + if (iterator == m_values.end()) { + return std::nullopt; + } + m_usage.splice(m_usage.begin(), m_usage, iterator->second.second); + return iterator->second.first; + } + + bool put(const Key& key, const NodeData& value) override + { + const auto iterator = m_values.find(key); + if (iterator != m_values.end()) { + iterator->second.first = value; + m_usage.splice(m_usage.begin(), m_usage, iterator->second.second); + return false; + } + if (m_capacity == 0) { + return false; + } + if (m_values.size() == m_capacity) { + m_values.erase(m_usage.back()); + m_usage.pop_back(); + } + m_usage.push_front(key); + m_values.emplace(key, + std::pair::iterator> { + value, + m_usage.begin(), + }); + return true; + } + + bool remove(const Key& key) noexcept override + { + const auto iterator = m_values.find(key); + if (iterator == m_values.end()) { + return false; + } + m_usage.erase(iterator->second.second); + m_values.erase(iterator); + return true; + } + + bool contains(const Key& key) const noexcept override { return m_values.contains(key); } + +private: + size_t m_capacity; + std::list m_usage; + std::unordered_map::iterator>, typename Traits::Hasher> m_values; +}; + +} // namespace store::cache diff --git a/src/terrainlib/store/open.h b/src/terrainlib/store/open.h new file mode 100644 index 00000000..5ac295c9 --- /dev/null +++ b/src/terrainlib/store/open.h @@ -0,0 +1,82 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "store/IndexedStorage.h" + +namespace store { + +template +Expected> open_index( + const std::filesystem::path& index_path, const IndexFormat format, const std::string_view expected_payload_class, CodecResolver&& resolve_codec) +{ + using Key = typename Traits::Key; + auto metadata_result = format.read(index_path); + if (!metadata_result) { + return Error::propagate(std::move(metadata_result), "read storage index \"" + index_path.string() + "\""); + } + IndexMetadata metadata = std::move(metadata_result.value()); + if (metadata.payload_class != expected_payload_class) { + return Error::fail(Error::Code::Unsupported, "unexpected payload class: " + metadata.payload_class); + } + for (const auto& [key, status] : metadata.index) { + static_cast(status); + if (!Traits::is_valid(key)) { + return Error::fail(Error::Code::CorruptData, "index contains invalid hierarchy key " + Traits::key_to_string(key)); + } + } + + const auto mapping = format.mapping_from_id(metadata.layout_id); + if (!mapping.has_value()) { + return Error::fail(Error::Code::Unsupported, "unknown storage layout: " + metadata.layout_id); + } + auto codec = resolve_codec(metadata.codec_selector); + if (!codec) { + return Error::propagate(std::move(codec), "resolve storage codec \"" + metadata.codec_selector + "\""); + } + + const std::filesystem::path base_path = index_path.parent_path(); + using Storage = store::Storage; + return IndexedStorage(RawStorage(path_layout::Resolver(base_path, mapping.value()), std::move(codec.value())), + std::move(metadata.index), + typename Storage::Persistence { + format, + index_path, + std::move(metadata.layout_id), + std::move(metadata.payload_class), + std::move(metadata.codec_selector), + }); +} + +template +Expected> make_storage(const std::filesystem::path& base_path, + const IndexFormat format, + const path_layout::Mapping mapping, + std::string payload_class, + std::string codec_selector, + std::unique_ptr> codec) +{ + using Key = typename Traits::Key; + std::error_code error; + std::filesystem::create_directories(base_path, error); + if (error) { + return Error::fail(Error::Code::Io, "create directories", base_path, error); + } + + typename Storage::Persistence persistence { + format, + base_path / format.index_filename, + std::string(mapping.id), + std::move(payload_class), + std::move(codec_selector), + }; + RawStorage raw(path_layout::Resolver(base_path, mapping), std::move(codec)); + return Storage(std::move(raw), Index {}, std::move(persistence), true); +} + +} // namespace store diff --git a/src/terrainlib/store/path_layout.h b/src/terrainlib/store/path_layout.h new file mode 100644 index 00000000..d08423d7 --- /dev/null +++ b/src/terrainlib/store/path_layout.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +namespace store::path_layout { + +template +struct Mapping { + std::string_view id; + std::filesystem::path (*key_to_node_path)(const Key&); + std::optional (*node_path_to_key)(const std::filesystem::path&); + + bool operator==(const Mapping&) const = default; +}; + +template +class Resolver { +public: + Resolver(std::filesystem::path base_path, Mapping mapping) + : m_base_path(std::move(base_path)) + , m_mapping(mapping) + { + } + + std::filesystem::path node_path(const Key& key) const { return m_base_path / m_mapping.key_to_node_path(key); } + + std::optional key_from_node_path(const std::filesystem::path& node_path) const + { + std::error_code error; + const std::filesystem::path relative = std::filesystem::relative(node_path, m_base_path, error); + if (error || relative.empty() || relative.is_absolute()) { + return std::nullopt; + } + return m_mapping.node_path_to_key(relative); + } + + const std::filesystem::path& base_path() const { return m_base_path; } + Mapping mapping() const { return m_mapping; } + +private: + std::filesystem::path m_base_path; + Mapping m_mapping; +}; + +} // namespace store::path_layout diff --git a/src/terrainlib/store/traverse.h b/src/terrainlib/store/traverse.h new file mode 100644 index 00000000..b491d06b --- /dev/null +++ b/src/terrainlib/store/traverse.h @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include +#include + +#include "store/Index.h" + +namespace store { + +enum class TraversalOrder { + DepthFirst, + BreadthFirst, +}; + +struct AlwaysRefine { + template + constexpr bool operator()(const Key&) const + { + return true; + } +}; + +template +Expected traverse(const Index& index, + VisitFn&& visit, + RefineFn&& refine = {}, + const typename Traits::Key& root = Traits::root(), + const TraversalOrder order = TraversalOrder::DepthFirst) +{ + using Key = typename Traits::Key; + if (!Traits::is_valid(root)) { + return store::invalid_key_error(root); + } + auto root_status = index.get(root); + if (!root_status) { + return Error::propagate(std::move(root_status), "read traversal root status for node " + Traits::key_to_string(root)); + } + if (!root_status.value().has_value()) { + return {}; + } + + if (order == TraversalOrder::DepthFirst) { + std::function(const Key&)> depth_first; + depth_first = [&](const Key& current) -> Expected { + auto status = index.get(current); + if (!status) { + return Error::propagate(std::move(status), "read node status during depth-first traversal for " + Traits::key_to_string(current)); + } + if (!status.value().has_value()) { + return {}; + } + visit(current, status.value().value()); + + if (refine(current)) { + const auto children = Traits::children(current); + if (children.has_value()) { + for (const Key& child : children.value()) { + auto result = depth_first(child); + if (!result) { + return result; + } + } + } + } + return {}; + }; + return depth_first(root); + } + + std::queue queue; + queue.push(root); + while (!queue.empty()) { + const Key current = queue.front(); + queue.pop(); + + auto status = index.get(current); + if (!status) { + return Error::propagate(std::move(status), "read node status during breadth-first traversal for " + Traits::key_to_string(current)); + } + if (!status.value().has_value()) { + continue; + } + visit(current, status.value().value()); + + if (refine(current)) { + const auto children = Traits::children(current); + if (children.has_value()) { + for (const Key& child : children.value()) { + queue.push(child); + } + } + } + } + return {}; +} + +} // namespace store diff --git a/src/terrainlib/tile_path.h b/src/terrainlib/tile_path.h index 8f866d80..e669057e 100644 --- a/src/terrainlib/tile_path.h +++ b/src/terrainlib/tile_path.h @@ -6,9 +6,7 @@ #include [[nodiscard]] inline std::filesystem::path google_tile_path( - const std::filesystem::path& base_path, - const radix::tile::Id& tile_id, - const std::string& extension) + const std::filesystem::path& base_path, const radix::tile::Id& tile_id, const std::string& extension) { return base_path / std::to_string(tile_id.zoom_level) / std::to_string(tile_id.coords.x) / (std::to_string(tile_id.coords.y) + extension); } diff --git a/src/tile_builder/CMakeLists.txt b/src/tile_builder/CMakeLists.txt index 576df41c..a05e9edb 100644 --- a/src/tile_builder/CMakeLists.txt +++ b/src/tile_builder/CMakeLists.txt @@ -1,13 +1,16 @@ add_library(tilebuilderlib - alpine_raster.cpp - DatasetReader.cpp - image_writer.cpp - ParallelTileGenerator.cpp - ParallelTiler.cpp - TileHeightsGenerator.cpp - Tiler.cpp - TopDownTiler.cpp + DatasetReader.h DatasetReader.cpp + Exception.h + ParallelTileGenerator.h ParallelTileGenerator.cpp + ParallelTiler.h ParallelTiler.cpp + TileHeightsGenerator.h TileHeightsGenerator.cpp + Tiler.h Tiler.cpp + TopDownTiler.h TopDownTiler.cpp + alpine_raster.h alpine_raster.cpp + depth_first_tile_traverser.h + image_writer.h image_writer.cpp ) + target_include_directories(tilebuilderlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(tilebuilderlib PUBLIC terrainlib ZLIB::ZLIB GDAL::GDAL fmt) diff --git a/src/tile_builder/alpine_raster.cpp b/src/tile_builder/alpine_raster.cpp index c5d8a6b0..7e4b1a0c 100644 --- a/src/tile_builder/alpine_raster.cpp +++ b/src/tile_builder/alpine_raster.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -38,5 +39,8 @@ ParallelTileGenerator alpine_raster::make_generator(const std::string& input_dat void alpine_raster::TileWriter::write(const std::string& file_path, const radix::tile::Descriptor&, const radix::Raster& heights) const { - image::saveImageAsPng(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); + auto result = image::save_image_as_png(radix::raster::transform(heights, radix::height_encoding::to_rgb), file_path); + if (!result) { + throw std::runtime_error(result.error().to_string()); + } } diff --git a/src/tile_builder/image_writer.cpp b/src/tile_builder/image_writer.cpp index 725cdfae..c0131610 100644 --- a/src/tile_builder/image_writer.cpp +++ b/src/tile_builder/image_writer.cpp @@ -20,27 +20,32 @@ #include "image_writer.h" +#include "io/conversion.h" + #include -#include -void image::saveImageAsPng(const radix::Raster& input_image, const std::string& path) +Expected image::save_image_as_png(const radix::Raster& input_image, const std::string& path) { - const int width = static_cast(input_image.width()); - const int height = static_cast(input_image.height()); - - cv::Mat image(height, width, CV_8UC3); + auto converted = io::conversion::to_mat(input_image); + if (!converted) { + return Error::propagate(std::move(converted), "convert raster for PNG output"); + } - for (int row = 0; row < height; ++row) { - for (int column = 0; column < width; ++column) { - const auto& pixel = input_image.pixel({ static_cast(column), static_cast(height - row - 1) }); - image.at(row, column) = cv::Vec3b(pixel.z, pixel.y, pixel.x); - } + cv::Mat flipped; + cv::Mat image; + try { + cv::flip(*converted, flipped, 0); + cv::cvtColor(flipped, image, cv::COLOR_RGB2BGR); + } catch (const cv::Exception& error) { + return Error::fail(Error::Code::Internal, "prepare PNG image: " + error.msg); } try { - if (!cv::imwrite(path, image)) - throw std::runtime_error("Failed to write PNG image to " + path); + if (!cv::imwrite(path, image)) { + return Error::fail(Error::Code::Io, "write PNG image to \"" + path + "\""); + } } catch (const cv::Exception& error) { - throw std::runtime_error("Failed to write PNG image to " + path + ": " + error.what()); + return Error::fail(Error::Code::Io, "write PNG image to \"" + path + "\": " + error.msg); } + return {}; } diff --git a/src/tile_builder/image_writer.h b/src/tile_builder/image_writer.h index db8ff8f9..9c72598f 100644 --- a/src/tile_builder/image_writer.h +++ b/src/tile_builder/image_writer.h @@ -23,28 +23,31 @@ #include #include #include -#include #include #include #include +#include "Error.h" + namespace image { -void saveImageAsPng(const radix::Raster& image, const std::string& path); +Expected save_image_as_png(const radix::Raster& image, const std::string& path); template -void debugOut(const radix::Raster& image, const std::string& path) +Expected debug_out(const radix::Raster& image, const std::string& path) { - if (image.buffer().empty()) - throw std::invalid_argument("Can't write an empty raster to " + path); + if (image.buffer().empty()) { + return Error::fail(Error::Code::InvalidInput, "cannot write an empty debug raster to \"" + path + "\""); + } const auto [min, max] = std::ranges::minmax(image); const auto range = float(max) - float(min); - saveImageAsPng(radix::raster::transform(image, [min, range](const auto value) { - const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); - return glm::u8vec3(intensity); - }), + return save_image_as_png(radix::raster::transform(image, + [min, range](const auto value) { + const auto intensity = range == 0.F ? std::uint8_t(0) : std::uint8_t(255.F * (float(value) - float(min)) / range); + return glm::u8vec3(intensity); + }), path); } diff --git a/src/tile_downloader/CMakeLists.txt b/src/tile_downloader/CMakeLists.txt index c1aebead..ac21b220 100644 --- a/src/tile_downloader/CMakeLists.txt +++ b/src/tile_downloader/CMakeLists.txt @@ -1,5 +1,11 @@ add_executable(tile-downloader + HttpClient.h + TileDownloader.h + TileLogger.h + TileUrlBuilder.h + cli.h cli.cpp main.cpp - cli.cpp + write_file.h ) + target_link_libraries(tile-downloader PRIVATE terrainlib CLI11::CLI11 CURL::libcurl) diff --git a/src/tile_downloader/HttpClient.h b/src/tile_downloader/HttpClient.h index d5ef1ca4..6eaeeb45 100644 --- a/src/tile_downloader/HttpClient.h +++ b/src/tile_downloader/HttpClient.h @@ -40,8 +40,8 @@ class HttpClient { HttpResponse get(const std::string &url, const ProgressFn &on_progress = {}) const { HttpResponse response; - WriteContext write_context{&response.body, &this->_write, {}}; - ProgressContext progress_context{&on_progress, {}}; + WriteContext write_context = {&response.body, &this->_write, {}}; + ProgressContext progress_context = {&on_progress, {}}; curl_easy_setopt(this->_curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(this->_curl, CURLOPT_WRITEFUNCTION, write_cb); diff --git a/src/tile_downloader/TileUrlBuilder.h b/src/tile_downloader/TileUrlBuilder.h index e28f3002..6e53cf84 100644 --- a/src/tile_downloader/TileUrlBuilder.h +++ b/src/tile_downloader/TileUrlBuilder.h @@ -10,12 +10,12 @@ enum class TileDownloadProvider { Basemap, - Gataki + Gataki, }; enum class TileYDirection { Down, - Up + Up, }; struct TileProviderConfig { diff --git a/src/tile_downloader/cli.h b/src/tile_downloader/cli.h index 8e2380f8..6e03207c 100644 --- a/src/tile_downloader/cli.h +++ b/src/tile_downloader/cli.h @@ -11,8 +11,8 @@ namespace cli { struct Args { - std::optional provider; - std::optional url_pattern; + std::optional provider = std::nullopt; + std::optional url_pattern = std::nullopt; unsigned int zoom; unsigned int x; unsigned int y; diff --git a/src/tile_downloader/write_file.h b/src/tile_downloader/write_file.h index ab2d07f2..b74e7ebe 100644 --- a/src/tile_downloader/write_file.h +++ b/src/tile_downloader/write_file.h @@ -9,7 +9,7 @@ namespace tile_downloader_detail { -inline void write_file_checked_direct(const std::filesystem::path &path, const std::vector &data) +inline void write_file_checked_direct(const std::filesystem::path& path, const std::vector& data) { std::ofstream output(path, std::ios::binary); if (!output) { @@ -27,23 +27,23 @@ inline void write_file_checked_direct(const std::filesystem::path &path, const s } } -} +} // namespace tile_downloader_detail -[[nodiscard]] inline std::filesystem::path partial_tile_path(const std::filesystem::path &path) +[[nodiscard]] inline std::filesystem::path partial_tile_path(const std::filesystem::path& path) { auto partial_path = path; partial_path += ".part"; return partial_path; } -[[nodiscard]] inline std::filesystem::path children_pending_tile_path(const std::filesystem::path &path) +[[nodiscard]] inline std::filesystem::path children_pending_tile_path(const std::filesystem::path& path) { auto pending_path = path; pending_path += ".children-pending"; return pending_path; } -inline void write_file_children_pending(const std::filesystem::path &path, const std::vector &data) +inline void write_file_children_pending(const std::filesystem::path& path, const std::vector& data) { const auto partial_path = partial_tile_path(path); const auto pending_path = children_pending_tile_path(path); @@ -58,7 +58,7 @@ inline void write_file_children_pending(const std::filesystem::path &path, const } } -inline void mark_tile_children_complete(const std::filesystem::path &path) +inline void mark_tile_children_complete(const std::filesystem::path& path) { const auto pending_path = children_pending_tile_path(path); std::filesystem::rename(pending_path, path); diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 2c1b83ec..f23bca02 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -15,23 +15,31 @@ endfunction() add_executable(unittests_terrainlib catch2_helpers.h opencv_helpers.h + temporary_directory.h terrainlib/convert.cpp terrainlib/cow.cpp terrainlib/dataset.cpp terrainlib/enumerate.cpp terrainlib/envelope.cpp + terrainlib/error.cpp terrainlib/fixed_vector.cpp terrainlib/geometry_frames.cpp terrainlib/grid.cpp terrainlib/hash_utils.cpp terrainlib/hybrid_index_pair_map.cpp terrainlib/hybrid_vector.cpp + terrainlib/io_conversion.cpp terrainlib/offset_table.cpp terrainlib/segmented_buffer.cpp + terrainlib/sf_validate_index.cpp terrainlib/stamp_set.cpp - terrainlib/index_map.cpp - terrainlib/index_traverse.cpp + terrainlib/store_compatibility.cpp + terrainlib/store_codec.cpp + terrainlib/store_index.cpp + terrainlib/store_layout.cpp + terrainlib/store_storage.cpp + terrainlib/store_traverse.cpp terrainlib/mesh_boundary.cpp terrainlib/mesh_bounds.cpp terrainlib/mesh_cleanup.cpp @@ -91,6 +99,16 @@ if(TARGET sfbuilderlib) ) target_link_libraries(unittests_sfbuilder PRIVATE sfbuilderlib Catch2::Catch2WithMain) atb_configure_test(unittests_sfbuilder) + + add_executable(unittests_sfbuilder_finalization + catch2_helpers.h + temporary_directory.h + sf_builder/storage_finalization.cpp + ) + target_link_libraries( + unittests_sfbuilder_finalization + PRIVATE sfbuilderlib Catch2::Catch2WithMain) + atb_configure_test(unittests_sfbuilder_finalization) endif() if(TARGET tile-downloader) @@ -110,6 +128,7 @@ endif() if(TARGET dagbuilderlib) add_executable(unittests_dagbuilder catch2_helpers.h + temporary_directory.h dag_builder/bake_cluster_texture.cpp dag_builder/correspondence.cpp dag_builder/reprojection.cpp @@ -121,6 +140,7 @@ if(TARGET dagbuilderlib) dag_builder/fixed_point.cpp dag_builder/texture_sizing.cpp dag_builder/texture_reuse.cpp + dag_builder/storage_compatibility.cpp ) target_link_libraries(unittests_dagbuilder PUBLIC dagbuilderlib Catch2::Catch2WithMain) atb_configure_test(unittests_dagbuilder) @@ -129,8 +149,10 @@ endif() if(TARGET sfmergerlib) add_executable(unittests_sfmerger catch2_helpers.h + temporary_directory.h sf_merger/mask.cpp sf_merger/sphere_projector.cpp + sf_merger/storage_boundaries.cpp ) target_link_libraries(unittests_sfmerger PRIVATE sfmergerlib Catch2::Catch2WithMain) if(TARGET sf-builder AND TARGET sf-merger) diff --git a/unittests/dag_builder/storage_compatibility.cpp b/unittests/dag_builder/storage_compatibility.cpp new file mode 100644 index 00000000..4762b5ff --- /dev/null +++ b/unittests/dag_builder/storage_compatibility.cpp @@ -0,0 +1,219 @@ +#include +#include + +#include + +#include "../temporary_directory.h" +#include "build.h" +#include "codec.h" +#include "dag_node.h" +#include "mesh/SimpleMesh.h" +#include "mesh/storage.h" +#include "storage.h" +#include "store/ThreadSafeStorage.h" + +namespace { + +using test::TemporaryDirectory; + +dag::ClusterBatch sample_batch() +{ + dag::ClusterBatch batch; + batch.metadata.group_assignment = { 0 }; + batch.metadata.groups = { { + .children = { { octree::Id::root().child(4).value(), 3 } }, + .error = 12.5, + .bounds = {}, + } }; + batch.clustering.positions = { + { 0.0, 0.0, 0.0 }, + { 1.0, 0.0, 0.0 }, + { 0.0, 1.0, 0.0 }, + }; + batch.clustering.clusters = { { + .id = 17, + .vertex_indices = { 0, 1, 2 }, + .local_triangles = { { 0, 1, 2 } }, + .uvs = {}, + .texture_id = std::nullopt, + .absolute_error = 0.0, + } }; + return batch; +} + +mesh::Simple sample_mesh() { return mesh::Simple({ { 0, 1, 2 } }, { { 0.0, 0.0, 0.0 }, { 1.0, 0.0, 0.0 }, { 0.0, 1.0, 0.0 } }); } + +} // namespace + +TEST_CASE("runtime DAG envelope codec is reentrant") +{ + const dag::ClusterBatch batch = sample_batch(); + + TemporaryDirectory output; + dag::codec::ClusterBatch codec; + std::vector> operations; + for (int index = 0; index < 8; ++index) { + operations.push_back(std::async(std::launch::async, [&codec, &batch, &output, index] { + const std::filesystem::path path(output.path() / std::to_string(index) / "node"); + if (!codec.write(path, batch).has_value()) { + return false; + } + const auto loaded = codec.read(path); + return loaded.has_value() && loaded->metadata.group_assignment == batch.metadata.group_assignment; + })); + } + for (auto& operation : operations) { + REQUIRE(operation.get()); + } +} + +TEST_CASE("DAG codec rejects invalid batches and inconsistent files") +{ + TemporaryDirectory output; + dag::codec::ClusterBatch codec; + const std::filesystem::path node_path(output.path() / "node"); + const auto node_paths = codec.paths(node_path); + + SECTION("write") + { + auto batch = sample_batch(); + batch.metadata.group_assignment.clear(); + + const auto result = codec.write(node_path, batch); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); + CHECK_FALSE(std::filesystem::exists(node_paths[0])); + CHECK_FALSE(std::filesystem::exists(node_paths[1])); + } + + SECTION("write invalid metadata") + { + auto batch = sample_batch(); + batch.metadata.group_assignment = { 1 }; + + const auto result = codec.write(node_path, batch); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); + CHECK_FALSE(std::filesystem::exists(node_paths[0])); + CHECK_FALSE(std::filesystem::exists(node_paths[1])); + } + + SECTION("read") + { + REQUIRE(codec.write(node_path, sample_batch()).has_value()); + const std::filesystem::path empty_node_path(output.path() / "empty-node"); + const auto empty_node_paths = codec.paths(empty_node_path); + REQUIRE(codec.write(empty_node_path, dag::ClusterBatch {}).has_value()); + REQUIRE(std::filesystem::copy_file(empty_node_paths[1], node_paths[1], std::filesystem::copy_options::overwrite_existing)); + + const auto result = codec.read(node_path); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::CorruptData); + } +} + +TEST_CASE("DAG resolvers expose writable batches and read-only metadata", "[store][open]") +{ + const auto batch_codec = dag::codec::cluster_batch_from_extension(".dag"); + REQUIRE(batch_codec.has_value()); + CHECK(batch_codec.value()->paths("node") == std::vector { "node.dag", "node.dagmeta" }); + + const auto metadata_codec = dag::codec::metadata_from_extension(".dag"); + REQUIRE(metadata_codec.has_value()); + CHECK(metadata_codec.value()->paths("node") == std::vector { "node.dagmeta" }); + const auto write_result = metadata_codec.value()->write("node", dag::NodeMetadata {}); + REQUIRE_FALSE(write_result.has_value()); + CHECK(write_result.error().code() == Error::Code::Unsupported); + + const auto unknown = dag::codec::cluster_batch_from_extension(".unknown"); + REQUIRE_FALSE(unknown.has_value()); + CHECK(unknown.error().code() == Error::Code::Unsupported); +} + +TEST_CASE("DAG indexed storage survives ThreadSafeStorage move and release", "[store][storage]") +{ + const dag::ClusterBatch batch = sample_batch(); + + TemporaryDirectory output; + auto storage_result = dag::storage::open_folder_indexed(output.path()); + REQUIRE(storage_result.has_value()); + store::ThreadSafeStorage synchronized(std::move(storage_result.value())); + REQUIRE(synchronized.save(octree::Id::root(), batch).has_value()); + + auto released = std::move(synchronized).release(); + REQUIRE(released.save_index().has_value()); + CHECK(std::filesystem::is_regular_file(output.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(output.path() / "octree.storemeta")); + + auto reopened_result = dag::storage::open_folder_indexed(output.path()); + REQUIRE(reopened_result.has_value()); + CHECK(reopened_result->has(octree::Id::root()).value()); + CHECK(reopened_result->load(octree::Id::root()).has_value()); + const auto full_paths = reopened_result->paths(octree::Id::root()); + REQUIRE(full_paths.has_value()); + REQUIRE(full_paths->size() == 2); + + auto metadata_result = dag::storage::open_metadata_indexed(output.path()); + REQUIRE(metadata_result.has_value()); + REQUIRE(std::filesystem::remove(full_paths->front())); + const auto metadata = metadata_result->load(octree::Id::root()); + REQUIRE(metadata.has_value()); + CHECK(metadata->group_assignment == batch.metadata.group_assignment); + CHECK(metadata->groups.front().error == batch.metadata.groups.front().error); +} + +TEST_CASE("DAG storage hard-links both files for the same format", "[store][copy]") +{ + TemporaryDirectory source_directory; + TemporaryDirectory target_directory; + auto source_result = dag::storage::open_folder_indexed(source_directory.path()); + auto target_result = dag::storage::open_folder_indexed(target_directory.path()); + REQUIRE(source_result.has_value()); + REQUIRE(target_result.has_value()); + auto source = std::move(*source_result); + auto target = std::move(*target_result); + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, sample_batch()).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + + const auto source_paths = source.paths(root); + const auto target_paths = target.paths(root); + REQUIRE(source_paths.has_value()); + REQUIRE(target_paths.has_value()); + REQUIRE(source_paths->size() == 2); + REQUIRE(target_paths->size() == 2); + CHECK(std::filesystem::equivalent((*source_paths)[0], (*target_paths)[0])); + CHECK(std::filesystem::equivalent((*source_paths)[1], (*target_paths)[1])); +} + +TEST_CASE("DAG builder rejects invalid SF topology before processing", "[dag][sf][validation]") +{ + TemporaryDirectory input_directory; + TemporaryDirectory output_directory; + auto input_result = mesh::storage::open_folder_indexed(input_directory.path()); + REQUIRE(input_result.has_value()); + auto input = std::move(input_result.value()); + const octree::Id root = octree::Id::root(); + const mesh::Simple mesh = sample_mesh(); + REQUIRE(input.save(root, mesh).has_value()); + REQUIRE(input.save(root.child(0).value(), mesh).has_value()); + REQUIRE(input.index().is(store::NodeStatus::Inner, root).value()); + + auto output_result = dag::storage::open_folder_indexed(output_directory.path()); + REQUIRE(output_result.has_value()); + auto output = std::move(output_result.value()); + const dag::BuildOptions options { + .clusters_per_partition = 1, + .target_ratio = 1.0f, + .relative_target_error = std::nullopt, + }; + + const auto built = dag::build_full(input, output, options); + REQUIRE_FALSE(built.has_value()); + CHECK(built.error().code() == Error::Code::CorruptData); + CHECK(built.error().to_string().contains(root.to_string())); + CHECK(output.index().empty()); +} diff --git a/unittests/sf_builder/storage_finalization.cpp b/unittests/sf_builder/storage_finalization.cpp new file mode 100644 index 00000000..dbdc1a24 --- /dev/null +++ b/unittests/sf_builder/storage_finalization.cpp @@ -0,0 +1,50 @@ +#include + +#include + +#include "../temporary_directory.h" +#include "mesh/storage.h" +#include "sf/finalize_storage.h" + +namespace { + +using test::TemporaryDirectory; + +mesh::Simple sample_mesh() { return mesh::Simple({ { 0, 1, 2 } }, { { 0.0, 0.0, 0.0 }, { 1.0, 0.0, 0.0 }, { 0.0, 1.0, 0.0 } }); } + +} // namespace + +TEST_CASE("SF builder finalization writes and validates a valid index", "[sfbuilder][sf]") +{ + TemporaryDirectory directory; + auto storage_result = mesh::storage::open_folder(directory.path()); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + REQUIRE(storage.save(octree::Id::root(), sample_mesh()).has_value()); + + CHECK(sf::finalize_storage(storage).has_value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storemeta")); +} + +TEST_CASE("SF builder finalization retains an invalid written index for diagnosis", "[sfbuilder][sf]") +{ + TemporaryDirectory directory; + auto storage_result = mesh::storage::open_folder(directory.path()); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + const octree::Id root = octree::Id::root(); + const mesh::Simple mesh = sample_mesh(); + REQUIRE(storage.save(root, mesh).has_value()); + REQUIRE(storage.save(root.child(0).value(), mesh).has_value()); + + const auto finalized = sf::finalize_storage(storage); + REQUIRE_FALSE(finalized.has_value()); + CHECK(finalized.error().code() == Error::Code::CorruptData); + CHECK(finalized.error().to_string().contains(root.to_string())); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + + auto reopened = mesh::storage::open_index(directory.path() / "octree.storeindex"); + REQUIRE(reopened.has_value()); + CHECK(reopened->index().is(store::NodeStatus::Inner, root).value()); +} diff --git a/unittests/sf_merger/integration.cpp b/unittests/sf_merger/integration.cpp index 608d4d91..bb7e68e0 100644 --- a/unittests/sf_merger/integration.cpp +++ b/unittests/sf_merger/integration.cpp @@ -58,7 +58,7 @@ int build_sf( " --min-texture-level 12 --max-texture-level 17" " --mesh-srs EPSG:4978 --verbosity warn" " batch --target-level 15 --output " + shell_quote(output) + - " --format .terrain --threads 1"; + " --format .sfmesh --threads 1"; return std::system(command.c_str()); } @@ -90,8 +90,8 @@ TEST_CASE("SF builders preserve geometry at regular and point-touching mask bord " --verbosity warn"; REQUIRE(std::system(merge_command.c_str()) == 0); - const std::filesystem::path working_relative = "15/26291/18610/27235.terrain"; - const std::filesystem::path regression_relative = "15/26290/18610/27235.terrain"; + const std::filesystem::path working_relative = "15/26291/18610/27235.sfmesh"; + const std::filesystem::path regression_relative = "15/26290/18610/27235.sfmesh"; const std::filesystem::path working_path = merged_output / working_relative; const std::filesystem::path regression_path = merged_output / regression_relative; @@ -115,7 +115,7 @@ TEST_CASE("SF builders preserve geometry at regular and point-touching mask bord const auto regression_mesh = mesh::io::load_from_path(regression_path); const std::string regression_error = regression_mesh.has_value() ? std::string() - : regression_mesh.error().description(); + : regression_mesh.error().to_string(); INFO("Point-touching mask-border merge: " << regression_error); REQUIRE(regression_mesh.has_value()); CHECK_FALSE(regression_mesh->is_empty()); diff --git a/unittests/sf_merger/storage_boundaries.cpp b/unittests/sf_merger/storage_boundaries.cpp new file mode 100644 index 00000000..9f4f3a6a --- /dev/null +++ b/unittests/sf_merger/storage_boundaries.cpp @@ -0,0 +1,178 @@ +#include + +#include + +#include "../temporary_directory.h" +#include "cut.h" +#include "merge.h" +#include "mesh/storage.h" + +namespace { + +using test::TemporaryDirectory; + +mesh::Simple triangle(const double x_offset = 0.0) +{ + return mesh::Simple({ { 0, 1, 2 } }, + { + { -1.0 + x_offset, 0.0, 0.0 }, + { 1.0 + x_offset, 0.0, 0.0 }, + { x_offset, 2.0, 0.0 }, + }); +} + +mesh::Simple clipping_box() +{ + return mesh::Simple( + { + { 0, 1, 2 }, + { 0, 2, 3 }, + { 1, 5, 6 }, + { 1, 6, 2 }, + { 5, 4, 7 }, + { 5, 7, 6 }, + { 4, 0, 3 }, + { 4, 3, 7 }, + { 3, 2, 6 }, + { 3, 6, 7 }, + { 4, 5, 1 }, + { 4, 1, 0 }, + }, + { + { 0.0, -1.0, -1.0 }, + { 2.0, -1.0, -1.0 }, + { 2.0, 3.0, -1.0 }, + { 0.0, 3.0, -1.0 }, + { 0.0, -1.0, 1.0 }, + { 2.0, -1.0, 1.0 }, + { 2.0, 3.0, 1.0 }, + { 0.0, 3.0, 1.0 }, + }); +} + +mesh::storage::IndexedStorage indexed_storage(const std::filesystem::path& path) +{ + auto result = mesh::storage::open_folder_indexed(path); + REQUIRE(result.has_value()); + return std::move(result.value()); +} + +mesh::storage::Storage unindexed_storage(const std::filesystem::path& path) +{ + auto result = mesh::storage::open_folder(path); + REQUIRE(result.has_value()); + return std::move(result.value()); +} + +} // namespace + +TEST_CASE("SF merge rejects Inner input before dispatch", "[sfmerger][sf][validation]") +{ + TemporaryDirectory left_directory("invalid-left"); + TemporaryDirectory right_directory("invalid-right"); + TemporaryDirectory output_directory("invalid-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + const octree::Id root = octree::Id::root(); + REQUIRE(left.save(root, triangle()).has_value()); + REQUIRE(left.save(root.child(0).value(), triangle()).has_value()); + + const auto merged = merge_datasets(left, right, output); + REQUIRE_FALSE(merged.has_value()); + CHECK(merged.error().code() == Error::Code::CorruptData); + CHECK(merged.error().to_string().contains(root.to_string())); + CHECK_FALSE(std::filesystem::exists(output_directory.path() / "octree.storeindex")); +} + +TEST_CASE("SF merge retains an invalid output index for diagnosis", "[sfmerger][sf][validation]") +{ + TemporaryDirectory left_directory("valid-left"); + TemporaryDirectory right_directory("valid-right"); + TemporaryDirectory output_directory("bad-final-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + const octree::Id root = octree::Id::root(); + REQUIRE(output.save(root, triangle()).has_value()); + REQUIRE(output.save(root.child(0).value(), triangle()).has_value()); + + const auto merged = merge_datasets(left, right, output); + REQUIRE_FALSE(merged.has_value()); + CHECK(merged.error().code() == Error::Code::CorruptData); + CHECK(merged.error().to_string().contains(root.to_string())); + CHECK(std::filesystem::is_regular_file(output_directory.path() / "octree.storeindex")); +} + +TEST_CASE("SF merge hard-links unchanged nodes and writes changed nodes", "[sfmerger][sf][copy]") +{ + const octree::Id root = octree::Id::root(); + + SECTION("unchanged") + { + TemporaryDirectory left_directory("unchanged-left"); + TemporaryDirectory right_directory("unchanged-right"); + TemporaryDirectory output_directory("unchanged-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(left.save(root, triangle()).has_value()); + + REQUIRE(merge_datasets(left, right, output).has_value()); + REQUIRE(output.has(root).value()); + CHECK(std::filesystem::equivalent(left.paths(root)->front(), output.paths(root)->front())); + } + + SECTION("changed") + { + TemporaryDirectory left_directory("changed-left"); + TemporaryDirectory right_directory("changed-right"); + TemporaryDirectory output_directory("changed-output"); + auto left = indexed_storage(left_directory.path()); + auto right = indexed_storage(right_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(left.save(root, triangle()).has_value()); + REQUIRE(right.save(root, triangle(10.0)).has_value()); + + REQUIRE(merge_datasets(left, right, output).has_value()); + REQUIRE(output.has(root).value()); + CHECK_FALSE(std::filesystem::equivalent(left.paths(root)->front(), output.paths(root)->front())); + CHECK_FALSE(std::filesystem::equivalent(right.paths(root)->front(), output.paths(root)->front())); + REQUIRE(output.load(root).has_value()); + CHECK(output.load(root)->face_count() == 2); + } +} + +TEST_CASE("SF cut hard-links unchanged leaves and writes clipped leaves", "[sfmerger][sf][copy]") +{ + const octree::Id root = octree::Id::root(); + + SECTION("unchanged") + { + TemporaryDirectory input_directory("cut-unchanged-input"); + TemporaryDirectory output_directory("cut-unchanged-output"); + auto input = indexed_storage(input_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(input.save(root, triangle()).has_value()); + + REQUIRE(cut_dataset(input, MeshMask {}, output, false).has_value()); + REQUIRE(output.has(root).value()); + CHECK(std::filesystem::equivalent(input.paths(root)->front(), output.paths(root)->front())); + } + + SECTION("clipped") + { + TemporaryDirectory input_directory("cut-clipped-input"); + TemporaryDirectory output_directory("cut-clipped-output"); + auto input = indexed_storage(input_directory.path()); + auto output = unindexed_storage(output_directory.path()); + REQUIRE(input.save(root, triangle()).has_value()); + + REQUIRE(cut_dataset(input, MeshMask { .components = { clipping_box() } }, output, true).has_value()); + REQUIRE(output.has(root).value()); + CHECK_FALSE(std::filesystem::equivalent(input.paths(root)->front(), output.paths(root)->front())); + const auto clipped = output.load(root); + REQUIRE(clipped.has_value()); + CHECK_FALSE(clipped->is_empty()); + } +} diff --git a/unittests/temporary_directory.h b/unittests/temporary_directory.h new file mode 100644 index 00000000..10f2b60f --- /dev/null +++ b/unittests/temporary_directory.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace test { + +class TemporaryDirectory { +public: + explicit TemporaryDirectory(const std::string_view label = {}) + { + static std::atomic_uint64_t counter = 0; + const auto timestamp = std::chrono::steady_clock::now().time_since_epoch().count(); + std::string name = "atb-test"; + if (!label.empty()) { + name += "-" + std::string(label); + } + name += "-" + std::to_string(timestamp) + "-" + std::to_string(counter++); + m_path = std::filesystem::temp_directory_path() / name; + REQUIRE(std::filesystem::create_directories(m_path)); + } + + TemporaryDirectory(const TemporaryDirectory&) = delete; + TemporaryDirectory& operator=(const TemporaryDirectory&) = delete; + TemporaryDirectory(TemporaryDirectory&&) = delete; + TemporaryDirectory& operator=(TemporaryDirectory&&) = delete; + + ~TemporaryDirectory() + { + std::error_code error; + std::filesystem::remove_all(m_path, error); + } + + const std::filesystem::path& path() const { return m_path; } + +private: + std::filesystem::path m_path; +}; + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(!std::is_move_constructible_v); +static_assert(!std::is_move_assignable_v); + +} // namespace test diff --git a/unittests/terrainlib/envelope.cpp b/unittests/terrainlib/envelope.cpp index 109f2b1b..06b21391 100644 --- a/unittests/terrainlib/envelope.cpp +++ b/unittests/terrainlib/envelope.cpp @@ -2,9 +2,10 @@ #include "io/envelope.h" -#include #include +#include +#include #include #include #include @@ -13,64 +14,66 @@ namespace { namespace v1 { -struct Payload { - std::uint32_t id; - std::string name; + struct Payload { + std::uint32_t id; + std::string name; - bool operator==(const Payload &) const = default; -}; + bool operator==(const Payload&) const = default; + }; } // namespace v1 namespace v2 { -struct Payload { - std::uint64_t id; - std::string name; - bool enabled; - - static Payload from_previous(v1::Payload previous) - { - return { - .id = previous.id, - .name = std::move(previous.name), - .enabled = true, - }; - } - - bool operator==(const Payload &) const = default; -}; + struct Payload { + std::uint64_t id; + std::string name; + bool enabled; + + static Payload from_previous(v1::Payload previous) + { + return { + .id = previous.id, + .name = std::move(previous.name), + .enabled = true, + }; + } + + bool operator==(const Payload&) const = default; + }; } // namespace v2 namespace v3 { -struct Payload { - std::uint64_t id; - std::string label; - bool enabled; - std::vector samples; - - static Payload from_previous(v2::Payload previous) - { - return { - .id = previous.id, - .label = std::move(previous.name), - .enabled = previous.enabled, - .samples = {}, - }; - } - - bool operator==(const Payload &) const = default; -}; + struct Payload { + std::uint64_t id; + std::string label; + bool enabled; + std::vector samples; + + static Payload from_previous(v2::Payload previous) + { + return { + .id = previous.id, + .label = std::move(previous.name), + .enabled = previous.enabled, + .samples = {}, + }; + } + + bool operator==(const Payload&) const = default; + }; } // namespace v3 -using Schema = io::envelope::PayloadSchema< - "test.Payload", - io::envelope::Version<1, v1::Payload>, - io::envelope::Version<2, v2::Payload>, - io::envelope::Version<3, v3::Payload>>; +using Schema = io::envelope:: + PayloadSchema<"test.Payload", io::envelope::Version<1, v1::Payload>, io::envelope::Version<2, v2::Payload>, io::envelope::Version<3, v3::Payload>>; + +constexpr std::array zstd_compression_algorithms { + io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, + io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum, +}; static_assert(std::is_aggregate_v); static_assert(std::is_aggregate_v); @@ -81,7 +84,7 @@ static_assert(std::same_as); static_assert(std::same_as, v1::Payload>); template -io::envelope::Bytes encode_value(const Value &value) +io::envelope::Bytes encode_value(const Value& value) { io::envelope::Bytes bytes; zpp::bits::out output(bytes); @@ -89,48 +92,39 @@ io::envelope::Bytes encode_value(const Value &value) return bytes; } -io::envelope::Bytes encode_envelope(const io::envelope::Envelope &envelope) -{ - return encode_value(envelope); -} +io::envelope::Bytes encode_envelope(const io::envelope::Envelope& envelope) { return encode_value(envelope); } -io::envelope::Envelope decode_envelope(const io::envelope::Bytes &bytes) +io::envelope::Envelope decode_envelope(const io::envelope::Bytes& bytes) { - io::envelope::Envelope envelope{}; + io::envelope::Envelope envelope {}; zpp::bits::in input(bytes); input(envelope).or_throw(); return envelope; } -io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes &uncompressed_data) +io::envelope::Bytes compress_without_content_size(const io::envelope::Bytes& uncompressed_data) { - const std::unique_ptr context{ - ZSTD_createCCtx(), &ZSTD_freeCCtx}; - if (!context - || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_contentSizeFlag, 0)) + const std::unique_ptr context { ZSTD_createCCtx(), &ZSTD_freeCCtx }; + if (!context || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_contentSizeFlag, 0)) || ZSTD_isError(ZSTD_CCtx_setParameter(context.get(), ZSTD_c_checksumFlag, 1))) { - throw std::runtime_error{"could not configure zstd test context"}; + throw std::runtime_error { "could not configure zstd test context" }; } io::envelope::Bytes compressed_data(ZSTD_compressBound(uncompressed_data.size())); - const std::size_t compressed_size = ZSTD_compress2( - context.get(), - compressed_data.data(), - compressed_data.size(), - uncompressed_data.data(), - uncompressed_data.size()); + const std::size_t compressed_size + = ZSTD_compress2(context.get(), compressed_data.data(), compressed_data.size(), uncompressed_data.data(), uncompressed_data.size()); if (ZSTD_isError(compressed_size)) { - throw std::runtime_error{"could not create zstd test data"}; + throw std::runtime_error { "could not create zstd test data" }; } compressed_data.resize(compressed_size); return compressed_data; } template -void check_error(const Result &result, const io::envelope::ErrorCode expected) +void check_error(const Result& result, const Error::Code expected) { REQUIRE_FALSE(result.has_value()); - CHECK(result.error().code == expected); + CHECK(result.error().code() == expected); } io::envelope::Bytes bytes_from_string(const std::string_view text) @@ -147,11 +141,11 @@ io::envelope::Bytes bytes_from_string(const std::string_view text) TEST_CASE("Envelope round trips the latest payload version") { - const v3::Payload expected{ + const v3::Payload expected { .id = 42, .label = "latest", .enabled = false, - .samples = {1, 2, 3}, + .samples = { 1, 2, 3 }, }; const auto bytes = io::envelope::serialize(expected); @@ -163,8 +157,7 @@ TEST_CASE("Envelope round trips the latest payload version") CHECK(envelope.class_version == 3); CHECK(envelope.checksum_algorithm == io::envelope::ChecksumAlgorithm::HandledByCompressionLib); CHECK(envelope.checksum.empty()); - CHECK(envelope.compression_algorithm - == io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum); + CHECK(envelope.compression_algorithm == io::envelope::CompressionAlgorithm::ZstdDefaultCompressionWithChecksum); CHECK(envelope.uncompressed_size == encode_value(expected).size()); const auto result = io::envelope::deserialize(*bytes); @@ -176,49 +169,48 @@ TEST_CASE("Envelope upgrades older payload versions") { SECTION("version 1 is upgraded through every subsequent version") { - const v1::Payload original{.id = 7, .name = "version one"}; + const v1::Payload original { .id = 7, .name = "version one" }; const auto bytes = io::envelope::serialize(original); REQUIRE(bytes.has_value()); const auto result = io::envelope::deserialize(*bytes); REQUIRE(result.has_value()); - CHECK(*result == v3::Payload{ - .id = 7, - .label = "version one", - .enabled = true, - .samples = {}, - }); + CHECK(*result + == v3::Payload { + .id = 7, + .label = "version one", + .enabled = true, + .samples = {}, + }); } SECTION("version 2 is upgraded to the latest version") { - const v2::Payload original{.id = 9, .name = "version two", .enabled = false}; + const v2::Payload original { .id = 9, .name = "version two", .enabled = false }; const auto bytes = io::envelope::serialize(original); REQUIRE(bytes.has_value()); const auto result = io::envelope::deserialize(*bytes); REQUIRE(result.has_value()); - CHECK(*result == v3::Payload{ - .id = 9, - .label = "version two", - .enabled = false, - .samples = {}, - }); + CHECK(*result + == v3::Payload { + .id = 9, + .label = "version two", + .enabled = false, + .samples = {}, + }); } } TEST_CASE("Envelope supports uncompressed data without a checksum") { - const v3::Payload expected{ + const v3::Payload expected { .id = 11, .label = "plain", .enabled = true, - .samples = {5, 8}, + .samples = { 5, 8 }, }; - const auto bytes = io::envelope::serialize( - expected, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::None); + const auto bytes = io::envelope::serialize(expected, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::None); REQUIRE(bytes.has_value()); const auto envelope = decode_envelope(*bytes); @@ -237,19 +229,14 @@ TEST_CASE("CRC-32C uses its canonical hexadecimal representation") SECTION("standard test vector") { const auto compressed = io::envelope::compress_with_checksum( - bytes_from_string("123456789"), - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + bytes_from_string("123456789"), io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(compressed.has_value()); CHECK(compressed->checksum == "e3069283"); } SECTION("empty input") { - const auto compressed = io::envelope::compress_with_checksum( - {}, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + const auto compressed = io::envelope::compress_with_checksum({}, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(compressed.has_value()); CHECK(compressed->checksum == "00000000"); } @@ -261,75 +248,49 @@ TEST_CASE("CRC-32C protects independently compressed data") SECTION("without compression") { - const auto compressed = io::envelope::compress_with_checksum( - original, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + const auto compressed + = io::envelope::compress_with_checksum(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(compressed.has_value()); const auto result = io::envelope::checked_decompress( - compressed->compressed_data, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c, - compressed->checksum); + compressed->compressed_data, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, compressed->checksum); REQUIRE(result.has_value()); CHECK(*result == original); auto corrupted = compressed->compressed_data; - corrupted.front() ^= std::byte{0x01}; - check_error( - io::envelope::checked_decompress( - corrupted, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c, - compressed->checksum), - io::envelope::ErrorCode::ChecksumMismatch); + corrupted.front() ^= std::byte { 0x01 }; + check_error(io::envelope::checked_decompress( + corrupted, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, compressed->checksum), + Error::Code::CorruptData); } SECTION("with zstd compression") { - const auto compressed = io::envelope::compress_with_checksum( - original, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::Crc32c); - REQUIRE(compressed.has_value()); - REQUIRE(compressed->checksum.size() == 8); + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto compressed = io::envelope::compress_with_checksum(original, compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c); + REQUIRE(compressed.has_value()); + REQUIRE(compressed->checksum.size() == 8); - const auto result = io::envelope::checked_decompress( - compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::Crc32c, - compressed->checksum); - REQUIRE(result.has_value()); - CHECK(*result == original); + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c, compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); - auto wrong_checksum = compressed->checksum; - wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; - check_error( - io::envelope::checked_decompress( - compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::Crc32c, - wrong_checksum), - io::envelope::ErrorCode::ChecksumMismatch); + auto wrong_checksum = compressed->checksum; + wrong_checksum.front() = wrong_checksum.front() == '0' ? '1' : '0'; + check_error( + io::envelope::checked_decompress(compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::Crc32c, wrong_checksum), + Error::Code::CorruptData); + } } SECTION("malformed checksum") { - check_error( - io::envelope::checked_decompress( - original, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c, - "E3069283"), - io::envelope::ErrorCode::ChecksumMismatch); - check_error( - io::envelope::checked_decompress( - original, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c, - {}), - io::envelope::ErrorCode::ChecksumMismatch); + check_error(io::envelope::checked_decompress(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, "E3069283"), + Error::Code::CorruptData); + check_error(io::envelope::checked_decompress(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c, {}), + Error::Code::CorruptData); } } @@ -337,16 +298,13 @@ TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") { SECTION("latest version") { - const v3::Payload expected{ + const v3::Payload expected { .id = 12, .label = "crc", .enabled = true, - .samples = {3, 5, 8}, + .samples = { 3, 5, 8 }, }; - const auto bytes = io::envelope::serialize( - expected, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + const auto bytes = io::envelope::serialize(expected, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(bytes.has_value()); const auto envelope = decode_envelope(*bytes); @@ -359,110 +317,82 @@ TEST_CASE("Envelope round trips and upgrades payloads protected by CRC-32C") CHECK(*result == expected); auto corrupted = envelope; - corrupted.compressed_data.front() ^= std::byte{0x01}; - check_error(io::envelope::deserialize(encode_envelope(corrupted)), - io::envelope::ErrorCode::ChecksumMismatch); + corrupted.compressed_data.front() ^= std::byte { 0x01 }; + check_error(io::envelope::deserialize(encode_envelope(corrupted)), Error::Code::CorruptData); } SECTION("older version") { - const v1::Payload original{.id = 13, .name = "crc version one"}; - const auto bytes = io::envelope::serialize( - original, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::Crc32c); + const v1::Payload original { .id = 13, .name = "crc version one" }; + const auto bytes = io::envelope::serialize(original, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::Crc32c); REQUIRE(bytes.has_value()); const auto result = io::envelope::deserialize(*bytes); REQUIRE(result.has_value()); - CHECK(*result == v3::Payload{ - .id = 13, - .label = "crc version one", - .enabled = true, - .samples = {}, - }); + CHECK(*result + == v3::Payload { + .id = 13, + .label = "crc version one", + .enabled = true, + .samples = {}, + }); } } TEST_CASE("Checked compression round trips and validates its checksum") { - const io::envelope::Bytes original(4096, std::byte{0x2a}); - const auto compressed = io::envelope::compress_with_checksum( - original, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib); - REQUIRE(compressed.has_value()); - CHECK(compressed->compressed_data.size() < original.size()); - CHECK(compressed->checksum.empty()); - - const auto result = io::envelope::checked_decompress( - compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - compressed->checksum); - REQUIRE(result.has_value()); - CHECK(*result == original); - - const auto empty_compressed = io::envelope::compress_with_checksum( - {}, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib); - REQUIRE(empty_compressed.has_value()); - const auto empty_result = io::envelope::checked_decompress( - empty_compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - empty_compressed->checksum); - REQUIRE(empty_result.has_value()); - CHECK(empty_result->empty()); - - auto corrupted = compressed->compressed_data; - corrupted.back() ^= std::byte{0x01}; - check_error( - io::envelope::checked_decompress( - corrupted, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}), - io::envelope::ErrorCode::ChecksumMismatch); - - check_error( - io::envelope::checked_decompress( - compressed->compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}, - original.size() - 1), - io::envelope::ErrorCode::SizeLimitExceeded); + const io::envelope::Bytes original(4096, std::byte { 0x2a }); + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto compressed = io::envelope::compress_with_checksum(original, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(compressed.has_value()); + CHECK(compressed->compressed_data.size() < original.size()); + CHECK(compressed->checksum.empty()); + + const auto result = io::envelope::checked_decompress( + compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, compressed->checksum); + REQUIRE(result.has_value()); + CHECK(*result == original); + + const auto empty_compressed = io::envelope::compress_with_checksum({}, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib); + REQUIRE(empty_compressed.has_value()); + const auto empty_result = io::envelope::checked_decompress( + empty_compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, empty_compressed->checksum); + REQUIRE(empty_result.has_value()); + CHECK(empty_result->empty()); + + auto corrupted = compressed->compressed_data; + corrupted.back() ^= std::byte { 0x01 }; + check_error(io::envelope::checked_decompress(corrupted, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}), + Error::Code::CorruptData); + + check_error(io::envelope::checked_decompress( + compressed->compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size() - 1), + Error::Code::ResourceExhausted); + } } TEST_CASE("Checked decompression uses its maximum when the format omits the content size") { - const io::envelope::Bytes original(4096, std::byte{0x37}); + const io::envelope::Bytes original(4096, std::byte { 0x37 }); const auto compressed_data = compress_without_content_size(original); - const auto result = io::envelope::checked_decompress( - compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}, - original.size()); - REQUIRE(result.has_value()); - CHECK(*result == original); - - check_error( - io::envelope::checked_decompress( - compressed_data, - io::envelope::CompressionAlgorithm::ZstdBestCompressionWithChecksum, - io::envelope::ChecksumAlgorithm::HandledByCompressionLib, - {}, - original.size() - 1), - io::envelope::ErrorCode::SizeLimitExceeded); + for (const auto compression_algorithm : zstd_compression_algorithms) { + CAPTURE(compression_algorithm); + const auto result = io::envelope::checked_decompress( + compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size()); + REQUIRE(result.has_value()); + CHECK(*result == original); + + check_error(io::envelope::checked_decompress( + compressed_data, compression_algorithm, io::envelope::ChecksumAlgorithm::HandledByCompressionLib, {}, original.size() - 1), + Error::Code::ResourceExhausted); + } } TEST_CASE("Envelope rejects incompatible metadata") { - const v3::Payload payload{.id = 1, .label = "metadata", .enabled = true, .samples = {}}; + const v3::Payload payload { .id = 1, .label = "metadata", .enabled = true, .samples = {} }; const auto serialized = io::envelope::serialize(payload); REQUIRE(serialized.has_value()); @@ -470,56 +400,49 @@ TEST_CASE("Envelope rejects incompatible metadata") { auto envelope = decode_envelope(*serialized); ++envelope.magic; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::InvalidMagic); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("class name") { auto envelope = decode_envelope(*serialized); envelope.class_name = "other.Payload"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::WrongClassName); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("class version") { auto envelope = decode_envelope(*serialized); envelope.class_version = 99; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::UnsupportedClassVersion); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::Unsupported); } SECTION("checksum algorithm") { auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = static_cast(99); - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::UnsupportedChecksumAlgorithm); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::Unsupported); } SECTION("compression algorithm") { auto envelope = decode_envelope(*serialized); envelope.compression_algorithm = static_cast(99); - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::UnsupportedCompressionAlgorithm); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::Unsupported); } SECTION("algorithm combination") { auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::None; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::InvalidAlgorithmCombination); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("external checksum") { auto envelope = decode_envelope(*serialized); envelope.checksum = "not used by zstd"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::InvalidAlgorithmCombination); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("malformed CRC-32C checksum") @@ -527,8 +450,7 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; envelope.checksum = "1234"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::ChecksumMismatch); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("incorrect CRC-32C checksum") @@ -536,76 +458,65 @@ TEST_CASE("Envelope rejects incompatible metadata") auto envelope = decode_envelope(*serialized); envelope.checksum_algorithm = io::envelope::ChecksumAlgorithm::Crc32c; envelope.checksum = "00000000"; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::ChecksumMismatch); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("uncompressed size is smaller than the payload") { auto envelope = decode_envelope(*serialized); --envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + const auto result = io::envelope::deserialize(encode_envelope(envelope)); + check_error(result, Error::Code::CorruptData); + CHECK(result.error().to_string().find("reclassified ResourceExhausted -> CorruptData") != std::string::npos); } SECTION("uncompressed size is larger than the payload") { auto envelope = decode_envelope(*serialized); ++envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("zero uncompressed size does not mean unspecified") { auto envelope = decode_envelope(*serialized); envelope.uncompressed_size = 0; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("uncompressed size exceeds the hard limit") { auto envelope = decode_envelope(*serialized); envelope.uncompressed_size = io::envelope::default_max_decompressed_size + 1; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::SizeLimitExceeded); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::ResourceExhausted); } SECTION("uncompressed size exceeds a caller limit") { auto envelope = decode_envelope(*serialized); - check_error( - io::envelope::deserialize( - encode_envelope(envelope), - static_cast(envelope.uncompressed_size - 1)), - io::envelope::ErrorCode::SizeLimitExceeded); + check_error(io::envelope::deserialize(encode_envelope(envelope), static_cast(envelope.uncompressed_size - 1)), + Error::Code::ResourceExhausted); } } TEST_CASE("Envelope validates the size of uncompressed data") { - const v3::Payload payload{.id = 2, .label = "plain", .enabled = true, .samples = {1}}; - const auto serialized = io::envelope::serialize( - payload, - io::envelope::CompressionAlgorithm::None, - io::envelope::ChecksumAlgorithm::None); + const v3::Payload payload { .id = 2, .label = "plain", .enabled = true, .samples = { 1 } }; + const auto serialized = io::envelope::serialize(payload, io::envelope::CompressionAlgorithm::None, io::envelope::ChecksumAlgorithm::None); REQUIRE(serialized.has_value()); SECTION("declared size is smaller") { auto envelope = decode_envelope(*serialized); --envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } SECTION("declared size is larger") { auto envelope = decode_envelope(*serialized); ++envelope.uncompressed_size; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DecompressionFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } } @@ -613,24 +524,22 @@ TEST_CASE("Envelope reports malformed serialized data") { SECTION("envelope") { - const io::envelope::Bytes malformed{std::byte{0x01}, std::byte{0x02}}; - check_error(io::envelope::deserialize(malformed), - io::envelope::ErrorCode::DeserializationFailed); + const io::envelope::Bytes malformed { std::byte { 0x01 }, std::byte { 0x02 } }; + check_error(io::envelope::deserialize(malformed), Error::Code::CorruptData); } SECTION("payload") { - const io::envelope::Envelope envelope{ + const io::envelope::Envelope envelope { .magic = io::envelope::magic, - .class_name = std::string{Schema::class_name}, + .class_name = std::string { Schema::class_name }, .class_version = 3, .checksum_algorithm = io::envelope::ChecksumAlgorithm::None, .checksum = {}, .compression_algorithm = io::envelope::CompressionAlgorithm::None, .uncompressed_size = 1, - .compressed_data = {std::byte{0x01}}, + .compressed_data = { std::byte { 0x01 } }, }; - check_error(io::envelope::deserialize(encode_envelope(envelope)), - io::envelope::ErrorCode::DeserializationFailed); + check_error(io::envelope::deserialize(encode_envelope(envelope)), Error::Code::CorruptData); } } diff --git a/unittests/terrainlib/error.cpp b/unittests/terrainlib/error.cpp new file mode 100644 index 00000000..c2b7a611 --- /dev/null +++ b/unittests/terrainlib/error.cpp @@ -0,0 +1,52 @@ +#include +#include + +#include + +#include "Error.h" + +TEST_CASE("Error::fail creates an expected failure", "[error]") +{ + const Expected result = Error::fail(Error::Code::Io, "open input"); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Io); + CHECK(result.error().to_string().find("open input") != std::string::npos); +} + +TEST_CASE("Error::propagate adds a call-site frame", "[error]") +{ + Expected source = Error::fail(Error::Code::Io, "open input"); + const Expected result = Error::propagate(std::move(source)); + + REQUIRE_FALSE(result.has_value()); + const std::string description = result.error().to_string(); + CHECK(description.find("propagated") != std::string::npos); + CHECK(description.find("unittests/terrainlib/error.cpp") != std::string::npos); + CHECK(description.find("open input") != std::string::npos); +} + +TEST_CASE("Error::propagate uses a supplied context as its call-site frame", "[error]") +{ + Expected source = Error::fail(Error::Code::ResourceExhausted, "decode payload"); + const Expected result = Error::propagate(std::move(source), Error::Code::CorruptData, "read envelope"); + + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::CorruptData); + const std::string description = result.error().to_string(); + CHECK(description.find("read envelope (reclassified ResourceExhausted -> CorruptData)") != std::string::npos); + CHECK(description.find("unittests/terrainlib/error.cpp") != std::string::npos); + CHECK(description.find("decode payload") != std::string::npos); +} + +TEST_CASE("Error::propagate adds supplied context to an Error", "[error]") +{ + Error source = Error::make(Error::Code::Io, "open input"); + const Expected result = Error::propagate(std::move(source), "read input"); + + REQUIRE_FALSE(result.has_value()); + const std::string description = result.error().to_string(); + CHECK(description.find("read input") != std::string::npos); + CHECK(description.find("unittests/terrainlib/error.cpp") != std::string::npos); + CHECK(description.find("open input") != std::string::npos); +} diff --git a/unittests/terrainlib/index_map.cpp b/unittests/terrainlib/index_map.cpp deleted file mode 100644 index 5d595c23..00000000 --- a/unittests/terrainlib/index_map.cpp +++ /dev/null @@ -1,100 +0,0 @@ -#include "../catch2_helpers.h" - -#include "octree/IndexMap.h" - -using namespace octree; - -TEST_CASE("IndexMap basic operations") { - IndexMap index; - - SECTION("Add root node") { - Id root = Id::root(); - index.add(root); - CHECK(index.is_present(root)); - CHECK(index.is(NodeStatus::Leaf, root)); - CHECK(index.get(root) == std::optional(NodeStatus::Leaf)); - } - - SECTION("Add child node creates virtual parent") { - Id root = Id::root(); - Id child = root.child(3).value(); - index.add(child); - CHECK(index.is_present(root)); - CHECK(index.is(NodeStatus::Virtual, root)); - CHECK(index.is_present(child)); - CHECK(index.is(NodeStatus::Leaf, child)); - } - - SECTION("Add child note promotes parent to inner") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - index.add(child1); - index.add(root); - CHECK(index.is(NodeStatus::Leaf, child1)); - CHECK(index.is(NodeStatus::Inner, root)); - } - - SECTION("Removing a leaf node cleans up correctly") { - Id root = Id::root(); - Id child = root.child(2).value(); - index.add(child); - CHECK(index.is_present(child)); - index.remove(child); - CHECK(index.is_absent(child)); - CHECK(index.is_absent(root)); - } - - SECTION("Removing one of two children of an inner keeps it inner") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - Id child2 = root.child(1).value(); - index.add(root); - index.add(child1); - index.add(child2); - index.remove(child1); - - CHECK(index.is_absent(child1)); - CHECK(index.is(NodeStatus::Leaf, child2)); - CHECK(index.is(NodeStatus::Inner, root)); - } - - SECTION("Removing last child of inner converts it to leaf") { - Id root = Id::root(); - Id child1 = root.child(0).value(); - index.add(child1); - index.add(root); - - index.remove(child1); - - CHECK(index.is_absent(child1)); - CHECK(index.is(NodeStatus::Leaf, root)); - } - - SECTION("Clear works") { - Id a = Id::root().child(1).value(); - Id b = Id::root().child(2).value(); - index.add(a); - index.add(b); - CHECK(index.size() == 3); // root + a + b - index.clear(); - CHECK(index.empty()); - } - - SECTION("Serialization round-trip") { - IndexMap original; - original.add(Id::root().child(3).value()); - - const std::errc success {}; - - std::vector buffer; - zpp::bits::out out(buffer); - CHECK(out(original).code == success); - - IndexMap loaded; - zpp::bits::in in(buffer); - CHECK(in(loaded).code == success); - - CHECK(original.size() == loaded.size()); - CHECK(loaded.is_present(Id::root().child(3).value())); - } -} diff --git a/unittests/terrainlib/index_traverse.cpp b/unittests/terrainlib/index_traverse.cpp deleted file mode 100644 index 5ada7575..00000000 --- a/unittests/terrainlib/index_traverse.cpp +++ /dev/null @@ -1,132 +0,0 @@ -#include -#include -#include -#include - -#include "octree/Id.h" -#include "octree/IndexMap.h" -#include "octree/traverse.h" - -using namespace octree; - -TEST_CASE("octree::traverse basic") { - IndexMap index; - - const Id root = Id::root(); - index.add(root); - - const auto children = root.children().value(); - for (const auto& child : children) { - index.add(child); - } - - SECTION("traversal visits root then children") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return true; }, // refine always - root, - TraversalOrder::DepthFirst); - - CHECK(visited.front() == root); - CHECK(visited.size() == 9); // root + 8 children - std::vector expected(children.begin(), children.end()); - visited.erase(visited.begin()); - CHECK(visited == expected); - } - - SECTION("Refine function disables recursion") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return false; }, // don't refine - root, - TraversalOrder::DepthFirst); - - CHECK(visited.size() == 1); - CHECK(visited.front() == root); - } - - SECTION("Traversal skips when root is missing") { - IndexMap empty; - bool visited = false; - - traverse( - empty, - [&](const Id&, const NodeStatus&) { - visited = true; - }, - [](const Id&) { return true; }, - root, - TraversalOrder::DepthFirst - ); - - CHECK_FALSE(visited); - } -} - -TEST_CASE("octree::traverse with depth") { - IndexMap index; - - const Id root = Id::root(); - index.add(root); - - const auto children = root.children().value(); - const Id c0 = children[0]; - const Id c1 = children[1]; - const Id c2 = children[2]; - - index.add(c0); - index.add(c1); - index.add(c2); - - // Add grandchildren - const Id gc0 = c0.child(0).value(); // grandchild of c0 - const Id gc1 = c2.child(1).value(); // grandchild of c2 - - index.add(gc0); - index.add(gc1); - - SECTION("DFS visits deeply before siblings") { - std::vector visited; - traverse( - index, - [&](const Id& id, const NodeStatus&) { - visited.push_back(id); - }, - [](const Id&) { return true; }, - root, - TraversalOrder::DepthFirst - ); - - CHECK(visited.size() == 6); - - // DFS order should be: root, c0, gc0, c1, c2, gc1 - std::vector expected{root, c0, gc0, c1, c2, gc1}; - CHECK(visited == expected); - } - - SECTION("BFS visits all nodes level by level") { - std::vector visited; - traverse( - index, - [&](const Id &id, const NodeStatus &) { - visited.push_back(id); - }, - [](const Id &) { return true; }, - root, - TraversalOrder::BreadthFirst); - - CHECK(visited.size() == 6); - - // BFS order should be: root, c0, c1, c2, gc0, gc1 - std::vector expected{root, c0, c1, c2, gc0, gc1}; - CHECK(visited == expected); - } -} diff --git a/unittests/terrainlib/io_conversion.cpp b/unittests/terrainlib/io_conversion.cpp new file mode 100644 index 00000000..e93f6ddc --- /dev/null +++ b/unittests/terrainlib/io_conversion.cpp @@ -0,0 +1,118 @@ +#include "../catch2_helpers.h" + +#include "io/conversion.h" + +#include +#include + +namespace { + +template +void check_round_trip(const int cv_type) +{ + cv::Mat source(2, 3, cv_type); + auto bytes = std::span(source.ptr(), source.total() * source.elemSize()); + for (std::size_t index = 0; index < bytes.size(); ++index) { + bytes[index] = static_cast(index + 1); + } + + const auto raster = io::conversion::to_raster(source); + REQUIRE(raster.has_value()); + CHECK(raster->size() == glm::uvec2(3, 2)); + + const auto result = io::conversion::to_mat(*raster); + REQUIRE(result.has_value()); + CHECK(result->type() == source.type()); + CHECK(result->rows == source.rows); + CHECK(result->cols == source.cols); + CHECK(std::equal(bytes.begin(), bytes.end(), result->template ptr())); +} + +} // namespace + +TEST_CASE("io::conversion round trips supported scalar and vector pixels") +{ + check_round_trip(CV_8UC1); + check_round_trip(CV_8SC1); + check_round_trip(CV_16UC1); + check_round_trip(CV_16SC1); + check_round_trip(CV_32SC1); + check_round_trip(CV_32FC1); + check_round_trip(CV_64FC1); + check_round_trip(CV_8UC2); + check_round_trip(CV_8UC3); + check_round_trip(CV_8UC4); + check_round_trip(CV_32FC3); + check_round_trip(CV_64FC4); +} + +TEST_CASE("io::conversion copies non-contiguous matrices row by row") +{ + cv::Mat backing(4, 6, CV_8UC3); + for (int row = 0; row < backing.rows; ++row) { + for (int column = 0; column < backing.cols; ++column) { + backing.at(row, column) + = cv::Vec3b(static_cast(row), static_cast(column), static_cast(row * backing.cols + column)); + } + } + const cv::Mat source = backing(cv::Rect(1, 1, 3, 2)); + REQUIRE_FALSE(source.isContinuous()); + + auto raster = io::conversion::to_raster(source); + REQUIRE(raster.has_value()); + CHECK(raster->pixel({ 0, 0 }) == glm::u8vec3(1, 1, 7)); + CHECK(raster->pixel({ 2, 1 }) == glm::u8vec3(2, 3, 15)); + + backing.setTo(cv::Scalar::all(0)); + CHECK(raster->pixel({ 0, 0 }) == glm::u8vec3(1, 1, 7)); + + const auto round_trip = io::conversion::to_mat(*raster); + REQUIRE(round_trip.has_value()); + raster->fill(glm::u8vec3(0)); + CHECK(round_trip->at(0, 0) == cv::Vec3b(1, 1, 7)); +} + +TEST_CASE("io::conversion reports invalid inputs") +{ + SECTION("empty matrix") + { + const auto result = io::conversion::to_raster(cv::Mat {}); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); + } + + SECTION("multidimensional matrix") + { + const std::array sizes { 2, 2, 2 }; + const cv::Mat source(static_cast(sizes.size()), sizes.data(), CV_8UC1); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Unsupported); + } + + SECTION("pixel type mismatch") + { + const cv::Mat source(2, 2, CV_8UC3); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); + } + + SECTION("unsupported pixel") + { + struct Pixel { + std::uint64_t value; + }; + const cv::Mat source(2, 2, CV_8UC1); + const auto result = io::conversion::to_raster(source); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Unsupported); + } + + SECTION("empty raster") + { + const auto result = io::conversion::to_mat(radix::Raster {}); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); + } +} diff --git a/unittests/terrainlib/mesh_clip.cpp b/unittests/terrainlib/mesh_clip.cpp index f3b2d369..a5646c0e 100644 --- a/unittests/terrainlib/mesh_clip.cpp +++ b/unittests/terrainlib/mesh_clip.cpp @@ -354,37 +354,3 @@ TEST_CASE("clip_on_bounds produces manifold mesh") { CHECK(mesh::is_manifold(clipped)); CHECK(clipped.positions.size() == 6); } - -#ifdef NDEBUG -TEST_CASE("mesh::clip_on_bounds benchmark") { - BENCHMARK_ADVANCED("clip based on octree")(Catch::Benchmark::Chronometer meter) { - const std::filesystem::path mesh_path = ALP_TEST_DATA_DIR "/meshes/6857.terrain"; - auto mesh_result = mesh::io::load_from_path(mesh_path); - REQUIRE(mesh_result.has_value()); - SimpleMesh &mesh = mesh_result.value(); - mesh.uvs.clear(); - mesh.texture = std::nullopt; - - const octree::Space space = octree::Space::earth(); - const auto mesh_bounds = calculate_bounds(mesh); - const octree::Id id = space.find_smallest_node_encompassing_bounds(mesh_bounds).value(); - const auto child_ids = id.children().value(); - std::vector child_bounds; - child_bounds.reserve(child_ids.size()); - for (const auto &child_id : child_ids) { - const auto bounds = space.get_node_bounds(child_id); - child_bounds.push_back(bounds); - } - - meter.measure([mesh, child_bounds] { - std::vector child_meshes; - child_meshes.reserve(child_bounds.size()); - for (const auto &bounds : child_bounds) { - const auto clipped_mesh = mesh::clip_on_bounds(mesh, bounds); - child_meshes.push_back(clipped_mesh); - } - return child_meshes; - }); - }; -} -#endif diff --git a/unittests/terrainlib/mesh_io.cpp b/unittests/terrainlib/mesh_io.cpp index fd886bac..f0186921 100644 --- a/unittests/terrainlib/mesh_io.cpp +++ b/unittests/terrainlib/mesh_io.cpp @@ -19,9 +19,11 @@ #include #include +#include #include "../catch2_helpers.h" #include "../opencv_helpers.h" +#include "../temporary_directory.h" #include "mesh/io.h" #include "mesh/encode.h" @@ -93,9 +95,9 @@ TEST_CASE("io roundtrip") { REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const std::expected result = mesh::io::load_from_path(mesh_path); + const auto result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { - FAIL(result.error().description()); + FAIL(result.error().to_string()); } std::filesystem::remove(mesh_path); const SimpleMesh roundtrip_mesh = result.value(); @@ -109,7 +111,7 @@ TEST_CASE("io roundtrip") { } TEST_CASE("io roundtrip high precision") { - for (const auto& format : {"terrain"}) { + for (const auto& format : {"sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; @@ -136,9 +138,9 @@ TEST_CASE("io roundtrip high precision") { REQUIRE(mesh::io::save_to_path(mesh, mesh_path, mesh::io::SaveOptions{.texture_format = ".png"}).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const std::expected result = mesh::io::load_from_path(mesh_path); + const auto result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { - FAIL(result.error().description()); + FAIL(result.error().to_string()); } std::filesystem::remove(mesh_path); const SimpleMesh roundtrip_mesh = result.value(); @@ -152,7 +154,7 @@ TEST_CASE("io roundtrip high precision") { } TEST_CASE("io roundtrip no texture") { - for (const auto &format : {"gltf", "glb", "terrain"}) { + for (const auto &format : {"gltf", "glb", "sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; @@ -179,9 +181,9 @@ TEST_CASE("io roundtrip no texture") { REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const std::expected result = mesh::io::load_from_path(mesh_path); + const auto result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { - FAIL(result.error().description()); + FAIL(result.error().to_string()); } // std::filesystem::remove(mesh_path); const SimpleMesh roundtrip_mesh = result.value(); @@ -194,7 +196,7 @@ TEST_CASE("io roundtrip no texture") { } TEST_CASE("io roundtrip no texture and uvs") { - for (const auto &format : {"gltf", "glb", "terrain"}) { + for (const auto &format : {"gltf", "glb", "sfmesh"}) { DYNAMIC_SECTION(format) { SimpleMesh mesh; @@ -216,9 +218,9 @@ TEST_CASE("io roundtrip no texture and uvs") { REQUIRE(mesh::io::save_to_path(mesh, mesh_path).has_value()); CHECK(std::filesystem::exists(mesh_path)); - const std::expected result = mesh::io::load_from_path(mesh_path); + const auto result = mesh::io::load_from_path(mesh_path); if (!result.has_value()) { - FAIL(result.error().description()); + FAIL(result.error().to_string()); } std::filesystem::remove(mesh_path); const SimpleMesh roundtrip_mesh = result.value(); @@ -229,3 +231,35 @@ TEST_CASE("io roundtrip no texture and uvs") { } } } + +TEST_CASE("mesh IO reports project errors") +{ + test::TemporaryDirectory temporary_directory("mesh-io-errors"); + + const auto unsupported_input = mesh::io::load_from_path("mesh.unsupported"); + REQUIRE_FALSE(unsupported_input.has_value()); + CHECK(unsupported_input.error().code() == Error::Code::Unsupported); + + const auto missing_input = mesh::io::load_from_path(temporary_directory.path() / "missing.glb"); + REQUIRE_FALSE(missing_input.has_value()); + CHECK(missing_input.error().code() == Error::Code::NotFound); + + const std::filesystem::path corrupt_path = temporary_directory.path() / "corrupt.gltf"; + std::ofstream(corrupt_path) << "not glTF"; + const auto corrupt_input = mesh::io::load_from_path(corrupt_path); + REQUIRE_FALSE(corrupt_input.has_value()); + CHECK(corrupt_input.error().code() == Error::Code::CorruptData); + + const auto unsupported_output = mesh::io::save_to_path(mesh::Simple {}, "mesh.unsupported"); + REQUIRE_FALSE(unsupported_output.has_value()); + CHECK(unsupported_output.error().code() == Error::Code::Unsupported); + + mesh::Simple mesh; + mesh.positions = { { 0, 0, 0 }, { 1, 0, 0 }, { 0, 1, 0 } }; + mesh.triangles = { { 0, 1, 2 } }; + const std::filesystem::path directory_output = temporary_directory.path() / "directory.glb"; + REQUIRE(std::filesystem::create_directory(directory_output)); + const auto failed_output = mesh::io::save_to_path(mesh, directory_output); + REQUIRE_FALSE(failed_output.has_value()); + CHECK(failed_output.error().code() == Error::Code::Io); +} diff --git a/unittests/terrainlib/sf_validate_index.cpp b/unittests/terrainlib/sf_validate_index.cpp new file mode 100644 index 00000000..9b6de302 --- /dev/null +++ b/unittests/terrainlib/sf_validate_index.cpp @@ -0,0 +1,27 @@ +#include + +#include "octree/StoreTraits.h" +#include "sf/validate_index.h" + +TEST_CASE("Structura Fundamentalis validation accepts Leaf and Virtual nodes", "[sf][store]") +{ + store::Index index; + const octree::Id leaf = octree::Id::root().child(2).value().child(5).value(); + REQUIRE(index.add(leaf).has_value()); + + CHECK(sf::validate_index(index).has_value()); +} + +TEST_CASE("Structura Fundamentalis validation reports the first Inner node", "[sf][store]") +{ + store::Index index; + const octree::Id root = octree::Id::root(); + const octree::Id child = root.child(3).value(); + REQUIRE(index.add(root).has_value()); + REQUIRE(index.add(child).has_value()); + + const auto result = sf::validate_index(index); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::CorruptData); + CHECK(result.error().to_string().contains(root.to_string())); +} diff --git a/unittests/terrainlib/store_codec.cpp b/unittests/terrainlib/store_codec.cpp new file mode 100644 index 00000000..2dcc9b12 --- /dev/null +++ b/unittests/terrainlib/store_codec.cpp @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include + +#include + +#include "../temporary_directory.h" +#include "io/bytes.h" +#include "mesh/codec/Gltf.h" +#include "mesh/codec/SfMesh.h" +#include "store/Codec.h" + +namespace { + +using test::TemporaryDirectory; + +class MultiFileCodec final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override + { + return { add_extension(node_path, ".data"), add_extension(node_path, ".metadata") }; + } +}; + +class WriteOnlyCodec final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override { return { add_extension(node_path, ".out") }; } + Expected write(const std::filesystem::path&, const int&) const override + { + ++writes; + return {}; + } + + mutable std::atomic_uint32_t writes = 0; +}; + +mesh::Simple triangle_mesh(const double offset) +{ + return mesh::Simple({ { 0, 1, 2 } }, { { offset, 0.0, 0.0 }, { offset + 1.0, 0.0, 0.0 }, { offset, 1.0, 0.0 } }); +} + +template +void check_mesh_codec_reentrancy(const Codec& codec, const std::filesystem::path& base) +{ + std::vector> writes; + for (int index = 0; index < 4; ++index) { + writes.push_back(std::async( + std::launch::async, [&codec, base, index] { return codec.write(base / std::to_string(index) / "node", triangle_mesh(index)).has_value(); })); + } + for (auto& write : writes) { + REQUIRE(write.get()); + } + + std::vector> reads; + for (int index = 0; index < 4; ++index) { + reads.push_back(std::async(std::launch::async, [&codec, base, index] { + const auto result = codec.read(base / std::to_string(index) / "node"); + return result.has_value() && result->face_count() == 1; + })); + } + for (auto& read : reads) { + REQUIRE(read.get()); + } +} + +} // namespace + +TEST_CASE("runtime codec defaults report unsupported operations", "[store][codec]") +{ + MultiFileCodec codec; + const std::filesystem::path path("12/34/56/78"); + CHECK(codec.paths(path) + == std::vector { + "12/34/56/78.data", + "12/34/56/78.metadata", + }); + + const auto read = codec.read(path); + REQUIRE_FALSE(read.has_value()); + CHECK(read.error().code() == Error::Code::Unsupported); + + const auto write = codec.write(path, 42); + REQUIRE_FALSE(write.has_value()); + CHECK(write.error().code() == Error::Code::Unsupported); +} + +TEST_CASE("runtime write-only codec remains explicitly unreadable", "[store][codec]") +{ + WriteOnlyCodec codec; + REQUIRE(codec.write("node", 42).has_value()); + CHECK(codec.writes == 1); + const auto read = codec.read("node"); + REQUIRE_FALSE(read.has_value()); + CHECK(read.error().code() == Error::Code::Unsupported); +} + +TEST_CASE("runtime mesh codecs are reentrant", "[store][codec]") +{ + TemporaryDirectory directory("mesh-reentrant"); + + SECTION("SF mesh") { check_mesh_codec_reentrancy(mesh::codec::SfMesh {}, directory.path() / "sfmesh"); } + SECTION("binary glTF") { check_mesh_codec_reentrancy(mesh::codec::Gltf(mesh::codec::GltfContainer::Binary), directory.path() / "glb"); } + SECTION("JSON glTF") { check_mesh_codec_reentrancy(mesh::codec::Gltf(mesh::codec::GltfContainer::Json), directory.path() / "gltf"); } +} + +TEST_CASE("runtime glTF codec reports writer IO errors", "[store][codec]") +{ + TemporaryDirectory directory("gltf-error"); + mesh::codec::Gltf codec(mesh::codec::GltfContainer::Binary); + const std::filesystem::path node_path(directory.path() / "blocked"); + REQUIRE(std::filesystem::create_directory(codec.paths(node_path).front())); + + const auto result = codec.write(node_path, triangle_mesh(0.0)); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Io); +} diff --git a/unittests/terrainlib/store_compatibility.cpp b/unittests/terrainlib/store_compatibility.cpp new file mode 100644 index 00000000..7579354e --- /dev/null +++ b/unittests/terrainlib/store_compatibility.cpp @@ -0,0 +1,243 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "../temporary_directory.h" +#include "io/bytes.h" +#include "mesh/SimpleMesh.h" +#include "mesh/codec/SfMeshFormat.h" +#include "mesh/codec/from_extension.h" +#include "mesh/storage.h" +#include "octree/storage/IndexFile.h" +#include "octree/store_layout/Mappings.h" + +namespace { + +using test::TemporaryDirectory; + +mesh::Simple triangle_mesh(const double offset) +{ + return mesh::Simple({ { 0, 1, 2 } }, { { offset, 0.0, 0.0 }, { offset + 1.0, 0.0, 0.0 }, { offset, 1.0, 0.0 } }); +} + +mesh::storage::IndexedStorage make_storage( + const std::filesystem::path& path, const store::path_layout::Mapping mapping, const std::string& extension) +{ + auto codec = mesh::codec::from_extension(extension); + REQUIRE(codec.has_value()); + return mesh::storage::IndexedStorage( + store::RawStorage(store::path_layout::Resolver(path, mapping), std::move(codec.value())), + store::Index {}, + mesh::storage::Storage::Persistence { + octree::storage::index_format(), + path / "octree.storeindex", + std::string(mapping.id), + std::string(mesh::storage::payload_class), + extension, + }); +} + +} // namespace + +TEST_CASE("octree layouts round-trip boundary IDs") +{ + const std::vector ids { + octree::Id::root(), + octree::Id(octree::Id::max_level(), octree::Id::Index { 0 }), + octree::Id(octree::Id::max_level(), octree::Id::max_index_on_level(octree::Id::max_level())), + }; + + const auto flat = octree::store_layout::flat(); + const auto coordinates = octree::store_layout::level_and_coordinate_directories(); + for (const octree::Id id : ids) { + const auto flat_path = flat.key_to_node_path(id); + CHECK(flat.node_path_to_key(flat_path) == id); + + const auto coordinate_path = coordinates.key_to_node_path(id); + CHECK(coordinates.node_path_to_key(coordinate_path) == id); + } +} + +TEST_CASE("versioned octree and SF payload validation is free-standing") +{ + const octree::storage::StoreIndex invalid_index { { { + .level = std::numeric_limits::max(), + .index = 0, + .status = static_cast(store::NodeStatus::Leaf), + } } }; + const auto decoded_index = octree::storage::decode_index(invalid_index); + REQUIRE_FALSE(decoded_index.has_value()); + CHECK(decoded_index.error().code() == Error::Code::CorruptData); + + const mesh::sf::Payload oversized_mesh { + .vertex_count = std::numeric_limits::max(), + .face_count = 0, + .triangles = {}, + .positions = { 0 }, + .uvs = {}, + .texture = {}, + }; + const auto validated_mesh = mesh::sf::validate(oversized_mesh); + REQUIRE_FALSE(validated_mesh.has_value()); + CHECK(validated_mesh.error().code() == Error::Code::InvalidInput); +} + +TEST_CASE("storage hard-links matching payload formats") +{ + TemporaryDirectory source_directory("hard-link-source"); + TemporaryDirectory target_directory("hard-link-target"); + auto source = make_storage(source_directory.path(), octree::store_layout::flat(), ".sfmesh"); + auto target = make_storage(target_directory.path(), octree::store_layout::flat(), ".sfmesh"); + + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, triangle_mesh(0.0)).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(std::filesystem::equivalent(source.path_for(root).value(), target.path_for(root).value())); + REQUIRE(source.save_index().has_value()); + REQUIRE(target.save_index().has_value()); +} + +TEST_CASE("storage re-encodes differing payload formats") +{ + TemporaryDirectory source_directory("reencode-source"); + TemporaryDirectory target_directory("reencode-target"); + auto source = make_storage(source_directory.path(), octree::store_layout::flat(), ".sfmesh"); + auto target = make_storage(target_directory.path(), octree::store_layout::flat(), ".glb"); + + const octree::Id root = octree::Id::root(); + REQUIRE(source.save(root, triangle_mesh(4.0)).has_value()); + REQUIRE(target.copy_from(root, source).has_value()); + CHECK_FALSE(std::filesystem::equivalent(source.path_for(root).value(), target.path_for(root).value())); + const auto loaded = target.load(root); + REQUIRE(loaded.has_value()); + CHECK(loaded->face_count() == 1); + REQUIRE(source.save_index().has_value()); + REQUIRE(target.save_index().has_value()); +} + +TEST_CASE("storage supports enabled overwrites") +{ + TemporaryDirectory directory("overwrite"); + auto storage = make_storage(directory.path(), octree::store_layout::flat(), ".sfmesh"); + storage.settings().allow_overwrite = true; + + const octree::Id root = octree::Id::root(); + REQUIRE(storage.save(root, triangle_mesh(1.0)).has_value()); + REQUIRE(storage.save(root, triangle_mesh(9.0)).has_value()); + const auto loaded = storage.load(root); + REQUIRE(loaded.has_value()); + REQUIRE_FALSE(loaded->positions.empty()); + CHECK(loaded->positions.front().x == 9.0); + REQUIRE(storage.save_index().has_value()); +} + +TEST_CASE("folder opening persists and reopens metadata-driven storage") +{ + TemporaryDirectory directory("folder-open"); + mesh::storage::OpenOptions options; + options.default_mapping = octree::store_layout::flat(); + options.preferred_extension = ".sfmesh"; + + { + auto storage_result = mesh::storage::open_folder(directory.path(), std::move(options)); + REQUIRE(storage_result.has_value()); + auto storage = std::move(storage_result.value()); + CHECK(storage.is_indexed()); + REQUIRE(storage.save(octree::Id::root(), triangle_mesh(2.0)).has_value()); + REQUIRE(storage.save_index().has_value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storemeta")); + } + + { + auto storage_result = mesh::storage::open_folder_indexed(directory.path()); + REQUIRE(storage_result.has_value()); + const auto storage = std::move(storage_result.value()); + CHECK(storage.is_indexed()); + CHECK(storage.has(octree::Id::root()).value()); + CHECK(std::filesystem::is_regular_file(directory.path() / "octree.storeindex")); + } + + const auto reopened = mesh::storage::open_index(directory.path() / "octree.storeindex"); + REQUIRE(reopened.has_value()); + CHECK(reopened->has(octree::Id::root()).value()); + + const auto metadata = octree::storage::read_store_metadata(directory.path()); + REQUIRE(metadata.has_value()); + CHECK(metadata->layout_id == "flat"); + CHECK(metadata->payload_class == mesh::storage::payload_class); + CHECK(metadata->codec_selector == ".sfmesh"); +} + +TEST_CASE("mesh codec resolver dispatches every supported selector", "[store][open]") +{ + for (const std::string extension : { ".sfmesh", ".glb", ".gltf" }) { + const auto codec = mesh::codec::from_extension(extension); + REQUIRE(codec.has_value()); + const auto paths = codec.value()->paths("node"); + REQUIRE_FALSE(paths.empty()); + CHECK(paths.front().extension() == extension); + } + + const auto unknown = mesh::codec::from_extension(".unknown"); + REQUIRE_FALSE(unknown.has_value()); + CHECK(unknown.error().code() == Error::Code::Unsupported); +} + +TEST_CASE("octree opening rejects a mismatched metadata payload class", "[store][open]") +{ + TemporaryDirectory directory("payload-class"); + const std::filesystem::path index_path = directory.path() / "octree.storeindex"; + const store::IndexMetadata index_file { + .index = {}, + .layout_id = "flat", + .payload_class = "dag.ClusterBatch", + .codec_selector = ".sfmesh", + }; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); + + const auto result = mesh::storage::open_index(index_path); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Unsupported); +} + +TEST_CASE("octree opening retains index failures without directory fallback", "[store][open]") +{ + TemporaryDirectory directory("open-errors"); + const std::filesystem::path index_path = directory.path() / "octree.storeindex"; + + store::IndexMetadata index_file { + .index = {}, + .layout_id = "unknown-layout", + .payload_class = std::string(mesh::storage::payload_class), + .codec_selector = ".sfmesh", + }; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); + + const auto unknown_layout = mesh::storage::open_folder_indexed(directory.path()); + REQUIRE_FALSE(unknown_layout.has_value()); + CHECK(unknown_layout.error().code() == Error::Code::Unsupported); + CHECK(unknown_layout.error().to_string().contains("unknown-layout")); + + index_file.layout_id = "flat"; + index_file.codec_selector = ".unknown"; + REQUIRE(octree::storage::write_index_file(index_path, index_file).has_value()); + const auto unknown_codec = mesh::storage::open_folder_indexed(directory.path()); + REQUIRE_FALSE(unknown_codec.has_value()); + CHECK(unknown_codec.error().code() == Error::Code::Unsupported); + + const std::array malformed_bytes { 0xff, 0x00, 0x01 }; + REQUIRE(io::write_bytes_to_path(malformed_bytes, index_path).has_value()); + const auto malformed = mesh::storage::open_folder_indexed(directory.path()); + REQUIRE_FALSE(malformed.has_value()); + CHECK(malformed.error().code() == Error::Code::CorruptData); + + const auto independent_metadata = octree::storage::read_store_metadata(directory.path()); + REQUIRE(independent_metadata.has_value()); + CHECK(independent_metadata->codec_selector == ".unknown"); +} diff --git a/unittests/terrainlib/store_index.cpp b/unittests/terrainlib/store_index.cpp new file mode 100644 index 00000000..e3a20065 --- /dev/null +++ b/unittests/terrainlib/store_index.cpp @@ -0,0 +1,80 @@ +#include +#include + +#include +#include + +#include "octree/StoreTraits.h" +#include "raster_store/StoreTraits.h" +#include "store/Index.h" + +static_assert(store::HierarchyTraits); +static_assert(store::HierarchyTraits); + +TEMPLATE_TEST_CASE("shared hierarchy index transitions", "[store][index]", octree::StoreTraits, raster_store::StoreTraits) +{ + using Traits = TestType; + using Index = store::Index; + + Index index; + const auto root = Traits::root(); + const auto children = Traits::children(root).value(); + const auto first = children[0]; + const auto second = children[1]; + + REQUIRE(index.add(first).value()); + CHECK(index.is(store::NodeStatus::Virtual, root).value()); + CHECK(index.is(store::NodeStatus::Leaf, first).value()); + + REQUIRE(index.add(root).value()); + CHECK(index.is(store::NodeStatus::Inner, root).value()); + CHECK_FALSE(index.add(root).value()); + + REQUIRE(index.add(second).value()); + REQUIRE(index.remove(first).value()); + CHECK(index.is(store::NodeStatus::Inner, root).value()); + REQUIRE(index.remove(second).value()); + CHECK(index.is(store::NodeStatus::Leaf, root).value()); + REQUIRE(index.remove(root).value()); + CHECK(index.empty()); +} + +TEST_CASE("raster store traits validate tile boundaries", "[store][index][raster]") +{ + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + + CHECK(Traits::is_valid(Key { 0, { 0, 0 } })); + CHECK_FALSE(Traits::is_valid(Key { 0, { 1, 0 } })); + CHECK(Traits::is_valid(Key { 31, { uint32_t { 0x7fffffff }, uint32_t { 0x7fffffff } } })); + CHECK_FALSE(Traits::is_valid(Key { 31, { uint32_t { 0x80000000 }, 0 } })); + CHECK(Traits::is_valid(Key { 32, { 0, 0 } })); + CHECK(Traits::is_valid(Key { 32, { UINT32_MAX, UINT32_MAX } })); + CHECK_FALSE(Traits::is_valid(Key { 33, { 0, 0 } })); + + CHECK_FALSE(Traits::parent(Traits::root()).has_value()); + CHECK_FALSE(Traits::children(Key { 32, { UINT32_MAX, UINT32_MAX } }).has_value()); + + const Key level_31 { 31, { uint32_t { 0x7fffffff }, uint32_t { 0x7fffffff } } }; + const auto children = Traits::children(level_31); + REQUIRE(children.has_value()); + CHECK(children->at(0) == Key { 32, { UINT32_MAX - 1, UINT32_MAX - 1 } }); + CHECK(children->at(1) == Key { 32, { UINT32_MAX, UINT32_MAX - 1 } }); + CHECK(children->at(2) == Key { 32, { UINT32_MAX - 1, UINT32_MAX } }); + CHECK(children->at(3) == Key { 32, { UINT32_MAX, UINT32_MAX } }); +} + +TEST_CASE("shared index rejects invalid keys", "[store][index][raster]") +{ + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + store::Index index; + const Key invalid { 4, { 16, 0 } }; + + CHECK(index.get(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.add(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.remove(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.is_present(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.is_absent(invalid).error().code() == Error::Code::InvalidInput); + CHECK(index.is(store::NodeStatus::Leaf, invalid).error().code() == Error::Code::InvalidInput); +} diff --git a/unittests/terrainlib/store_layout.cpp b/unittests/terrainlib/store_layout.cpp new file mode 100644 index 00000000..9a0cc837 --- /dev/null +++ b/unittests/terrainlib/store_layout.cpp @@ -0,0 +1,70 @@ +#include + +#include + +#include "mesh/codec/Gltf.h" +#include "mesh/codec/SfMesh.h" +#include "octree/store_layout/Mappings.h" +#include "store/path_layout.h" + +TEST_CASE("extensionless octree mappings preserve physical paths", "[store][layout]") +{ + const std::filesystem::path dataset = "dataset"; + mesh::codec::SfMesh sfmesh; + + SECTION("flat") + { + const store::path_layout::Resolver layout(dataset / "sf-flat", octree::store_layout::flat()); + const std::filesystem::path node_path = layout.node_path(octree::Id::root()); + CHECK(node_path == dataset / "sf-flat/0-0"); + CHECK(sfmesh.paths(node_path) == std::vector { dataset / "sf-flat/0-0.sfmesh" }); + CHECK(layout.key_from_node_path(node_path) == octree::Id::root()); + } + + SECTION("level and coordinate directories") + { + const store::path_layout::Resolver layout(dataset / "sf-coordinates", octree::store_layout::level_and_coordinate_directories()); + const octree::Id child = octree::Id::root().child(2).value(); + const octree::Id deep = octree::Id::root().child(5).value().child(7).value(); + CHECK(sfmesh.paths(layout.node_path(child)) == std::vector { dataset / "sf-coordinates/1/0/1/0.sfmesh" }); + CHECK(sfmesh.paths(layout.node_path(deep)) == std::vector { dataset / "sf-coordinates/2/3/1/3.sfmesh" }); + CHECK(layout.key_from_node_path(layout.node_path(child)) == child); + CHECK(layout.key_from_node_path(layout.node_path(deep)) == deep); + } +} + +TEST_CASE("octree mapping lookup is explicit", "[store][layout]") +{ + REQUIRE(octree::store_layout::from_id("flat").has_value()); + CHECK(octree::store_layout::from_id("flat")->id == "flat"); + REQUIRE(octree::store_layout::from_id("level_and_coordinate_directories").has_value()); + CHECK(octree::store_layout::from_id("level_and_coordinate_directories")->id == "level_and_coordinate_directories"); + CHECK_FALSE(octree::store_layout::from_id("unknown").has_value()); + CHECK(octree::store_layout::all().size() == 2); +} + +TEST_CASE("extensionless octree mappings validate complete paths", "[store][layout]") +{ + const auto flat = octree::store_layout::flat(); + CHECK_FALSE(flat.node_path_to_key("1-2.sfmesh").has_value()); + CHECK_FALSE(flat.node_path_to_key("nested/1-2").has_value()); + CHECK_FALSE(flat.node_path_to_key("1-2-extra").has_value()); + + const auto coordinates = octree::store_layout::level_and_coordinate_directories(); + CHECK_FALSE(coordinates.node_path_to_key("1/0/0/0.sfmesh").has_value()); + CHECK_FALSE(coordinates.node_path_to_key("1/0/0").has_value()); + CHECK_FALSE(coordinates.node_path_to_key("1/0/0/0/extra").has_value()); + CHECK_FALSE(coordinates.node_path_to_key("22/0/0/0").has_value()); +} + +TEST_CASE("runtime mesh codecs own their filename endings", "[store][layout][codec]") +{ + const std::filesystem::path path("12/34/56/78"); + mesh::codec::SfMesh sfmesh; + mesh::codec::Gltf binary(mesh::codec::GltfContainer::Binary); + mesh::codec::Gltf json(mesh::codec::GltfContainer::Json); + + CHECK(sfmesh.paths(path) == std::vector { "12/34/56/78.sfmesh" }); + CHECK(binary.paths(path) == std::vector { "12/34/56/78.glb" }); + CHECK(json.paths(path) == std::vector { "12/34/56/78.gltf" }); +} diff --git a/unittests/terrainlib/store_storage.cpp b/unittests/terrainlib/store_storage.cpp new file mode 100644 index 00000000..a70c1638 --- /dev/null +++ b/unittests/terrainlib/store_storage.cpp @@ -0,0 +1,482 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "../temporary_directory.h" +#include "io/bytes.h" +#include "octree/StoreTraits.h" +#include "octree/store_layout/Mappings.h" +#include "raster_store/StoreTraits.h" +#include "store/IndexedStorage.h" +#include "string_utils.h" + +namespace { + +using test::TemporaryDirectory; + +std::filesystem::path tile_to_path(const radix::tile::Id& key) +{ + return std::to_string(key.zoom_level) + "/" + std::to_string(key.coords.x) + "/" + std::to_string(key.coords.y); +} + +std::optional path_to_tile(const std::filesystem::path& node_path) +{ + if (node_path.is_absolute() || std::distance(node_path.begin(), node_path.end()) != 3) { + return std::nullopt; + } + auto part = node_path.begin(); + const auto zoom = from_chars(part->string()); + ++part; + const auto x = from_chars(part->string()); + ++part; + const auto y = from_chars(part->string()); + if (!zoom.has_value() || !x.has_value() || !y.has_value()) { + return std::nullopt; + } + const radix::tile::Id key { zoom.value(), { x.value(), y.value() } }; + return raster_store::StoreTraits::is_valid(key) ? std::optional(key) : std::nullopt; +} + +template +store::path_layout::Mapping test_mapping(); + +template <> +store::path_layout::Mapping test_mapping() +{ + return octree::store_layout::flat(); +} + +template <> +store::path_layout::Mapping test_mapping() +{ + return { "test_tiles", tile_to_path, path_to_tile }; +} + +Expected read_int(const std::filesystem::path& path) +{ + auto bytes = io::read_bytes_from_path(path); + if (!bytes) { + return Error::propagate(std::move(bytes), "read test integer"); + } + if (bytes->size() != sizeof(int)) { + return Error::fail(Error::Code::CorruptData, "test integer has the wrong byte count"); + } + int value; + std::memcpy(&value, bytes->data(), sizeof(value)); + return value; +} + +Expected write_int(const int value, const std::filesystem::path& path) +{ + const auto bytes = std::span(reinterpret_cast(&value), sizeof(value)); + auto written = io::write_bytes_to_path(bytes, path); + if (!written) { + return Error::propagate(std::move(written), "write test integer"); + } + return {}; +} + +class IntCodec final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override { return { add_extension(node_path, ".data") }; } + + Expected read(const std::filesystem::path& node_path) const override { return read_int(paths(node_path).front()); } + + Expected write(const std::filesystem::path& node_path, const int& value) const override { return write_int(value, paths(node_path).front()); } +}; + +template +store::Storage make_storage(const std::filesystem::path& path) +{ + return store::Storage( + store::RawStorage(store::path_layout::Resolver(path, test_mapping()), std::make_unique())); +} + +class MultiFileCodec final : public store::Codec { +public: + std::vector paths(const std::filesystem::path& node_path) const override + { + return { add_extension(node_path, ".data"), add_extension(node_path, ".metadata") }; + } + + Expected read(const std::filesystem::path& node_path) const override + { + auto value = read_int(paths(node_path).front()); + if (!value) { + return Error::propagate(std::move(value), "read multi-file test value"); + } + return value.value(); + } + + Expected write(const std::filesystem::path& node_path, const int& value) const override + { + const auto node_paths = paths(node_path); + if (!write_int(value, node_paths[0]) || !write_int(value + 1, node_paths[1])) { + return Error::fail(Error::Code::Io, "multi-file test write failed"); + } + return {}; + } +}; + +class ConfigurableCodec final : public store::Codec { +public: + explicit ConfigurableCodec(std::string extension) + : extension(std::move(extension)) + { + } + + std::vector paths(const std::filesystem::path& node_path) const override { return { add_extension(node_path, extension) }; } + + Expected read(const std::filesystem::path& node_path) const override + { + if (fail_read) { + return Error::fail(Error::Code::CorruptData, "injected decode failure"); + } + auto value = read_int(paths(node_path).front()); + if (!value) { + return Error::propagate(std::move(value), "read configurable test value"); + } + return value.value(); + } + + Expected write(const std::filesystem::path& node_path, const int& value) const override + { + if (fail_write) { + return Error::fail(Error::Code::InvalidInput, "injected encode failure"); + } + if (!write_int(value, paths(node_path).front())) { + return Error::fail(Error::Code::Io, "configurable test write failed"); + } + return {}; + } + + std::string extension; + bool fail_read = false; + bool fail_write = false; +}; + +std::atomic_uint32_t index_writes = 0; + +Expected> unused_index_read(const std::filesystem::path& path) +{ + return Error::fail(Error::Code::NotFound, "read unused test index from", path); +} + +Expected count_index_write(const std::filesystem::path&, const store::IndexMetadata&) +{ + ++index_writes; + return {}; +} + +std::optional> fake_mapping_from_id(const std::string_view id) +{ + return id == "flat" ? std::optional>(octree::store_layout::flat()) : std::nullopt; +} + +store::path_layout::Mapping fake_default_mapping() { return octree::store_layout::flat(); } + +store::IndexFormat counting_index_format() +{ + return { + "octree.storeindex", + unused_index_read, + count_index_write, + fake_mapping_from_id, + fake_default_mapping, + }; +} + +store::IndexedStorage make_indexed_storage( + const std::filesystem::path& path, std::unique_ptr> codec = std::make_unique()) +{ + using Storage = store::Storage; + return store::IndexedStorage( + store::RawStorage(store::path_layout::Resolver(path, octree::store_layout::flat()), std::move(codec)), + store::Index {}, + Storage::Persistence { + counting_index_format(), + path / "octree.storeindex", + "flat", + "test.Int", + ".data", + }); +} + +} // namespace + +TEST_CASE("byte writes report directory creation errors", "[io][bytes]") +{ + TemporaryDirectory directory("write-parent-error"); + const std::filesystem::path blocker = directory.path() / "blocker"; + REQUIRE(write_int(1, blocker).has_value()); + + const auto result = io::write_bytes_to_path(std::span {}, blocker / "payload"); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Io); +} + +TEMPLATE_TEST_CASE("shared raw storage has no hidden octree dependency", "[store][storage]", octree::StoreTraits, raster_store::StoreTraits) +{ + using Traits = TestType; + TemporaryDirectory directory("traits"); + auto storage = make_storage(directory.path()); + const auto root = Traits::root(); + + CHECK_FALSE(storage.has(root).value()); + REQUIRE(storage.save(root, 42).has_value()); + CHECK(storage.has(root).value()); + REQUIRE(storage.load(root).has_value()); + CHECK(storage.load(root).value() == 42); + REQUIRE(storage.remove(root).has_value()); + CHECK_FALSE(storage.has(root).value()); +} + +TEST_CASE("shared storage rejects invalid 2D keys", "[store][storage]") +{ + TemporaryDirectory directory("invalid-key"); + auto storage = make_storage(directory.path()); + const radix::tile::Id invalid { 3, { 8, 0 } }; + + CHECK(storage.load(invalid).error().code() == Error::Code::InvalidInput); + CHECK(storage.save(invalid, 1).error().code() == Error::Code::InvalidInput); + CHECK(storage.has(invalid).error().code() == Error::Code::InvalidInput); + CHECK(storage.remove(invalid).error().code() == Error::Code::InvalidInput); +} + +TEST_CASE("shared storage preserves overwrite settings", "[store][storage]") +{ + TemporaryDirectory directory("overwrite"); + auto storage = make_storage(directory.path()); + const octree::Id root = octree::Id::root(); + + REQUIRE(storage.save(root, 1).has_value()); + const auto rejected = storage.save(root, 2); + REQUIRE_FALSE(rejected.has_value()); + CHECK(rejected.error().code() == Error::Code::AlreadyExists); + CHECK(storage.load(root).value() == 1); + + storage.settings().allow_overwrite = true; + REQUIRE(storage.save(root, 2).has_value()); + CHECK(storage.load(root).value() == 2); +} + +TEST_CASE("raw storage requires and removes every codec path", "[store][storage]") +{ + TemporaryDirectory directory("multi-file"); + store::RawStorage storage( + store::path_layout::Resolver(directory.path(), octree::store_layout::flat()), std::make_unique()); + const octree::Id root = octree::Id::root(); + const auto paths = storage.paths(root).value(); + REQUIRE(paths.size() == 2); + + REQUIRE(write_int(1, paths[0]).has_value()); + CHECK_FALSE(storage.has(root).value()); + REQUIRE(write_int(2, paths[1]).has_value()); + CHECK(storage.has(root).value()); + REQUIRE(storage.remove(root).value()); + CHECK_FALSE(std::filesystem::exists(paths[0])); + CHECK_FALSE(std::filesystem::exists(paths[1])); +} + +TEST_CASE("storage moves transfer and finalize dirty index state", "[store][storage]") +{ + index_writes = 0; + TemporaryDirectory first_directory("move-first"); + TemporaryDirectory second_directory("move-second"); + + { + auto original = make_indexed_storage(first_directory.path()); + REQUIRE(original.save(octree::Id::root(), 1).has_value()); + auto moved = std::move(original); + CHECK(index_writes == 0); + REQUIRE(moved.save_index().has_value()); + CHECK(index_writes == 1); + } + CHECK(index_writes == 1); + + { + auto destination = make_indexed_storage(first_directory.path()); + destination.settings().allow_overwrite = true; + REQUIRE(destination.save(octree::Id::root(), 2).has_value()); + auto source = make_indexed_storage(second_directory.path()); + REQUIRE(source.save(octree::Id::root(), 3).has_value()); + destination = std::move(source); + CHECK(index_writes == 2); + } + CHECK(index_writes == 3); +} + +TEST_CASE("copy_from hard-links every matching codec file", "[store][storage][copy]") +{ + const octree::Id root = octree::Id::root(); + + SECTION("one file") + { + TemporaryDirectory source_directory("copy-one-source"); + TemporaryDirectory target_directory("copy-one-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 41).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + REQUIRE(target.has(root).value()); + CHECK(target.load(root).value() == 41); + CHECK(std::filesystem::equivalent(source.paths(root)->front(), target.paths(root)->front())); + } + + SECTION("multiple files") + { + TemporaryDirectory source_directory("copy-many-source"); + TemporaryDirectory target_directory("copy-many-target"); + auto source = make_indexed_storage(source_directory.path(), std::make_unique()); + auto target = make_indexed_storage(target_directory.path(), std::make_unique()); + REQUIRE(source.save(root, 42).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + const auto source_paths = source.paths(root).value(); + const auto target_paths = target.paths(root).value(); + REQUIRE(source_paths.size() == 2); + REQUIRE(target_paths.size() == 2); + CHECK(std::filesystem::equivalent(source_paths[0], target_paths[0])); + CHECK(std::filesystem::equivalent(source_paths[1], target_paths[1])); + } + + SECTION("same storage") + { + TemporaryDirectory directory("copy-self"); + auto storage = make_indexed_storage(directory.path()); + REQUIRE(storage.save(root, 43).has_value()); + storage.settings().allow_overwrite = true; + + REQUIRE(storage.copy_from(root, storage).has_value()); + CHECK(storage.load(root).value() == 43); + } +} + +TEST_CASE("copy_from re-encodes differing path lists", "[store][storage][copy]") +{ + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-reencode-source"); + TemporaryDirectory target_directory("copy-reencode-target"); + auto source = make_indexed_storage(source_directory.path(), std::make_unique()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 43).has_value()); + + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(target.load(root).value() == 43); + CHECK(target.paths(root)->size() == 1); + CHECK(source.paths(root)->size() == 2); + CHECK_FALSE(std::filesystem::equivalent(source.paths(root)->front(), target.paths(root)->front())); +} + +TEST_CASE("copy_from enforces overwrite settings", "[store][storage][copy]") +{ + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-overwrite-source"); + TemporaryDirectory target_directory("copy-overwrite-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(source.save(root, 44).has_value()); + REQUIRE(target.save(root, 1).has_value()); + + const auto rejected = target.copy_from(root, source); + REQUIRE_FALSE(rejected.has_value()); + CHECK(rejected.error().code() == Error::Code::AlreadyExists); + CHECK(target.load(root).value() == 1); + + target.settings().allow_overwrite = true; + REQUIRE(target.copy_from(root, source).has_value()); + CHECK(target.load(root).value() == 44); + CHECK(std::filesystem::equivalent(source.paths(root)->front(), target.paths(root)->front())); +} + +TEST_CASE("failed multi-file overwrite leaves the target node unindexed", "[store][storage][copy]") +{ + const octree::Id root = octree::Id::root(); + TemporaryDirectory source_directory("copy-partial-source"); + TemporaryDirectory target_directory("copy-partial-target"); + auto source = make_indexed_storage(source_directory.path(), std::make_unique()); + auto target = make_indexed_storage(target_directory.path(), std::make_unique()); + REQUIRE(source.save(root, 45).has_value()); + REQUIRE(target.save(root, 1).has_value()); + target.settings().allow_overwrite = true; + + const auto target_paths = target.paths(root).value(); + REQUIRE(std::filesystem::remove(target_paths[1])); + REQUIRE(std::filesystem::create_directory(target_paths[1])); + REQUIRE(write_int(9, target_paths[1] / "blocker.data").has_value()); + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(copied.error().code() == Error::Code::Io); + CHECK_FALSE(target.has(root).value()); + CHECK_FALSE(target.index().get(root).value().has_value()); + CHECK(std::filesystem::equivalent(source.paths(root)->front(), target_paths.front())); + CHECK(std::filesystem::is_directory(target_paths[1])); +} + +TEST_CASE("copy_from propagates source and codec failures", "[store][storage][copy]") +{ + const octree::Id root = octree::Id::root(); + + SECTION("missing source") + { + TemporaryDirectory source_directory("copy-missing-source"); + TemporaryDirectory target_directory("copy-missing-target"); + auto source = make_indexed_storage(source_directory.path()); + auto target = make_indexed_storage(target_directory.path()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(copied.error().code() == Error::Code::NotFound); + CHECK(target.has(root).value()); + CHECK(target.load(root).value() == 7); + } + + SECTION("decode failure") + { + TemporaryDirectory source_directory("copy-decode-source"); + TemporaryDirectory target_directory("copy-decode-target"); + auto source_codec = std::make_unique(".source"); + ConfigurableCodec* source_codec_ptr = source_codec.get(); + auto source = make_indexed_storage(source_directory.path(), std::move(source_codec)); + auto target = make_indexed_storage(target_directory.path(), std::make_unique(".target")); + REQUIRE(source.save(root, 46).has_value()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + source_codec_ptr->fail_read = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(copied.error().code() == Error::Code::CorruptData); + CHECK(target.has(root).value()); + CHECK(target.load(root).value() == 7); + } + + SECTION("encode failure") + { + TemporaryDirectory source_directory("copy-encode-source"); + TemporaryDirectory target_directory("copy-encode-target"); + auto source = make_indexed_storage(source_directory.path(), std::make_unique(".source")); + auto target_codec = std::make_unique(".target"); + ConfigurableCodec* target_codec_ptr = target_codec.get(); + auto target = make_indexed_storage(target_directory.path(), std::move(target_codec)); + REQUIRE(source.save(root, 47).has_value()); + REQUIRE(target.save(root, 7).has_value()); + target.settings().allow_overwrite = true; + target_codec_ptr->fail_write = true; + + const auto copied = target.copy_from(root, source); + REQUIRE_FALSE(copied.has_value()); + CHECK(copied.error().code() == Error::Code::InvalidInput); + CHECK_FALSE(target.has(root).value()); + } +} diff --git a/unittests/terrainlib/store_traverse.cpp b/unittests/terrainlib/store_traverse.cpp new file mode 100644 index 00000000..465867d6 --- /dev/null +++ b/unittests/terrainlib/store_traverse.cpp @@ -0,0 +1,55 @@ +#include + +#include +#include + +#include "octree/StoreTraits.h" +#include "raster_store/StoreTraits.h" +#include "store/traverse.h" + +TEMPLATE_TEST_CASE("shared hierarchy traversal preserves trait child order", "[store][traverse]", octree::StoreTraits, raster_store::StoreTraits) +{ + using Traits = TestType; + using Key = typename Traits::Key; + + store::Index index; + const Key root = Traits::root(); + const auto children = Traits::children(root).value(); + const Key first = children[0]; + const Key second = children[1]; + const Key grandchild = Traits::children(first).value()[0]; + REQUIRE(index.add(root).has_value()); + REQUIRE(index.add(first).has_value()); + REQUIRE(index.add(second).has_value()); + REQUIRE(index.add(grandchild).has_value()); + + std::vector depth_first; + REQUIRE(store::traverse(index, [&](const Key& key, const store::NodeStatus) { depth_first.push_back(key); }).has_value()); + CHECK(depth_first == std::vector { root, first, grandchild, second }); + + std::vector breadth_first; + REQUIRE(store::traverse( + index, + [&](const Key& key, const store::NodeStatus) { breadth_first.push_back(key); }, + store::AlwaysRefine {}, + root, + store::TraversalOrder::BreadthFirst) + .has_value()); + CHECK(breadth_first == std::vector { root, first, second, grandchild }); + + std::vector explicit_root; + REQUIRE(store::traverse(index, [&](const Key& key, const store::NodeStatus) { explicit_root.push_back(key); }, store::AlwaysRefine {}, first).has_value()); + CHECK(explicit_root == std::vector { first, grandchild }); +} + +TEST_CASE("shared traversal rejects an invalid explicit root", "[store][traverse][raster]") +{ + using Traits = raster_store::StoreTraits; + using Key = Traits::Key; + const Key invalid { 1, { 2, 0 } }; + store::Index index; + + const auto result = store::traverse(index, [](const Key&, const store::NodeStatus) { }, store::AlwaysRefine {}, invalid); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); +} diff --git a/unittests/terrainlib/string_utils.cpp b/unittests/terrainlib/string_utils.cpp index c54490a4..9ef3699a 100644 --- a/unittests/terrainlib/string_utils.cpp +++ b/unittests/terrainlib/string_utils.cpp @@ -21,7 +21,7 @@ TEST_CASE("from_chars rejects non-numeric input", "[string_utils]") { } TEST_CASE("from_chars rejects trailing garbage after a valid number", "[string_utils]") { - CHECK(from_chars(std::string_view("123.terrain")) == std::nullopt); + CHECK(from_chars(std::string_view("123.sfmesh")) == std::nullopt); CHECK(from_chars(std::string_view("123abc")) == std::nullopt); } diff --git a/unittests/tile_downloader/downloader.cpp b/unittests/tile_downloader/downloader.cpp index 083ecd49..a2dcd67c 100644 --- a/unittests/tile_downloader/downloader.cpp +++ b/unittests/tile_downloader/downloader.cpp @@ -11,55 +11,52 @@ namespace { class TemporaryPyramid { public: explicit TemporaryPyramid(std::string_view name) - : _path(std::filesystem::temp_directory_path() / name) { + : m_path(std::filesystem::temp_directory_path() / name) + { std::error_code error; - std::filesystem::remove_all(_path, error); - std::filesystem::create_directories(_path); + std::filesystem::remove_all(m_path, error); + std::filesystem::create_directories(m_path); } - ~TemporaryPyramid() { + ~TemporaryPyramid() + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); } - [[nodiscard]] const std::filesystem::path &path() const { - return _path; - } + [[nodiscard]] const std::filesystem::path& path() const { return m_path; } - [[nodiscard]] std::filesystem::path tile_path(const radix::tile::Id &tile) const { - return google_tile_path(_path, tile, ".jpeg"); - } + [[nodiscard]] std::filesystem::path tile_path(const radix::tile::Id& tile) const { return google_tile_path(m_path, tile, ".jpeg"); } - void create_pending(const radix::tile::Id &tile) const { + void create_pending(const radix::tile::Id& tile) const + { const auto path = tile_path(tile); std::filesystem::create_directories(path.parent_path()); - write_file_children_pending(path, std::vector{'t', 'i', 'l', 'e'}); + write_file_children_pending(path, std::vector { 't', 'i', 'l', 'e' }); } - void create_complete(const radix::tile::Id &tile) const { + void create_complete(const radix::tile::Id& tile) const + { create_pending(tile); mark_tile_children_complete(tile_path(tile)); } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -const TileUrlBuilder missing_file_url({ - "file:///definitely-missing-atb-tile/{zoom}/{x}/{y}.jpeg", - TileYDirection::Down -}); +const TileUrlBuilder missing_file_url({ "file:///definitely-missing-atb-tile/{zoom}/{x}/{y}.jpeg", TileYDirection::Down }); -} +} // namespace TEST_CASE("tile downloader promotes parents after completed children") { const TemporaryPyramid pyramid("atb-downloader-complete-pyramid"); - const radix::tile::Id root{0, {0, 0}}; + const radix::tile::Id root { 0, { 0, 0 } }; const auto children = root.children(); pyramid.create_pending(root); - for (const auto &child : children) { + for (const auto& child : children) { pyramid.create_pending(child); } @@ -69,7 +66,7 @@ TEST_CASE("tile downloader promotes parents after completed children") REQUIRE(std::filesystem::exists(pyramid.tile_path(root))); CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(root)))); - for (const auto &child : children) { + for (const auto& child : children) { CHECK(std::filesystem::exists(pyramid.tile_path(child))); CHECK_FALSE(std::filesystem::exists(children_pending_tile_path(pyramid.tile_path(child)))); } @@ -83,7 +80,7 @@ TEST_CASE("tile downloader promotes parents after completed children") TEST_CASE("tile downloader leaves ancestors pending after a child failure") { const TemporaryPyramid pyramid("atb-downloader-failed-pyramid"); - const radix::tile::Id root{0, {0, 0}}; + const radix::tile::Id root { 0, { 0, 0 } }; const auto children = root.children(); pyramid.create_pending(root); diff --git a/unittests/tile_downloader/http_client.cpp b/unittests/tile_downloader/http_client.cpp index 61311a8c..7b036cf6 100644 --- a/unittests/tile_downloader/http_client.cpp +++ b/unittests/tile_downloader/http_client.cpp @@ -12,39 +12,39 @@ namespace { class TemporaryFile { public: TemporaryFile() - : _path(std::filesystem::temp_directory_path() / "atb-http-client-test.txt") { - std::ofstream output(_path, std::ios::binary); + : m_path(std::filesystem::temp_directory_path() / "atb-http-client-test.txt") + { + std::ofstream output(m_path, std::ios::binary); output << "response body"; REQUIRE(output); } - ~TemporaryFile() { + ~TemporaryFile() + { std::error_code error; - std::filesystem::remove(_path, error); + std::filesystem::remove(m_path, error); } - [[nodiscard]] std::string url() const { - return "file://" + _path.string(); - } + [[nodiscard]] std::string url() const { return "file://" + m_path.string(); } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; class CallbackError : public std::runtime_error { public: CallbackError() - : std::runtime_error("callback failed") {} + : std::runtime_error("callback failed") + { + } }; -} +} // namespace TEST_CASE("http client propagates response writer exceptions") { const TemporaryFile source; - HttpClient client([](std::vector &, const char *, size_t) { - throw CallbackError(); - }); + HttpClient client([](std::vector&, const char*, size_t) { throw CallbackError(); }); CHECK_THROWS_AS(client.get(source.url()), CallbackError); } @@ -54,9 +54,5 @@ TEST_CASE("http client propagates progress callback exceptions") const TemporaryFile source; HttpClient client; - CHECK_THROWS_AS( - client.get(source.url(), [](double) { - throw CallbackError(); - }), - CallbackError); + CHECK_THROWS_AS(client.get(source.url(), [](double) { throw CallbackError(); }), CallbackError); } diff --git a/unittests/tile_downloader/logger.cpp b/unittests/tile_downloader/logger.cpp index c41ecb4c..5a31d33a 100644 --- a/unittests/tile_downloader/logger.cpp +++ b/unittests/tile_downloader/logger.cpp @@ -28,6 +28,6 @@ TEST_CASE("tile logger session can finish normally") TileLogger logger(0); auto session = logger.start(); - logger.skipped(radix::tile::Id{0, {0, 0}}); + logger.skipped(radix::tile::Id { 0, { 0, 0 } }); session.finish(); } diff --git a/unittests/tile_downloader/write_file.cpp b/unittests/tile_downloader/write_file.cpp index 2773c4ed..07c12ebb 100644 --- a/unittests/tile_downloader/write_file.cpp +++ b/unittests/tile_downloader/write_file.cpp @@ -14,42 +14,38 @@ namespace { class TemporaryOutput { public: explicit TemporaryOutput(std::string_view name) - : _path(std::filesystem::temp_directory_path() / name) { + : m_path(std::filesystem::temp_directory_path() / name) + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); std::filesystem::remove(partial_path(), error); std::filesystem::remove(pending_path(), error); } - ~TemporaryOutput() { + ~TemporaryOutput() + { std::error_code error; - std::filesystem::remove_all(_path, error); + std::filesystem::remove_all(m_path, error); std::filesystem::remove(partial_path(), error); std::filesystem::remove(pending_path(), error); } - [[nodiscard]] const std::filesystem::path &path() const { - return _path; - } + [[nodiscard]] const std::filesystem::path& path() const { return m_path; } - [[nodiscard]] std::filesystem::path partial_path() const { - return partial_tile_path(_path); - } + [[nodiscard]] std::filesystem::path partial_path() const { return partial_tile_path(m_path); } - [[nodiscard]] std::filesystem::path pending_path() const { - return children_pending_tile_path(_path); - } + [[nodiscard]] std::filesystem::path pending_path() const { return children_pending_tile_path(m_path); } private: - std::filesystem::path _path; + std::filesystem::path m_path; }; -} +} // namespace TEST_CASE("checked file writer persists the complete response") { const TemporaryOutput output("atb-write-file-success.bin"); - const std::vector expected{'t', 'i', 'l', 'e'}; + const std::vector expected { 't', 'i', 'l', 'e' }; write_file_children_pending(output.path(), expected); @@ -59,7 +55,7 @@ TEST_CASE("checked file writer persists the complete response") std::ifstream input(output.pending_path(), std::ios::binary); const auto begin = std::istreambuf_iterator(input); - const std::vector actual(begin, std::istreambuf_iterator{}); + const std::vector actual(begin, std::istreambuf_iterator {}); CHECK(actual == expected); mark_tile_children_complete(output.path()); @@ -71,11 +67,9 @@ TEST_CASE("checked file writer preserves the final path when promotion fails") { const TemporaryOutput output("atb-write-file-promotion-failure"); REQUIRE(std::filesystem::create_directory(output.path())); - write_file_children_pending(output.path(), std::vector{'t', 'i', 'l', 'e'}); + write_file_children_pending(output.path(), std::vector { 't', 'i', 'l', 'e' }); - CHECK_THROWS_AS( - mark_tile_children_complete(output.path()), - std::filesystem::filesystem_error); + CHECK_THROWS_AS(mark_tile_children_complete(output.path()), std::filesystem::filesystem_error); CHECK(std::filesystem::is_directory(output.path())); CHECK(std::filesystem::exists(output.pending_path())); diff --git a/unittests/tilebuilder/dataset_reading.cpp b/unittests/tilebuilder/dataset_reading.cpp index ba2804db..2b505789 100644 --- a/unittests/tilebuilder/dataset_reading.cpp +++ b/unittests/tilebuilder/dataset_reading.cpp @@ -122,7 +122,7 @@ TEST_CASE("reading") const auto s = std::string("/austria/").length(); const auto l = dataset_name.length() - std::string(".tif").length() - s; const auto debug_file = fmt::format("./heights_{}_srs{}_{}.png", test_name, test_srs, dataset_name.substr(s, l)); - image::debugOut(heights, debug_file); + REQUIRE(image::debug_out(heights, debug_file).has_value()); } const auto heights = reader.read(srs_bounds, 100, 111); @@ -170,8 +170,9 @@ TEST_CASE("reading") require_projection_available(*ref_dataset, geodetic_srs); const auto ref_reader = DatasetReader(ref_dataset, geodetic_srs, 1); const auto ref_heights = ref_reader.read(ref_bounds, render_width, render_height); - if (ALP_UNITTESTS_DEBUG_IMAGES) - image::debugOut(ref_heights, fmt::format("./heights_ref.png")); + if (ALP_UNITTESTS_DEBUG_IMAGES) { + REQUIRE(image::debug_out(ref_heights, fmt::format("./heights_ref.png")).has_value()); + } for (std::string dataset_name : datasets) { const auto dataset = Dataset::open_shared_raster(ALP_TEST_DATA_DIR + std::string(dataset_name)).value(); @@ -183,12 +184,12 @@ TEST_CASE("reading") const auto l = dataset_name.length() - std::string(".tif").length() - s; if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debugOut(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))); + REQUIRE(image::debug_out(ref_heights, fmt::format("./heights_{}_{}.png", test_name, dataset_name.substr(s, l))).has_value()); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(ref_heights.begin(), ref_heights.end(), heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); const auto path = fmt::format("./diffs_{}_{}.png", test_name, dataset_name.substr(s, l)); - image::debugOut(height_diffs, path); + REQUIRE(image::debug_out(height_diffs, path).has_value()); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(ref_heights.begin(), ref_heights.end(), heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -246,12 +247,12 @@ TEST_CASE("reading") fmt::print("low res time: {}; high res time: {}\n", double(low_res_time) / 1000.0, double(high_res_time) / 1000.0); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debugOut(low_res_heights, fmt::format("./low_res_heights.png")); - image::debugOut(high_res_heights, fmt::format("./high_res_heights.png")); + REQUIRE(image::debug_out(low_res_heights, fmt::format("./low_res_heights.png")).has_value()); + REQUIRE(image::debug_out(high_res_heights, fmt::format("./high_res_heights.png")).has_value()); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debugOut(height_diffs, "./diff_low_res_high_res.png"); + REQUIRE(image::debug_out(height_diffs, "./diff_low_res_high_res.png").has_value()); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -288,12 +289,12 @@ TEST_CASE("reading") REQUIRE(high_res_heights.height() == render_height); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debugOut(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")); - image::debugOut(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")); + REQUIRE(image::debug_out(low_res_heights, fmt::format("./ov_with_warping_low_res_heights.png")).has_value()); + REQUIRE(image::debug_out(high_res_heights, fmt::format("./ov_with_warping_high_res_heights.png")).has_value()); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debugOut(height_diffs, "./ov_with_warping_diff_low_res_high_res.png"); + REQUIRE(image::debug_out(height_diffs, "./ov_with_warping_diff_low_res_high_res.png").has_value()); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { @@ -330,12 +331,12 @@ TEST_CASE("reading") REQUIRE(high_res_heights.height() == render_height); if (ALP_UNITTESTS_DEBUG_IMAGES) { - image::debugOut(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")); - image::debugOut(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")); + REQUIRE(image::debug_out(low_res_heights, fmt::format("./lowres_ov_with_warping_low_res_heights.png")).has_value()); + REQUIRE(image::debug_out(high_res_heights, fmt::format("./lowres_ov_with_warping_high_res_heights.png")).has_value()); auto height_diffs = radix::Raster({ render_width, render_height }); std::transform(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), height_diffs.begin(), [](auto a, auto b) { return std::abs(a - b); }); - image::debugOut(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png"); + REQUIRE(image::debug_out(height_diffs, "./lowres_ov_with_warping_diff_low_res_high_res.png").has_value()); } auto largest_abs_diff = 0.0; const auto mse = std::transform_reduce(low_res_heights.begin(), low_res_heights.end(), high_res_heights.begin(), 0.0, std::plus<>(), [&largest_abs_diff](auto a, auto b) { diff --git a/unittests/tilebuilder/image_writer.cpp b/unittests/tilebuilder/image_writer.cpp index aeddf2f2..8fc548f2 100644 --- a/unittests/tilebuilder/image_writer.cpp +++ b/unittests/tilebuilder/image_writer.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -38,7 +37,7 @@ TEST_CASE("tile builder writes radix rasters as PNG images") raster.pixel({ 0, 1 }) = { 0, 0, 255 }; raster.pixel({ 1, 1 }) = { 255, 255, 255 }; - image::saveImageAsPng(raster, output_path.string()); + REQUIRE(image::save_image_as_png(raster, output_path.string()).has_value()); const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); REQUIRE(decoded.rows == 2); @@ -55,7 +54,9 @@ TEST_CASE("tile builder handles image writer edge cases") { SECTION("empty debug raster is rejected") { - CHECK_THROWS_AS(image::debugOut(radix::Raster {}, "empty.png"), std::invalid_argument); + const auto result = image::debug_out(radix::Raster {}, "empty.png"); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::InvalidInput); } SECTION("constant debug raster is written as black") @@ -65,7 +66,7 @@ TEST_CASE("tile builder handles image writer edge cases") radix::Raster raster({ 2, 2 }); std::ranges::fill(raster, 42.F); - image::debugOut(raster, output_path.string()); + REQUIRE(image::debug_out(raster, output_path.string()).has_value()); const auto decoded = cv::imread(output_path.string(), cv::IMREAD_COLOR); REQUIRE(decoded.rows == 2); @@ -81,6 +82,8 @@ TEST_CASE("tile builder handles image writer edge cases") std::filesystem::remove_all(missing_directory); const auto output_path = missing_directory / "image.png"; - CHECK_THROWS_AS(image::saveImageAsPng(radix::Raster({ 1, 1 }), output_path.string()), std::runtime_error); + const auto result = image::save_image_as_png(radix::Raster({ 1, 1 }), output_path.string()); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().code() == Error::Code::Io); } }